diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/invidious.cr | 519 | ||||
| -rw-r--r-- | src/invidious/channels/channels.cr | 24 | ||||
| -rw-r--r-- | src/invidious/jobs/refresh_channels_job.cr | 2 | ||||
| -rw-r--r-- | src/invidious/routes/api/v1/authenticated.cr | 2 | ||||
| -rw-r--r-- | src/invidious/routes/notifications.cr | 78 | ||||
| -rw-r--r-- | src/invidious/routes/preferences.cr | 187 | ||||
| -rw-r--r-- | src/invidious/routes/subscriptions.cr | 168 | ||||
| -rw-r--r-- | src/invidious/routes/watch.cr | 66 | ||||
| -rw-r--r-- | src/invidious/users.cr | 2 | ||||
| -rw-r--r-- | src/invidious/views/components/player_sources.ecr | 30 | ||||
| -rw-r--r-- | src/invidious/views/embed.ecr | 4 | ||||
| -rw-r--r-- | src/invidious/views/licenses.ecr | 22 |
12 files changed, 561 insertions, 543 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index d3ad18bd..b09f31c2 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -114,6 +114,18 @@ LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) # Check table integrity Invidious::Database.check_integrity(CONFIG) +# Resolve player dependencies. This is done at compile time. +# +# Running the script by itself would show some colorful feedback while this doesn't. +# Perhaps we should just move the script to runtime in order to get that feedback? + +{% puts "\nChecking player dependencies...\n" %} +{% if flag?(:minified_player_dependencies) %} + {% puts run("../scripts/fetch-player-dependencies.cr", "--minified").stringify %} +{% else %} + {% puts run("../scripts/fetch-player-dependencies.cr").stringify %} +{% end %} + # Start jobs if CONFIG.channel_threads > 0 @@ -324,6 +336,7 @@ end end Invidious::Routing.get "/watch", Invidious::Routes::Watch, :handle + Invidious::Routing.post "/watch_ajax", Invidious::Routes::Watch, :mark_watched Invidious::Routing.get "/watch/:id", Invidious::Routes::Watch, :redirect Invidious::Routing.get "/shorts/:id", Invidious::Routes::Watch, :redirect Invidious::Routing.get "/w/:id", Invidious::Routes::Watch, :redirect @@ -357,6 +370,8 @@ end Invidious::Routing.get "/preferences", Invidious::Routes::PreferencesRoute, :show Invidious::Routing.post "/preferences", Invidious::Routes::PreferencesRoute, :update Invidious::Routing.get "/toggle_theme", Invidious::Routes::PreferencesRoute, :toggle_theme + Invidious::Routing.get "/data_control", Invidious::Routes::PreferencesRoute, :data_control + Invidious::Routing.post "/data_control", Invidious::Routes::PreferencesRoute, :update_data_control # Feeds Invidious::Routing.get "/view_all_playlists", Invidious::Routes::Feeds, :view_all_playlists_redirect @@ -375,6 +390,11 @@ end # Support push notifications via PubSubHubbub Invidious::Routing.get "/feed/webhook/:token", Invidious::Routes::Feeds, :push_notifications_get Invidious::Routing.post "/feed/webhook/:token", Invidious::Routes::Feeds, :push_notifications_post + + Invidious::Routing.get "/modify_notifications", Invidious::Routes::Notifications, :modify + + Invidious::Routing.post "/subscription_ajax", Invidious::Routes::Subscriptions, :toggle_subscription + Invidious::Routing.get "/subscription_manager", Invidious::Routes::Subscriptions, :subscription_manager {% end %} Invidious::Routing.get "/ggpht/*", Invidious::Routes::Images, :ggpht @@ -391,505 +411,6 @@ define_v1_api_routes() define_api_manifest_routes() define_video_playback_routes() -# Users - -post "/watch_ajax" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env, "/feed/subscriptions") - - redirect = env.params.query["redirect"]? - redirect ||= "true" - redirect = redirect == "true" - - if !user - if redirect - next env.redirect referer - else - next error_json(403, "No such user") - end - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - id = env.params.query["id"]? - if !id - env.response.status_code = 400 - next - end - - begin - validate_request(token, sid, env.request, HMAC_KEY, locale) - rescue ex - if redirect - next error_template(400, ex) - else - next error_json(400, ex) - end - end - - if env.params.query["action_mark_watched"]? - action = "action_mark_watched" - elsif env.params.query["action_mark_unwatched"]? - action = "action_mark_unwatched" - else - next env.redirect referer - end - - case action - when "action_mark_watched" - if !user.watched.includes? id - Invidious::Database::Users.mark_watched(user, id) - end - when "action_mark_unwatched" - Invidious::Database::Users.mark_unwatched(user, id) - else - next error_json(400, "Unsupported action #{action}") - end - - if redirect - env.redirect referer - else - env.response.content_type = "application/json" - "{}" - end -end - -# /modify_notifications -# will "ding" all subscriptions. -# /modify_notifications?receive_all_updates=false&receive_no_updates=false -# will "unding" all subscriptions. -get "/modify_notifications" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env, "/") - - redirect = env.params.query["redirect"]? - redirect ||= "false" - redirect = redirect == "true" - - if !user - if redirect - next env.redirect referer - else - next error_json(403, "No such user") - end - end - - user = user.as(User) - - if !user.password - channel_req = {} of String => String - - channel_req["receive_all_updates"] = env.params.query["receive_all_updates"]? || "true" - channel_req["receive_no_updates"] = env.params.query["receive_no_updates"]? || "" - channel_req["receive_post_updates"] = env.params.query["receive_post_updates"]? || "true" - - channel_req.reject! { |k, v| v != "true" && v != "false" } - - headers = HTTP::Headers.new - headers["Cookie"] = env.request.headers["Cookie"] - - html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers) - - cookies = HTTP::Cookies.from_client_headers(headers) - html.cookies.each do |cookie| - if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name - if cookies[cookie.name]? - cookies[cookie.name] = cookie - else - cookies << cookie - end - end - end - headers = cookies.add_request_headers(headers) - - if match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[^"]+)"/) - session_token = match["session_token"] - else - next env.redirect referer - end - - headers["content-type"] = "application/x-www-form-urlencoded" - channel_req["session_token"] = session_token - - subs = XML.parse_html(html.body) - subs.xpath_nodes(%q(//a[@class="subscription-title yt-uix-sessionlink"]/@href)).each do |channel| - channel_id = channel.content.lstrip("/channel/").not_nil! - channel_req["channel_id"] = channel_id - - YT_POOL.client &.post("/subscription_ajax?action_update_subscription_preferences=1", headers, form: channel_req) - end - end - - if redirect - env.redirect referer - else - env.response.content_type = "application/json" - "{}" - end -end - -post "/subscription_ajax" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env, "/") - - redirect = env.params.query["redirect"]? - redirect ||= "true" - redirect = redirect == "true" - - if !user - if redirect - next env.redirect referer - else - next error_json(403, "No such user") - end - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - begin - validate_request(token, sid, env.request, HMAC_KEY, locale) - rescue ex - if redirect - next error_template(400, ex) - else - next error_json(400, ex) - end - end - - if env.params.query["action_create_subscription_to_channel"]?.try &.to_i?.try &.== 1 - action = "action_create_subscription_to_channel" - elsif env.params.query["action_remove_subscriptions"]?.try &.to_i?.try &.== 1 - action = "action_remove_subscriptions" - else - next env.redirect referer - end - - channel_id = env.params.query["c"]? - channel_id ||= "" - - if !user.password - # Sync subscriptions with YouTube - subscribe_ajax(channel_id, action, env.request.headers) - end - - case action - when "action_create_subscription_to_channel" - if !user.subscriptions.includes? channel_id - get_channel(channel_id, false, false) - Invidious::Database::Users.subscribe_channel(user, channel_id) - end - when "action_remove_subscriptions" - Invidious::Database::Users.unsubscribe_channel(user, channel_id) - else - next error_json(400, "Unsupported action #{action}") - end - - if redirect - env.redirect referer - else - env.response.content_type = "application/json" - "{}" - end -end - -get "/subscription_manager" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - - if !user.password - # Refresh account - headers = HTTP::Headers.new - headers["Cookie"] = env.request.headers["Cookie"] - - user, sid = get_user(sid, headers) - end - - action_takeout = env.params.query["action_takeout"]?.try &.to_i? - action_takeout ||= 0 - action_takeout = action_takeout == 1 - - format = env.params.query["format"]? - format ||= "rss" - - subscriptions = Invidious::Database::Channels.select(user.subscriptions) - subscriptions.sort_by!(&.author.downcase) - - if action_takeout - if format == "json" - env.response.content_type = "application/json" - env.response.headers["content-disposition"] = "attachment" - playlists = Invidious::Database::Playlists.select_like_iv(user.email) - - next JSON.build do |json| - json.object do - json.field "subscriptions", user.subscriptions - json.field "watch_history", user.watched - json.field "preferences", user.preferences - json.field "playlists" do - json.array do - playlists.each do |playlist| - json.object do - json.field "title", playlist.title - json.field "description", html_to_content(playlist.description_html) - json.field "privacy", playlist.privacy.to_s - json.field "videos" do - json.array do - Invidious::Database::PlaylistVideos.select_ids(playlist.id, playlist.index, limit: 500).each do |video_id| - json.string video_id - end - end - end - end - end - end - end - end - end - else - env.response.content_type = "application/xml" - env.response.headers["content-disposition"] = "attachment" - export = XML.build do |xml| - xml.element("opml", version: "1.1") do - xml.element("body") do - if format == "newpipe" - title = "YouTube Subscriptions" - else - title = "Invidious Subscriptions" - end - - xml.element("outline", text: title, title: title) do - subscriptions.each do |channel| - if format == "newpipe" - xml_url = "https://www.youtube.com/feeds/videos.xml?channel_id=#{channel.id}" - else - xml_url = "#{HOST_URL}/feed/channel/#{channel.id}" - end - - xml.element("outline", text: channel.author, title: channel.author, - "type": "rss", xmlUrl: xml_url) - end - end - end - end - end - - next export.gsub(%(<?xml version="1.0"?>\n), "") - end - end - - templated "subscription_manager" -end - -get "/data_control" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - - templated "data_control" -end - -post "/data_control" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - referer = get_referer(env) - - if user - user = user.as(User) - - # TODO: Find a way to prevent browser timeout - - HTTP::FormData.parse(env.request) do |part| - body = part.body.gets_to_end - type = part.headers["Content-Type"] - - next if body.empty? - - # TODO: Unify into single import based on content-type - case part.name - when "import_invidious" - body = JSON.parse(body) - - if body["subscriptions"]? - user.subscriptions += body["subscriptions"].as_a.map(&.as_s) - user.subscriptions.uniq! - - user.subscriptions = get_batch_channels(user.subscriptions, false, false) - - Invidious::Database::Users.update_subscriptions(user) - end - - if body["watch_history"]? - user.watched += body["watch_history"].as_a.map(&.as_s) - user.watched.uniq! - Invidious::Database::Users.update_watch_history(user) - end - - if body["preferences"]? - user.preferences = Preferences.from_json(body["preferences"].to_json) - Invidious::Database::Users.update_preferences(user) - end - - if playlists = body["playlists"]?.try &.as_a? - playlists.each do |item| - title = item["title"]?.try &.as_s?.try &.delete("<>") - description = item["description"]?.try &.as_s?.try &.delete("\r") - privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy } - - next if !title - next if !description - next if !privacy - - playlist = create_playlist(title, privacy, user) - Invidious::Database::Playlists.update_description(playlist.id, description) - - videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| - raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500 - - video_id = video_id.try &.as_s? - next if !video_id - - begin - video = get_video(video_id) - rescue ex - next - end - - playlist_video = PlaylistVideo.new({ - title: video.title, - id: video.id, - author: video.author, - ucid: video.ucid, - length_seconds: video.length_seconds, - published: video.published, - plid: playlist.id, - live_now: video.live_now, - index: Random::Secure.rand(0_i64..Int64::MAX), - }) - - Invidious::Database::PlaylistVideos.insert(playlist_video) - Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index) - end - end - end - when "import_youtube" - filename = part.filename || "" - extension = filename.split(".").last - - if extension == "xml" || type == "application/xml" || type == "text/xml" - subscriptions = XML.parse(body) - user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| - channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] - end - elsif extension == "json" || type == "application/json" - subscriptions = JSON.parse(body) - user.subscriptions += subscriptions.as_a.compact_map do |entry| - entry["snippet"]["resourceId"]["channelId"].as_s - end - elsif extension == "csv" || type == "text/csv" - subscriptions = parse_subscription_export_csv(body) - user.subscriptions += subscriptions - else - halt(env, status_code: 415, - response: error_template(415, "Invalid subscription file uploaded") - ) - end - - user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, false, false) - - Invidious::Database::Users.update_subscriptions(user) - when "import_freetube" - user.subscriptions += body.scan(/"channelId":"(?<channel_id>[a-zA-Z0-9_-]{24})"/).map do |md| - md["channel_id"] - end - user.subscriptions.uniq! - - user.subscriptions = get_batch_channels(user.subscriptions, false, false) - - Invidious::Database::Users.update_subscriptions(user) - when "import_newpipe_subscriptions" - body = JSON.parse(body) - user.subscriptions += body["subscriptions"].as_a.compact_map do |channel| - if match = channel["url"].as_s.match(/\/channel\/(?<channel>UC[a-zA-Z0-9_-]{22})/) - next match["channel"] - elsif match = channel["url"].as_s.match(/\/user\/(?<user>.+)/) - response = YT_POOL.client &.get("/user/#{match["user"]}?disable_polymer=1&hl=en&gl=US") - html = XML.parse_html(response.body) - ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1] - next ucid if ucid - end - - nil - end - user.subscriptions.uniq! - - user.subscriptions = get_batch_channels(user.subscriptions, false, false) - - Invidious::Database::Users.update_subscriptions(user) - when "import_newpipe" - Compress::Zip::Reader.open(IO::Memory.new(body)) do |file| - file.each_entry do |entry| - if entry.filename == "newpipe.db" - tempfile = File.tempfile(".db") - File.write(tempfile.path, entry.io.gets_to_end) - db = DB.open("sqlite3://" + tempfile.path) - - user.watched += db.query_all("SELECT url FROM streams", as: String).map(&.lchop("https://www.youtube.com/watch?v=")) - user.watched.uniq! - - Invidious::Database::Users.update_watch_history(user) - - user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map(&.lchop("https://www.youtube.com/channel/")) - user.subscriptions.uniq! - - user.subscriptions = get_batch_channels(user.subscriptions, false, false) - - Invidious::Database::Users.update_subscriptions(user) - - db.close - tempfile.delete - end - end - end - else nil # Ignore - end - end - end - - env.redirect referer -end - get "/change_password" do |env| locale = env.get("preferences").as(Preferences).locale diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index 155ec559..6905b6f8 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -114,8 +114,9 @@ class ChannelRedirect < Exception end end -def get_batch_channels(channels, refresh = false, pull_all_videos = true, max_threads = 10) +def get_batch_channels(channels) finished_channel = Channel(String | Nil).new + max_threads = 10 spawn do active_threads = 0 @@ -130,7 +131,7 @@ def get_batch_channels(channels, refresh = false, pull_all_videos = true, max_th active_threads += 1 spawn do begin - get_channel(ucid, refresh, pull_all_videos) + get_channel(ucid) finished_channel.send(ucid) rescue ex finished_channel.send(nil) @@ -151,23 +152,20 @@ def get_batch_channels(channels, refresh = false, pull_all_videos = true, max_th return final end -def get_channel(id, refresh = true, pull_all_videos = true) - if channel = Invidious::Database::Channels.select(id) - if refresh && Time.utc - channel.updated > 10.minutes - channel = fetch_channel(id, pull_all_videos: pull_all_videos) - Invidious::Database::Channels.insert(channel, update_on_conflict: true) - end - else - channel = fetch_channel(id, pull_all_videos: pull_all_videos) - Invidious::Database::Channels.insert(channel) +def get_channel(id) : InvidiousChannel + channel = Invidious::Database::Channels.select(id) + + if channel.nil? || (Time.utc - channel.updated) > 2.days + channel = fetch_channel(id, pull_all_videos: false) + Invidious::Database::Channels.insert(channel, update_on_conflict: true) end return channel end -def fetch_channel(ucid, pull_all_videos = true, locale = nil) +def fetch_channel(ucid, pull_all_videos : Bool) LOGGER.debug("fetch_channel: #{ucid}") - LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}, locale = #{locale}") + LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}") LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed") rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr index 941089c1..55fb8154 100644 --- a/src/invidious/jobs/refresh_channels_job.cr +++ b/src/invidious/jobs/refresh_channels_job.cr @@ -30,7 +30,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob spawn do begin LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel") - channel = fetch_channel(id, CONFIG.full_refresh) + channel = fetch_channel(id, pull_all_videos: CONFIG.full_refresh) lim_fibers = max_fibers diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index fda655ef..4d0fe030 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -92,7 +92,7 @@ module Invidious::Routes::API::V1::Authenticated ucid = env.params.url["ucid"] if !user.subscriptions.includes? ucid - get_channel(ucid, false, false) + get_channel(ucid) Invidious::Database::Users.subscribe_channel(user, ucid) end diff --git a/src/invidious/routes/notifications.cr b/src/invidious/routes/notifications.cr new file mode 100644 index 00000000..272a3dc7 --- /dev/null +++ b/src/invidious/routes/notifications.cr @@ -0,0 +1,78 @@ +module Invidious::Routes::Notifications + # /modify_notifications + # will "ding" all subscriptions. + # /modify_notifications?receive_all_updates=false&receive_no_updates=false + # will "unding" all subscriptions. + def self.modify(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env, "/") + + redirect = env.params.query["redirect"]? + redirect ||= "false" + redirect = redirect == "true" + + if !user + if redirect + return env.redirect referer + else + return error_json(403, "No such user") + end + end + + user = user.as(User) + + if !user.password + channel_req = {} of String => String + + channel_req["receive_all_updates"] = env.params.query["receive_all_updates"]? || "true" + channel_req["receive_no_updates"] = env.params.query["receive_no_updates"]? || "" + channel_req["receive_post_updates"] = env.params.query["receive_post_updates"]? || "true" + + channel_req.reject! { |k, v| v != "true" && v != "false" } + + headers = HTTP::Headers.new + headers["Cookie"] = env.request.headers["Cookie"] + + html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers) + + cookies = HTTP::Cookies.from_client_headers(headers) + html.cookies.each do |cookie| + if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name + if cookies[cookie.name]? + cookies[cookie.name] = cookie + else + cookies << cookie + end + end + end + headers = cookies.add_request_headers(headers) + + if match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[^"]+)"/) + session_token = match["session_token"] + else + return env.redirect referer + end + + headers["content-type"] = "application/x-www-form-urlencoded" + channel_req["session_token"] = session_token + + subs = XML.parse_html(html.body) + subs.xpath_nodes(%q(//a[@class="subscription-title yt-uix-sessionlink"]/@href)).each do |channel| + channel_id = channel.content.lstrip("/channel/").not_nil! + channel_req["channel_id"] = channel_id + + YT_POOL.client &.post("/subscription_ajax?action_update_subscription_preferences=1", headers, form: channel_req) + end + end + + if redirect + env.redirect referer + else + env.response.content_type = "application/json" + "{}" + end + end +end diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index a832076c..9c740cf2 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -285,4 +285,191 @@ module Invidious::Routes::PreferencesRoute "{}" end end + + def self.data_control(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + + templated "data_control" + end + + def self.update_data_control(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + referer = get_referer(env) + + if user + user = user.as(User) + + # TODO: Find a way to prevent browser timeout + + HTTP::FormData.parse(env.request) do |part| + body = part.body.gets_to_end + type = part.headers["Content-Type"] + + next if body.empty? + + # TODO: Unify into single import based on content-type + case part.name + when "import_invidious" + body = JSON.parse(body) + + if body["subscriptions"]? + user.subscriptions += body["subscriptions"].as_a.map(&.as_s) + user.subscriptions.uniq! + + user.subscriptions = get_batch_channels(user.subscriptions) + + Invidious::Database::Users.update_subscriptions(user) + end + + if body["watch_history"]? + user.watched += body["watch_history"].as_a.map(&.as_s) + user.watched.uniq! + Invidious::Database::Users.update_watch_history(user) + end + + if body["preferences"]? + user.preferences = Preferences.from_json(body["preferences"].to_json) + Invidious::Database::Users.update_preferences(user) + end + + if playlists = body["playlists"]?.try &.as_a? + playlists.each do |item| + title = item["title"]?.try &.as_s?.try &.delete("<>") + description = item["description"]?.try &.as_s?.try &.delete("\r") + privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy } + + next if !title + next if !description + next if !privacy + + playlist = create_playlist(title, privacy, user) + Invidious::Database::Playlists.update_description(playlist.id, description) + + videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| + raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500 + + video_id = video_id.try &.as_s? + next if !video_id + + begin + video = get_video(video_id) + rescue ex + next + end + + playlist_video = PlaylistVideo.new({ + title: video.title, + id: video.id, + author: video.author, + ucid: video.ucid, + length_seconds: video.length_seconds, + published: video.published, + plid: playlist.id, + live_now: video.live_now, + index: Random::Secure.rand(0_i64..Int64::MAX), + }) + + Invidious::Database::PlaylistVideos.insert(playlist_video) + Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index) + end + end + end + when "import_youtube" + filename = part.filename || "" + extension = filename.split(".").last + + if extension == "xml" || type == "application/xml" || type == "text/xml" + subscriptions = XML.parse(body) + user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| + channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] + end + elsif extension == "json" || type == "application/json" + subscriptions = JSON.parse(body) + user.subscriptions += subscriptions.as_a.compact_map do |entry| + entry["snippet"]["resourceId"]["channelId"].as_s + end + elsif extension == "csv" || type == "text/csv" + subscriptions = parse_subscription_export_csv(body) + user.subscriptions += subscriptions + else + haltf(env, status_code: 415, + response: error_template(415, "Invalid subscription file uploaded") + ) + end + + user.subscriptions.uniq! + user.subscriptions = get_batch_channels(user.subscriptions) + + Invidious::Database::Users.update_subscriptions(user) + when "import_freetube" + user.subscriptions += body.scan(/"channelId":"(?<channel_id>[a-zA-Z0-9_-]{24})"/).map do |md| + md["channel_id"] + end + user.subscriptions.uniq! + + user.subscriptions = get_batch_channels(user.subscriptions) + + Invidious::Database::Users.update_subscriptions(user) + when "import_newpipe_subscriptions" + body = JSON.parse(body) + user.subscriptions += body["subscriptions"].as_a.compact_map do |channel| + if match = channel["url"].as_s.match(/\/channel\/(?<channel>UC[a-zA-Z0-9_-]{22})/) + next match["channel"] + elsif match = channel["url"].as_s.match(/\/user\/(?<user>.+)/) + response = YT_POOL.client &.get("/user/#{match["user"]}?disable_polymer=1&hl=en&gl=US") + html = XML.parse_html(response.body) + ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1] + next ucid if ucid + end + + nil + end + user.subscriptions.uniq! + + user.subscriptions = get_batch_channels(user.subscriptions) + + Invidious::Database::Users.update_subscriptions(user) + when "import_newpipe" + Compress::Zip::Reader.open(IO::Memory.new(body)) do |file| + file.each_entry do |entry| + if entry.filename == "newpipe.db" + tempfile = File.tempfile(".db") + File.write(tempfile.path, entry.io.gets_to_end) + db = DB.open("sqlite3://" + tempfile.path) + + user.watched += db.query_all("SELECT url FROM streams", as: String).map(&.lchop("https://www.youtube.com/watch?v=")) + user.watched.uniq! + + Invidious::Database::Users.update_watch_history(user) + + user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map(&.lchop("https://www.youtube.com/channel/")) + user.subscriptions.uniq! + + user.subscriptions = get_batch_channels(user.subscriptions) + + Invidious::Database::Users.update_subscriptions(user) + + db.close + tempfile.delete + end + end + end + else nil # Ignore + end + end + end + + env.redirect referer + end end diff --git a/src/invidious/routes/subscriptions.cr b/src/invidious/routes/subscriptions.cr new file mode 100644 index 00000000..ec8fe67b --- /dev/null +++ b/src/invidious/routes/subscriptions.cr @@ -0,0 +1,168 @@ +module Invidious::Routes::Subscriptions + def self.toggle_subscription(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env, "/") + + redirect = env.params.query["redirect"]? + redirect ||= "true" + redirect = redirect == "true" + + if !user + if redirect + return env.redirect referer + else + return error_json(403, "No such user") + end + end + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + begin + validate_request(token, sid, env.request, HMAC_KEY, locale) + rescue ex + if redirect + return error_template(400, ex) + else + return error_json(400, ex) + end + end + + if env.params.query["action_create_subscription_to_channel"]?.try &.to_i?.try &.== 1 + action = "action_create_subscription_to_channel" + elsif env.params.query["action_remove_subscriptions"]?.try &.to_i?.try &.== 1 + action = "action_remove_subscriptions" + else + return env.redirect referer + end + + channel_id = env.params.query["c"]? + channel_id ||= "" + + if !user.password + # Sync subscriptions with YouTube + subscribe_ajax(channel_id, action, env.request.headers) + end + + case action + when "action_create_subscription_to_channel" + if !user.subscriptions.includes? channel_id + get_channel(channel_id) + Invidious::Database::Users.subscribe_channel(user, channel_id) + end + when "action_remove_subscriptions" + Invidious::Database::Users.unsubscribe_channel(user, channel_id) + else + return error_json(400, "Unsupported action #{action}") + end + + if redirect + env.redirect referer + else + env.response.content_type = "application/json" + "{}" + end + end + + def self.subscription_manager(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + sid = sid.as(String) + + if !user.password + # Refresh account + headers = HTTP::Headers.new + headers["Cookie"] = env.request.headers["Cookie"] + + user, sid = get_user(sid, headers) + end + + action_takeout = env.params.query["action_takeout"]?.try &.to_i? + action_takeout ||= 0 + action_takeout = action_takeout == 1 + + format = env.params.query["format"]? + format ||= "rss" + + subscriptions = Invidious::Database::Channels.select(user.subscriptions) + subscriptions.sort_by!(&.author.downcase) + + if action_takeout + if format == "json" + env.response.content_type = "application/json" + env.response.headers["content-disposition"] = "attachment" + playlists = Invidious::Database::Playlists.select_like_iv(user.email) + + return JSON.build do |json| + json.object do + json.field "subscriptions", user.subscriptions + json.field "watch_history", user.watched + json.field "preferences", user.preferences + json.field "playlists" do + json.array do + playlists.each do |playlist| + json.object do + json.field "title", playlist.title + json.field "description", html_to_content(playlist.description_html) + json.field "privacy", playlist.privacy.to_s + json.field "videos" do + json.array do + Invidious::Database::PlaylistVideos.select_ids(playlist.id, playlist.index, limit: 500).each do |video_id| + json.string video_id + end + end + end + end + end + end + end + end + end + else + env.response.content_type = "application/xml" + env.response.headers["content-disposition"] = "attachment" + export = XML.build do |xml| + xml.element("opml", version: "1.1") do + xml.element("body") do + if format == "newpipe" + title = "YouTube Subscriptions" + else + title = "Invidious Subscriptions" + end + + xml.element("outline", text: title, title: title) do + subscriptions.each do |channel| + if format == "newpipe" + xml_url = "https://www.youtube.com/feeds/videos.xml?channel_id=#{channel.id}" + else + xml_url = "#{HOST_URL}/feed/channel/#{channel.id}" + end + + xml.element("outline", text: channel.author, title: channel.author, + "type": "rss", xmlUrl: xml_url) + end + end + end + end + end + + return export.gsub(%(<?xml version="1.0"?>\n), "") + end + end + + templated "subscription_manager" + end +end diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 1198f48f..7d048ce8 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -200,4 +200,70 @@ module Invidious::Routes::Watch return env.redirect url end + + def self.mark_watched(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env, "/feed/subscriptions") + + redirect = env.params.query["redirect"]? + redirect ||= "true" + redirect = redirect == "true" + + if !user + if redirect + return env.redirect referer + else + return error_json(403, "No such user") + end + end + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + id = env.params.query["id"]? + if !id + env.response.status_code = 400 + return + end + + begin + validate_request(token, sid, env.request, HMAC_KEY, locale) + rescue ex + if redirect + return error_template(400, ex) + else + return error_json(400, ex) + end + end + + if env.params.query["action_mark_watched"]? + action = "action_mark_watched" + elsif env.params.query["action_mark_unwatched"]? + action = "action_mark_unwatched" + else + return env.redirect referer + end + + case action + when "action_mark_watched" + if !user.watched.includes? id + Invidious::Database::Users.mark_watched(user, id) + end + when "action_mark_unwatched" + Invidious::Database::Users.mark_unwatched(user, id) + else + return error_json(400, "Unsupported action #{action}") + end + + if redirect + env.redirect referer + else + env.response.content_type = "application/json" + "{}" + end + end end diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 49074994..a7ee72a9 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -74,7 +74,7 @@ def fetch_user(sid, headers) end end - channels = get_batch_channels(channels, false, false) + channels = get_batch_channels(channels) email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"])) if email diff --git a/src/invidious/views/components/player_sources.ecr b/src/invidious/views/components/player_sources.ecr index 0d97d35a..9af3899c 100644 --- a/src/invidious/views/components/player_sources.ecr +++ b/src/invidious/views/components/player_sources.ecr @@ -1,18 +1,18 @@ -<link rel="stylesheet" href="/css/video-js.min.css?v=<%= ASSET_COMMIT %>"> -<link rel="stylesheet" href="/css/videojs-http-source-selector.css?v=<%= ASSET_COMMIT %>"> -<link rel="stylesheet" href="/css/videojs.markers.min.css?v=<%= ASSET_COMMIT %>"> -<link rel="stylesheet" href="/css/videojs-share.css?v=<%= ASSET_COMMIT %>"> -<link rel="stylesheet" href="/css/videojs-vtt-thumbnails.css?v=<%= ASSET_COMMIT %>"> -<link rel="stylesheet" href="/css/videojs-mobile-ui.css?v=<%= ASSET_COMMIT %>"> +<link rel="stylesheet" href="/videojs/video.js/video-js.css?v=<%= ASSET_COMMIT %>"> +<link rel="stylesheet" href="/videojs/videojs-http-source-selector/videojs-http-source-selector.css?v=<%= ASSET_COMMIT %>"> +<link rel="stylesheet" href="/videojs/videojs-markers/videojs.markers.css?v=<%= ASSET_COMMIT %>"> +<link rel="stylesheet" href="/videojs/videojs-share/videojs-share.css?v=<%= ASSET_COMMIT %>"> +<link rel="stylesheet" href="/videojs/videojs-vtt-thumbnails/videojs-vtt-thumbnails.css?v=<%= ASSET_COMMIT %>"> +<link rel="stylesheet" href="/videojs/videojs-mobile-ui/videojs-mobile-ui.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/player.css?v=<%= ASSET_COMMIT %>"> -<script src="/js/video.min.js?v=<%= ASSET_COMMIT %>"></script> -<script src="/js/videojs-mobile-ui.min.js?v=<%= ASSET_COMMIT %>"></script> -<script src="/js/videojs-contrib-quality-levels.min.js?v=<%= ASSET_COMMIT %>"></script> -<script src="/js/videojs-http-source-selector.min.js?v=<%= ASSET_COMMIT %>"></script> -<script src="/js/videojs-markers.min.js?v=<%= ASSET_COMMIT %>"></script> -<script src="/js/videojs-share.min.js?v=<%= ASSET_COMMIT %>"></script> -<script src="/js/videojs-vtt-thumbnails.min.js?v=<%= ASSET_COMMIT %>"></script> +<script src="/videojs/video.js/video.js?v=<%= ASSET_COMMIT %>"></script> +<script src="/videojs/videojs-mobile-ui/videojs-mobile-ui.js?v=<%= ASSET_COMMIT %>"></script> +<script src="/videojs/videojs-contrib-quality-levels/videojs-contrib-quality-levels.js?v=<%= ASSET_COMMIT %>"></script> +<script src="/videojs/videojs-http-source-selector/videojs-http-source-selector.js?v=<%= ASSET_COMMIT %>"></script> +<script src="/videojs/videojs-markers/videojs-markers.js?v=<%= ASSET_COMMIT %>"></script> +<script src="/videojs/videojs-share/videojs-share.js?v=<%= ASSET_COMMIT %>"></script> +<script src="/videojs/videojs-vtt-thumbnails/videojs-vtt-thumbnails.js?v=<%= ASSET_COMMIT %>"></script> <% if params.annotations %> @@ -26,6 +26,6 @@ <% end %> <% if !params.listen && params.vr_mode %> - <link rel="stylesheet" href="/css/videojs-vr.css?v=<%= ASSET_COMMIT %>"> - <script src="/js/videojs-vr.js?v=<%= ASSET_COMMIT %>"></script> + <link rel="stylesheet" href="/videojs/videojs-vr/videojs-vr.css?v=<%= ASSET_COMMIT %>"> + <script src="/videojs/videojs-vr/videojs-vr.js?v=<%= ASSET_COMMIT %>"></script> <% end %> diff --git a/src/invidious/views/embed.ecr b/src/invidious/views/embed.ecr index dbb86009..cd0fd0d5 100644 --- a/src/invidious/views/embed.ecr +++ b/src/invidious/views/embed.ecr @@ -6,8 +6,8 @@ <meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="thumbnail" content="<%= thumbnail %>"> <%= rendered "components/player_sources" %> - <link rel="stylesheet" href="/css/videojs-overlay.css?v=<%= ASSET_COMMIT %>"> - <script src="/js/videojs-overlay.min.js?v=<%= ASSET_COMMIT %>"></script> + <link rel="stylesheet" href="/videojs/videojs-overlay/videojs-overlay.css?v=<%= ASSET_COMMIT %>"> + <script src="videojs/videojs-overlay/videojs-overlay.js?v=<%= ASSET_COMMIT %>"></script> <link rel="stylesheet" href="/css/default.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/embed.css?v=<%= ASSET_COMMIT %>"> <title><%= HTML.escape(video.title) %> - Invidious</title> diff --git a/src/invidious/views/licenses.ecr b/src/invidious/views/licenses.ecr index 9f5bcbdd..861913d0 100644 --- a/src/invidious/views/licenses.ecr +++ b/src/invidious/views/licenses.ecr @@ -75,7 +75,7 @@ </td> <td> - <a href="https://github.com/omarroth/videojs-quality-selector"><%= translate(locale, "source") %></a> + <a href="https://github.com/iv-org/videojs-quality-selector"><%= translate(locale, "source") %></a> </td> </tr> @@ -123,7 +123,7 @@ <tr> <td> - <a href="/js/videojs-contrib-quality-levels.min.js?v=<%= ASSET_COMMIT %>">videojs-contrib-quality-levels.min.js</a> + <a href="/videojs/videojs-contrib-quality-levels/videojs-contrib-quality-levels.js?v=<%= ASSET_COMMIT %>">videojs-contrib-quality-levels.js</a> </td> <td> @@ -137,7 +137,7 @@ <tr> <td> - <a href="/js/videojs-http-source-selector.min.js?v=<%= ASSET_COMMIT %>">videojs-http-source-selector.min.js</a> + <a href="/videojs/videojs-http-source-selector/videojs-http-source-selector.js?v=<%= ASSET_COMMIT %>">videojs-http-source-selector.js</a> </td> <td> @@ -151,7 +151,7 @@ <tr> <td> - <a href="/js/videojs-mobile-ui.min.js?v=<%= ASSET_COMMIT %>">videojs-mobile-ui.min.js</a> + <a href="/videojs/videojs-mobile-ui/videojs-mobile-ui.js?v=<%= ASSET_COMMIT %>">videojs-mobile-ui.js</a> </td> <td> @@ -165,7 +165,7 @@ <tr> <td> - <a href="/js/videojs-markers.min.js?v=<%= ASSET_COMMIT %>">videojs-markers.min.js</a> + <a href="/videojs/videojs-markers/videojs-markers.js?v=<%= ASSET_COMMIT %>">videojs-markers.js</a> </td> <td> @@ -179,7 +179,7 @@ <tr> <td> - <a href="/js/videojs-overlay.min.js?v=<%= ASSET_COMMIT %>">videojs-overlay.min.js</a> + <a href="/videojs/videojs-overlay/videojs-overlay.js?v=<%= ASSET_COMMIT %>">videojs-overlay.js</a> </td> <td> @@ -193,7 +193,7 @@ <tr> <td> - <a href="/js/videojs-share.min.js?v=<%= ASSET_COMMIT %>">videojs-share.min.js</a> + <a href="/videojs/videojs-share/videojs-share.js?v=<%= ASSET_COMMIT %>">videojs-share.js</a> </td> <td> @@ -207,7 +207,7 @@ <tr> <td> - <a href="/js/videojs-vtt-thumbnails.min.js?v=<%= ASSET_COMMIT %>">videojs-vtt-thumbnails.min.js</a> + <a href="/videojs/videojs-vtt-thumbnails/videojs-vtt-thumbnails.js?v=<%= ASSET_COMMIT %>">videojs-vtt-thumbnails.js</a> </td> <td> @@ -215,7 +215,7 @@ </td> <td> - <a href="https://github.com/omarroth/videojs-vtt-thumbnails"><%= translate(locale, "source") %></a> + <a href="https://github.com/chrisboustead/videojs-vtt-thumbnails"><%= translate(locale, "source") %></a> </td> </tr> @@ -235,7 +235,7 @@ <tr> <td> - <a href="/js/videojs-vr.js?v=<%= ASSET_COMMIT %>">videojs-vr.js</a> + <a href="/videojs/videojs-vr/videojs-vr.js?v=<%= ASSET_COMMIT %>">videojs-vr.js</a> </td> <td> @@ -249,7 +249,7 @@ <tr> <td> - <a href="/js/video.min.js?v=<%= ASSET_COMMIT %>">video.min.js</a> + <a href="/videojs/video.js/video.js?v=<%= ASSET_COMMIT %>">video.js</a> </td> <td> |