diff options
Diffstat (limited to 'src')
88 files changed, 5160 insertions, 2841 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index e29b73a8..8ba62503 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -20,15 +20,18 @@ require "kemal" require "athena-negotiation" require "openssl/hmac" require "option_parser" -require "pg" require "sqlite3" require "xml" require "yaml" require "compress/zip" require "protodec/utils" + +require "./invidious/database/*" require "./invidious/helpers/*" +require "./invidious/yt_backend/*" require "./invidious/*" require "./invidious/channels/*" +require "./invidious/user/*" require "./invidious/routes/**" require "./invidious/jobs/**" @@ -67,7 +70,7 @@ SOFTWARE = { "branch" => "#{CURRENT_BRANCH}", } -YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size, timeout: 2.0, use_quic: CONFIG.use_quic) +YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size, use_quic: CONFIG.use_quic) # CLI Kemal.config.extra_options do |parser| @@ -110,19 +113,19 @@ LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) # Check table integrity if CONFIG.check_tables - check_enum(PG_DB, "privacy", PlaylistPrivacy) + Invidious::Database.check_enum(PG_DB, "privacy", PlaylistPrivacy) - check_table(PG_DB, "channels", InvidiousChannel) - check_table(PG_DB, "channel_videos", ChannelVideo) - check_table(PG_DB, "playlists", InvidiousPlaylist) - check_table(PG_DB, "playlist_videos", PlaylistVideo) - check_table(PG_DB, "nonces", Nonce) - check_table(PG_DB, "session_ids", SessionId) - check_table(PG_DB, "users", User) - check_table(PG_DB, "videos", Video) + Invidious::Database.check_table(PG_DB, "channels", InvidiousChannel) + Invidious::Database.check_table(PG_DB, "channel_videos", ChannelVideo) + Invidious::Database.check_table(PG_DB, "playlists", InvidiousPlaylist) + Invidious::Database.check_table(PG_DB, "playlist_videos", PlaylistVideo) + Invidious::Database.check_table(PG_DB, "nonces", Nonce) + Invidious::Database.check_table(PG_DB, "session_ids", SessionId) + Invidious::Database.check_table(PG_DB, "users", User) + Invidious::Database.check_table(PG_DB, "videos", Video) if CONFIG.cache_annotations - check_table(PG_DB, "annotations", Annotation) + Invidious::Database.check_table(PG_DB, "annotations", Annotation) end end @@ -165,10 +168,6 @@ if CONFIG.popular_enabled Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB) end -if CONFIG.captcha_key - Invidious::Jobs.register Invidious::Jobs::BypassCaptchaJob.new -end - connection_channel = Channel({Bool, Channel(PQ::Notification)}).new(32) Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(connection_channel, CONFIG.database_url) @@ -260,8 +259,8 @@ before_all do |env| # Invidious users only have SID if !env.request.cookies.has_key? "SSID" - if email = PG_DB.query_one?("SELECT email FROM session_ids WHERE id = $1", sid, as: String) - user = PG_DB.query_one("SELECT * FROM users WHERE email = $1", email, as: User) + if email = Invidious::Database::SessionIDs.select_email(sid) + user = Invidious::Database::Users.select!(email: email) csrf_token = generate_response(sid, { ":authorize_token", ":playlist_ajax", @@ -269,7 +268,7 @@ before_all do |env| ":subscription_ajax", ":token_ajax", ":watch_ajax", - }, HMAC_KEY, PG_DB, 1.week) + }, HMAC_KEY, 1.week) preferences = user.preferences env.set "preferences", preferences @@ -283,7 +282,7 @@ before_all do |env| headers["Cookie"] = env.request.headers["Cookie"] begin - user, sid = get_user(sid, headers, PG_DB, false) + user, sid = get_user(sid, headers, false) csrf_token = generate_response(sid, { ":authorize_token", ":playlist_ajax", @@ -291,7 +290,7 @@ before_all do |env| ":subscription_ajax", ":token_ajax", ":watch_ajax", - }, HMAC_KEY, PG_DB, 1.week) + }, HMAC_KEY, 1.week) preferences = user.preferences env.set "preferences", preferences @@ -328,80 +327,97 @@ before_all do |env| env.set "current_page", URI.encode_www_form(current_page) end -Invidious::Routing.get "/", Invidious::Routes::Misc, :home -Invidious::Routing.get "/privacy", Invidious::Routes::Misc, :privacy -Invidious::Routing.get "/licenses", Invidious::Routes::Misc, :licenses - -Invidious::Routing.get "/channel/:ucid", Invidious::Routes::Channels, :home -Invidious::Routing.get "/channel/:ucid/home", Invidious::Routes::Channels, :home -Invidious::Routing.get "/channel/:ucid/videos", Invidious::Routes::Channels, :videos -Invidious::Routing.get "/channel/:ucid/playlists", Invidious::Routes::Channels, :playlists -Invidious::Routing.get "/channel/:ucid/community", Invidious::Routes::Channels, :community -Invidious::Routing.get "/channel/:ucid/about", Invidious::Routes::Channels, :about - -["", "/videos", "/playlists", "/community", "/about"].each do |path| - # /c/LinusTechTips - Invidious::Routing.get "/c/:user#{path}", Invidious::Routes::Channels, :brand_redirect - # /user/linustechtips | Not always the same as /c/ - Invidious::Routing.get "/user/:user#{path}", Invidious::Routes::Channels, :brand_redirect - # /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow - Invidious::Routing.get "/attribution_link#{path}", Invidious::Routes::Channels, :brand_redirect - # /profile?user=linustechtips - Invidious::Routing.get "/profile/#{path}", Invidious::Routes::Channels, :profile -end +{% unless flag?(:api_only) %} + Invidious::Routing.get "/", Invidious::Routes::Misc, :home + Invidious::Routing.get "/privacy", Invidious::Routes::Misc, :privacy + Invidious::Routing.get "/licenses", Invidious::Routes::Misc, :licenses + + Invidious::Routing.get "/channel/:ucid", Invidious::Routes::Channels, :home + Invidious::Routing.get "/channel/:ucid/home", Invidious::Routes::Channels, :home + Invidious::Routing.get "/channel/:ucid/videos", Invidious::Routes::Channels, :videos + Invidious::Routing.get "/channel/:ucid/playlists", Invidious::Routes::Channels, :playlists + Invidious::Routing.get "/channel/:ucid/community", Invidious::Routes::Channels, :community + Invidious::Routing.get "/channel/:ucid/about", Invidious::Routes::Channels, :about + + ["", "/videos", "/playlists", "/community", "/about"].each do |path| + # /c/LinusTechTips + Invidious::Routing.get "/c/:user#{path}", Invidious::Routes::Channels, :brand_redirect + # /user/linustechtips | Not always the same as /c/ + Invidious::Routing.get "/user/:user#{path}", Invidious::Routes::Channels, :brand_redirect + # /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow + Invidious::Routing.get "/attribution_link#{path}", Invidious::Routes::Channels, :brand_redirect + # /profile?user=linustechtips + Invidious::Routing.get "/profile/#{path}", Invidious::Routes::Channels, :profile + end + + Invidious::Routing.get "/watch", Invidious::Routes::Watch, :handle + Invidious::Routing.post "/watch_ajax", Invidious::Routes::Watch, :mark_watched + Invidious::Routing.get "/watch/:id", Invidious::Routes::Watch, :redirect + Invidious::Routing.get "/shorts/:id", Invidious::Routes::Watch, :redirect + Invidious::Routing.get "/w/:id", Invidious::Routes::Watch, :redirect + Invidious::Routing.get "/v/:id", Invidious::Routes::Watch, :redirect + Invidious::Routing.get "/e/:id", Invidious::Routes::Watch, :redirect + Invidious::Routing.get "/redirect", Invidious::Routes::Misc, :cross_instance_redirect + + Invidious::Routing.get "/embed/", Invidious::Routes::Embed, :redirect + Invidious::Routing.get "/embed/:id", Invidious::Routes::Embed, :show + + Invidious::Routing.get "/create_playlist", Invidious::Routes::Playlists, :new + Invidious::Routing.post "/create_playlist", Invidious::Routes::Playlists, :create + Invidious::Routing.get "/subscribe_playlist", Invidious::Routes::Playlists, :subscribe + Invidious::Routing.get "/delete_playlist", Invidious::Routes::Playlists, :delete_page + Invidious::Routing.post "/delete_playlist", Invidious::Routes::Playlists, :delete + Invidious::Routing.get "/edit_playlist", Invidious::Routes::Playlists, :edit + Invidious::Routing.post "/edit_playlist", Invidious::Routes::Playlists, :update + Invidious::Routing.get "/add_playlist_items", Invidious::Routes::Playlists, :add_playlist_items_page + Invidious::Routing.post "/playlist_ajax", Invidious::Routes::Playlists, :playlist_ajax + Invidious::Routing.get "/playlist", Invidious::Routes::Playlists, :show + Invidious::Routing.get "/mix", Invidious::Routes::Playlists, :mix + + Invidious::Routing.get "/opensearch.xml", Invidious::Routes::Search, :opensearch + Invidious::Routing.get "/results", Invidious::Routes::Search, :results + Invidious::Routing.get "/search", Invidious::Routes::Search, :search + + Invidious::Routing.get "/login", Invidious::Routes::Login, :login_page + Invidious::Routing.post "/login", Invidious::Routes::Login, :login + Invidious::Routing.post "/signout", Invidious::Routes::Login, :signout + + Invidious::Routing.get "/preferences", Invidious::Routes::PreferencesRoute, :show + Invidious::Routing.post "/preferences", Invidious::Routes::PreferencesRoute, :update + Invidious::Routing.get "/toggle_theme", Invidious::Routes::PreferencesRoute, :toggle_theme + Invidious::Routing.get "/data_control", Invidious::Routes::PreferencesRoute, :data_control + Invidious::Routing.post "/data_control", Invidious::Routes::PreferencesRoute, :update_data_control + + # Feeds + Invidious::Routing.get "/view_all_playlists", Invidious::Routes::Feeds, :view_all_playlists_redirect + Invidious::Routing.get "/feed/playlists", Invidious::Routes::Feeds, :playlists + Invidious::Routing.get "/feed/popular", Invidious::Routes::Feeds, :popular + Invidious::Routing.get "/feed/trending", Invidious::Routes::Feeds, :trending + Invidious::Routing.get "/feed/subscriptions", Invidious::Routes::Feeds, :subscriptions + Invidious::Routing.get "/feed/history", Invidious::Routes::Feeds, :history + + # RSS Feeds + Invidious::Routing.get "/feed/channel/:ucid", Invidious::Routes::Feeds, :rss_channel + Invidious::Routing.get "/feed/private", Invidious::Routes::Feeds, :rss_private + Invidious::Routing.get "/feed/playlist/:plid", Invidious::Routes::Feeds, :rss_playlist + Invidious::Routing.get "/feeds/videos.xml", Invidious::Routes::Feeds, :rss_videos + + # Support push notifications via PubSubHubbub + Invidious::Routing.get "/feed/webhook/:token", Invidious::Routes::Feeds, :push_notifications_get + Invidious::Routing.post "/feed/webhook/:token", Invidious::Routes::Feeds, :push_notifications_post + + Invidious::Routing.get "/modify_notifications", Invidious::Routes::Notifications, :modify + + Invidious::Routing.post "/subscription_ajax", Invidious::Routes::Subscriptions, :toggle_subscription + Invidious::Routing.get "/subscription_manager", Invidious::Routes::Subscriptions, :subscription_manager +{% end %} -Invidious::Routing.get "/watch", Invidious::Routes::Watch, :handle -Invidious::Routing.get "/watch/:id", Invidious::Routes::Watch, :redirect -Invidious::Routing.get "/shorts/:id", Invidious::Routes::Watch, :redirect -Invidious::Routing.get "/w/:id", Invidious::Routes::Watch, :redirect -Invidious::Routing.get "/v/:id", Invidious::Routes::Watch, :redirect -Invidious::Routing.get "/e/:id", Invidious::Routes::Watch, :redirect -Invidious::Routing.get "/redirect", Invidious::Routes::Misc, :cross_instance_redirect - -Invidious::Routing.get "/embed/", Invidious::Routes::Embed, :redirect -Invidious::Routing.get "/embed/:id", Invidious::Routes::Embed, :show - -Invidious::Routing.get "/create_playlist", Invidious::Routes::Playlists, :new -Invidious::Routing.post "/create_playlist", Invidious::Routes::Playlists, :create -Invidious::Routing.get "/subscribe_playlist", Invidious::Routes::Playlists, :subscribe -Invidious::Routing.get "/delete_playlist", Invidious::Routes::Playlists, :delete_page -Invidious::Routing.post "/delete_playlist", Invidious::Routes::Playlists, :delete -Invidious::Routing.get "/edit_playlist", Invidious::Routes::Playlists, :edit -Invidious::Routing.post "/edit_playlist", Invidious::Routes::Playlists, :update -Invidious::Routing.get "/add_playlist_items", Invidious::Routes::Playlists, :add_playlist_items_page -Invidious::Routing.post "/playlist_ajax", Invidious::Routes::Playlists, :playlist_ajax -Invidious::Routing.get "/playlist", Invidious::Routes::Playlists, :show -Invidious::Routing.get "/mix", Invidious::Routes::Playlists, :mix - -Invidious::Routing.get "/opensearch.xml", Invidious::Routes::Search, :opensearch -Invidious::Routing.get "/results", Invidious::Routes::Search, :results -Invidious::Routing.get "/search", Invidious::Routes::Search, :search - -Invidious::Routing.get "/login", Invidious::Routes::Login, :login_page -Invidious::Routing.post "/login", Invidious::Routes::Login, :login -Invidious::Routing.post "/signout", Invidious::Routes::Login, :signout - -Invidious::Routing.get "/preferences", Invidious::Routes::PreferencesRoute, :show -Invidious::Routing.post "/preferences", Invidious::Routes::PreferencesRoute, :update -Invidious::Routing.get "/toggle_theme", Invidious::Routes::PreferencesRoute, :toggle_theme - -# Feeds -Invidious::Routing.get "/view_all_playlists", Invidious::Routes::Feeds, :view_all_playlists_redirect -Invidious::Routing.get "/feed/playlists", Invidious::Routes::Feeds, :playlists -Invidious::Routing.get "/feed/popular", Invidious::Routes::Feeds, :popular -Invidious::Routing.get "/feed/trending", Invidious::Routes::Feeds, :trending -Invidious::Routing.get "/feed/subscriptions", Invidious::Routes::Feeds, :subscriptions -Invidious::Routing.get "/feed/history", Invidious::Routes::Feeds, :history - -# RSS Feeds -Invidious::Routing.get "/feed/channel/:ucid", Invidious::Routes::Feeds, :rss_channel -Invidious::Routing.get "/feed/private", Invidious::Routes::Feeds, :rss_private -Invidious::Routing.get "/feed/playlist/:plid", Invidious::Routes::Feeds, :rss_playlist -Invidious::Routing.get "/feeds/videos.xml", Invidious::Routes::Feeds, :rss_videos - -# Support push notifications via PubSubHubbub -Invidious::Routing.get "/feed/webhook/:token", Invidious::Routes::Feeds, :push_notifications_get -Invidious::Routing.post "/feed/webhook/:token", Invidious::Routes::Feeds, :push_notifications_post +Invidious::Routing.get "/ggpht/*", Invidious::Routes::Images, :ggpht +Invidious::Routing.options "/sb/:authority/:id/:storyboard/:index", Invidious::Routes::Images, :options_storyboard +Invidious::Routing.get "/sb/:authority/:id/:storyboard/:index", Invidious::Routes::Images, :get_storyboard +Invidious::Routing.get "/s_p/:id/:name", Invidious::Routes::Images, :s_p_image +Invidious::Routing.get "/yts/img/:name", Invidious::Routes::Images, :yts_image +Invidious::Routing.get "/vi/:id/:name", Invidious::Routes::Images, :thumbnails # API routes (macro) define_v1_api_routes() @@ -410,504 +426,8 @@ define_v1_api_routes() define_api_manifest_routes() define_video_playback_routes() -# Users - -post "/watch_ajax" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env, "/feed/subscriptions") - - redirect = env.params.query["redirect"]? - redirect ||= "true" - redirect = redirect == "true" - - if !user - if redirect - next env.redirect referer - else - next error_json(403, "No such user") - end - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - id = env.params.query["id"]? - if !id - env.response.status_code = 400 - next - end - - begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) - rescue ex - if redirect - next error_template(400, ex) - else - next error_json(400, ex) - end - end - - if env.params.query["action_mark_watched"]? - action = "action_mark_watched" - elsif env.params.query["action_mark_unwatched"]? - action = "action_mark_unwatched" - else - next env.redirect referer - end - - case action - when "action_mark_watched" - if !user.watched.includes? id - PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.email) - end - when "action_mark_unwatched" - PG_DB.exec("UPDATE users SET watched = array_remove(watched, $1) WHERE email = $2", id, user.email) - else - next error_json(400, "Unsupported action #{action}") - end - - if redirect - env.redirect referer - else - env.response.content_type = "application/json" - "{}" - end -end - -# /modify_notifications -# will "ding" all subscriptions. -# /modify_notifications?receive_all_updates=false&receive_no_updates=false -# will "unding" all subscriptions. -get "/modify_notifications" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env, "/") - - redirect = env.params.query["redirect"]? - redirect ||= "false" - redirect = redirect == "true" - - if !user - if redirect - next env.redirect referer - else - next error_json(403, "No such user") - end - end - - user = user.as(User) - - if !user.password - channel_req = {} of String => String - - channel_req["receive_all_updates"] = env.params.query["receive_all_updates"]? || "true" - channel_req["receive_no_updates"] = env.params.query["receive_no_updates"]? || "" - channel_req["receive_post_updates"] = env.params.query["receive_post_updates"]? || "true" - - channel_req.reject! { |k, v| v != "true" && v != "false" } - - headers = HTTP::Headers.new - headers["Cookie"] = env.request.headers["Cookie"] - - html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers) - - cookies = HTTP::Cookies.from_client_headers(headers) - html.cookies.each do |cookie| - if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name - if cookies[cookie.name]? - cookies[cookie.name] = cookie - else - cookies << cookie - end - end - end - headers = cookies.add_request_headers(headers) - - if match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[^"]+)"/) - session_token = match["session_token"] - else - next env.redirect referer - end - - headers["content-type"] = "application/x-www-form-urlencoded" - channel_req["session_token"] = session_token - - subs = XML.parse_html(html.body) - subs.xpath_nodes(%q(//a[@class="subscription-title yt-uix-sessionlink"]/@href)).each do |channel| - channel_id = channel.content.lstrip("/channel/").not_nil! - channel_req["channel_id"] = channel_id - - YT_POOL.client &.post("/subscription_ajax?action_update_subscription_preferences=1", headers, form: channel_req) - end - end - - if redirect - env.redirect referer - else - env.response.content_type = "application/json" - "{}" - end -end - -post "/subscription_ajax" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env, "/") - - redirect = env.params.query["redirect"]? - redirect ||= "true" - redirect = redirect == "true" - - if !user - if redirect - next env.redirect referer - else - next error_json(403, "No such user") - end - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) - rescue ex - if redirect - next error_template(400, ex) - else - next error_json(400, ex) - end - end - - if env.params.query["action_create_subscription_to_channel"]?.try &.to_i?.try &.== 1 - action = "action_create_subscription_to_channel" - elsif env.params.query["action_remove_subscriptions"]?.try &.to_i?.try &.== 1 - action = "action_remove_subscriptions" - else - next env.redirect referer - end - - channel_id = env.params.query["c"]? - channel_id ||= "" - - if !user.password - # Sync subscriptions with YouTube - subscribe_ajax(channel_id, action, env.request.headers) - end - email = user.email - - case action - when "action_create_subscription_to_channel" - if !user.subscriptions.includes? channel_id - get_channel(channel_id, PG_DB, false, false) - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_append(subscriptions, $1) WHERE email = $2", channel_id, email) - end - when "action_remove_subscriptions" - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_remove(subscriptions, $1) WHERE email = $2", channel_id, email) - else - next error_json(400, "Unsupported action #{action}") - end - - if redirect - env.redirect referer - else - env.response.content_type = "application/json" - "{}" - end -end - -get "/subscription_manager" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - - if !user.password - # Refresh account - headers = HTTP::Headers.new - headers["Cookie"] = env.request.headers["Cookie"] - - user, sid = get_user(sid, headers, PG_DB) - end - - action_takeout = env.params.query["action_takeout"]?.try &.to_i? - action_takeout ||= 0 - action_takeout = action_takeout == 1 - - format = env.params.query["format"]? - format ||= "rss" - - if user.subscriptions.empty? - values = "'{}'" - else - values = "VALUES #{user.subscriptions.map { |id| %(('#{id}')) }.join(",")}" - end - - subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousChannel) - subscriptions.sort_by! { |channel| channel.author.downcase } - - if action_takeout - if format == "json" - env.response.content_type = "application/json" - env.response.headers["content-disposition"] = "attachment" - playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) - - next JSON.build do |json| - json.object do - json.field "subscriptions", user.subscriptions - json.field "watch_history", user.watched - json.field "preferences", user.preferences - json.field "playlists" do - json.array do - playlists.each do |playlist| - json.object do - json.field "title", playlist.title - json.field "description", html_to_content(playlist.description_html) - json.field "privacy", playlist.privacy.to_s - json.field "videos" do - json.array do - PG_DB.query_all("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 500", playlist.id, playlist.index, as: String).each do |video_id| - json.string video_id - end - end - end - end - end - end - end - end - end - else - env.response.content_type = "application/xml" - env.response.headers["content-disposition"] = "attachment" - export = XML.build do |xml| - xml.element("opml", version: "1.1") do - xml.element("body") do - if format == "newpipe" - title = "YouTube Subscriptions" - else - title = "Invidious Subscriptions" - end - - xml.element("outline", text: title, title: title) do - subscriptions.each do |channel| - if format == "newpipe" - xmlUrl = "https://www.youtube.com/feeds/videos.xml?channel_id=#{channel.id}" - else - xmlUrl = "#{HOST_URL}/feed/channel/#{channel.id}" - end - - xml.element("outline", text: channel.author, title: channel.author, - "type": "rss", xmlUrl: xmlUrl) - end - end - end - end - end - - next export.gsub(%(<?xml version="1.0"?>\n), "") - end - end - - templated "subscription_manager" -end - -get "/data_control" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - - templated "data_control" -end - -post "/data_control" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - referer = get_referer(env) - - if user - user = user.as(User) - - # TODO: Find a way to prevent browser timeout - - HTTP::FormData.parse(env.request) do |part| - body = part.body.gets_to_end - next if body.empty? - - # TODO: Unify into single import based on content-type - case part.name - when "import_invidious" - body = JSON.parse(body) - - if body["subscriptions"]? - user.subscriptions += body["subscriptions"].as_a.map { |a| a.as_s } - user.subscriptions.uniq! - - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) - - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) - end - - if body["watch_history"]? - user.watched += body["watch_history"].as_a.map { |a| a.as_s } - user.watched.uniq! - PG_DB.exec("UPDATE users SET watched = $1 WHERE email = $2", user.watched, user.email) - end - - if body["preferences"]? - user.preferences = Preferences.from_json(body["preferences"].to_json) - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", user.preferences.to_json, user.email) - end - - if playlists = body["playlists"]?.try &.as_a? - playlists.each do |item| - title = item["title"]?.try &.as_s?.try &.delete("<>") - description = item["description"]?.try &.as_s?.try &.delete("\r") - privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy } - - next if !title - next if !description - next if !privacy - - playlist = create_playlist(PG_DB, title, privacy, user) - PG_DB.exec("UPDATE playlists SET description = $1 WHERE id = $2", description, playlist.id) - - videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| - raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500 - - video_id = video_id.try &.as_s? - next if !video_id - - begin - video = get_video(video_id, PG_DB) - rescue ex - next - end - - playlist_video = PlaylistVideo.new({ - title: video.title, - id: video.id, - author: video.author, - ucid: video.ucid, - length_seconds: video.length_seconds, - published: video.published, - plid: playlist.id, - live_now: video.live_now, - index: Random::Secure.rand(0_i64..Int64::MAX), - }) - - video_array = playlist_video.to_a - args = arg_array(video_array) - - PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array) - PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, playlist.id) - end - end - end - when "import_youtube" - if body[0..4] == "<opml" - subscriptions = XML.parse(body) - user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| - channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] - end - else - subscriptions = JSON.parse(body) - user.subscriptions += subscriptions.as_a.compact_map do |entry| - entry["snippet"]["resourceId"]["channelId"].as_s - end - end - user.subscriptions.uniq! - - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) - - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) - when "import_freetube" - user.subscriptions += body.scan(/"channelId":"(?<channel_id>[a-zA-Z0-9_-]{24})"/).map do |md| - md["channel_id"] - end - user.subscriptions.uniq! - - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) - - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) - when "import_newpipe_subscriptions" - body = JSON.parse(body) - user.subscriptions += body["subscriptions"].as_a.compact_map do |channel| - if match = channel["url"].as_s.match(/\/channel\/(?<channel>UC[a-zA-Z0-9_-]{22})/) - next match["channel"] - elsif match = channel["url"].as_s.match(/\/user\/(?<user>.+)/) - response = YT_POOL.client &.get("/user/#{match["user"]}?disable_polymer=1&hl=en&gl=US") - html = XML.parse_html(response.body) - ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1] - next ucid if ucid - end - - nil - end - user.subscriptions.uniq! - - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) - - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) - when "import_newpipe" - Compress::Zip::Reader.open(IO::Memory.new(body)) do |file| - file.each_entry do |entry| - if entry.filename == "newpipe.db" - tempfile = File.tempfile(".db") - File.write(tempfile.path, entry.io.gets_to_end) - db = DB.open("sqlite3://" + tempfile.path) - - user.watched += db.query_all("SELECT url FROM streams", as: String).map { |url| url.lchop("https://www.youtube.com/watch?v=") } - user.watched.uniq! - - PG_DB.exec("UPDATE users SET watched = $1 WHERE email = $2", user.watched, user.email) - - user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map { |url| url.lchop("https://www.youtube.com/channel/") } - user.subscriptions.uniq! - - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) - - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) - - db.close - tempfile.delete - end - end - end - else nil # Ignore - end - end - end - - env.redirect referer -end - get "/change_password" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" sid = env.get? "sid" @@ -919,13 +439,13 @@ get "/change_password" do |env| user = user.as(User) sid = sid.as(String) - csrf_token = generate_response(sid, {":change_password"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":change_password"}, HMAC_KEY) templated "change_password" end post "/change_password" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" sid = env.get? "sid" @@ -945,7 +465,7 @@ post "/change_password" do |env| end begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex next error_template(400, ex) end @@ -975,13 +495,13 @@ post "/change_password" do |env| end new_password = Crypto::Bcrypt::Password.create(new_password, cost: 10) - PG_DB.exec("UPDATE users SET password = $1 WHERE email = $2", new_password.to_s, user.email) + Invidious::Database::Users.update_password(user, new_password.to_s) env.redirect referer end get "/delete_account" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" sid = env.get? "sid" @@ -993,13 +513,13 @@ get "/delete_account" do |env| user = user.as(User) sid = sid.as(String) - csrf_token = generate_response(sid, {":delete_account"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":delete_account"}, HMAC_KEY) templated "delete_account" end post "/delete_account" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" sid = env.get? "sid" @@ -1014,14 +534,14 @@ post "/delete_account" do |env| token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex next error_template(400, ex) end view_name = "subscriptions_#{sha256(user.email)}" - PG_DB.exec("DELETE FROM users * WHERE email = $1", user.email) - PG_DB.exec("DELETE FROM session_ids * WHERE email = $1", user.email) + Invidious::Database::Users.delete(user) + Invidious::Database::SessionIDs.delete(email: user.email) PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}") env.request.cookies.each do |cookie| @@ -1033,7 +553,7 @@ post "/delete_account" do |env| end get "/clear_watch_history" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" sid = env.get? "sid" @@ -1045,13 +565,13 @@ get "/clear_watch_history" do |env| user = user.as(User) sid = sid.as(String) - csrf_token = generate_response(sid, {":clear_watch_history"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":clear_watch_history"}, HMAC_KEY) templated "clear_watch_history" end post "/clear_watch_history" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" sid = env.get? "sid" @@ -1066,17 +586,17 @@ post "/clear_watch_history" do |env| token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex next error_template(400, ex) end - PG_DB.exec("UPDATE users SET watched = '{}' WHERE email = $1", user.email) + Invidious::Database::Users.clear_watch_history(user) env.redirect referer end get "/authorize_token" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" sid = env.get? "sid" @@ -1088,7 +608,7 @@ get "/authorize_token" do |env| user = user.as(User) sid = sid.as(String) - csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY) scopes = env.params.query["scopes"]?.try &.split(",") scopes ||= [] of String @@ -1104,7 +624,7 @@ get "/authorize_token" do |env| end post "/authorize_token" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" sid = env.get? "sid" @@ -1119,7 +639,7 @@ post "/authorize_token" do |env| token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex next error_template(400, ex) end @@ -1128,7 +648,7 @@ post "/authorize_token" do |env| callback_url = env.params.body["callbackUrl"]? expire = env.params.body["expire"]?.try &.to_i? - access_token = generate_token(user.email, scopes, expire, HMAC_KEY, PG_DB) + access_token = generate_token(user.email, scopes, expire, HMAC_KEY) if callback_url access_token = URI.encode_www_form(access_token) @@ -1152,7 +672,7 @@ post "/authorize_token" do |env| end get "/token_manager" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" sid = env.get? "sid" @@ -1163,14 +683,13 @@ get "/token_manager" do |env| end user = user.as(User) - - tokens = PG_DB.query_all("SELECT id, issued FROM session_ids WHERE email = $1 ORDER BY issued DESC", user.email, as: {session: String, issued: Time}) + tokens = Invidious::Database::SessionIDs.select_all(user.email) templated "token_manager" end post "/token_ajax" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" sid = env.get? "sid" @@ -1193,7 +712,7 @@ post "/token_ajax" do |env| token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex if redirect next error_template(400, ex) @@ -1213,7 +732,7 @@ post "/token_ajax" do |env| case action when .starts_with? "action_revoke_token" - PG_DB.exec("DELETE FROM session_ids * WHERE id = $1 AND email = $2", session, user.email) + Invidious::Database::SessionIDs.delete(sid: session, email: user.email) else next error_json(400, "Unsupported action #{action}") end @@ -1230,7 +749,7 @@ end {"/channel/:ucid/live", "/user/:user/live", "/c/:user/live"}.each do |route| get route do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale # Appears to be a bug in routing, having several routes configured # as `/a/:a`, `/b/:a`, `/c/:a` results in 404 @@ -1287,194 +806,6 @@ post "/api/v1/auth/notifications" do |env| create_notification_stream(env, topics, connection_channel) end -get "/ggpht/*" do |env| - url = env.request.path.lchop("/ggpht") - - headers = HTTP::Headers{":authority" => "yt3.ggpht.com"} - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end - - begin - YT_POOL.client &.get(url, headers) do |response| - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - env.response.headers.delete("Transfer-Encoding") - break - end - - proxy_file(response, env) - end - rescue ex - end -end - -options "/sb/:authority/:id/:storyboard/:index" do |env| - env.response.headers["Access-Control-Allow-Origin"] = "*" - env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" - env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range" -end - -get "/sb/:authority/:id/:storyboard/:index" do |env| - authority = env.params.url["authority"] - id = env.params.url["id"] - storyboard = env.params.url["storyboard"] - index = env.params.url["index"] - - url = "/sb/#{id}/#{storyboard}/#{index}?#{env.params.query}" - - headers = HTTP::Headers.new - - headers[":authority"] = "#{authority}.ytimg.com" - - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end - - begin - YT_POOL.client &.get(url, headers) do |response| - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Connection"] = "close" - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - env.response.headers.delete("Transfer-Encoding") - break - end - - proxy_file(response, env) - end - rescue ex - end -end - -get "/s_p/:id/:name" do |env| - id = env.params.url["id"] - name = env.params.url["name"] - - url = env.request.resource - - headers = HTTP::Headers{":authority" => "i9.ytimg.com"} - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end - - begin - YT_POOL.client &.get(url, headers) do |response| - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - env.response.headers.delete("Transfer-Encoding") - break - end - - proxy_file(response, env) - end - rescue ex - end -end - -get "/yts/img/:name" do |env| - headers = HTTP::Headers.new - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end - - begin - YT_POOL.client &.get(env.request.resource, headers) do |response| - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - env.response.headers.delete("Transfer-Encoding") - break - end - - proxy_file(response, env) - end - rescue ex - end -end - -get "/vi/:id/:name" do |env| - id = env.params.url["id"] - name = env.params.url["name"] - - headers = HTTP::Headers{":authority" => "i.ytimg.com"} - - if name == "maxres.jpg" - build_thumbnails(id).each do |thumb| - if YT_POOL.client &.head("/vi/#{id}/#{thumb[:url]}.jpg", headers).status_code == 200 - name = thumb[:url] + ".jpg" - break - end - end - end - url = "/vi/#{id}/#{name}" - - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end - - begin - YT_POOL.client &.get(url, headers) do |response| - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - env.response.headers.delete("Transfer-Encoding") - break - end - - proxy_file(response, env) - end - rescue ex - end -end - get "/Captcha" do |env| headers = HTTP::Headers{":authority" => "accounts.google.com"} response = YT_POOL.client &.get(env.request.resource, headers) @@ -1540,11 +871,11 @@ error 404 do |env| end error 500 do |env, ex| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale error_template(500, ex) end -static_headers do |response, filepath, filestat| +static_headers do |response| response.headers.add("Cache-Control", "max-age=2629800") end @@ -1563,4 +894,11 @@ Kemal.config.logger = LOGGER Kemal.config.host_binding = Kemal.config.host_binding != "0.0.0.0" ? Kemal.config.host_binding : CONFIG.host_binding Kemal.config.port = Kemal.config.port != 3000 ? Kemal.config.port : CONFIG.port Kemal.config.app_name = "Invidious" + +# Use in kemal's production mode. +# Users can also set the KEMAL_ENV environmental variable for this to be set automatically. +{% if flag?(:release) || flag?(:production) %} + Kemal.config.env = "production" if !ENV.has_key?("KEMAL_ENV") +{% end %} + Kemal.run diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index 628d5b6f..8cae7ae2 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -1,33 +1,26 @@ # TODO: Refactor into either SearchChannel or InvidiousChannel -struct AboutChannel - include DB::Serializable - - property ucid : String - property author : String - property auto_generated : Bool - property author_url : String - property author_thumbnail : String - property banner : String? - property description_html : String - property total_views : Int64 - property sub_count : Int32 - property joined : Time - property is_family_friendly : Bool - property allowed_regions : Array(String) - property related_channels : Array(AboutRelatedChannel) - property tabs : Array(String) -end - -struct AboutRelatedChannel - include DB::Serializable - - property ucid : String - property author : String - property author_url : String - property author_thumbnail : String -end - -def get_about_info(ucid, locale) +record AboutChannel, + ucid : String, + author : String, + auto_generated : Bool, + author_url : String, + author_thumbnail : String, + banner : String?, + description_html : String, + total_views : Int64, + sub_count : Int32, + joined : Time, + is_family_friendly : Bool, + allowed_regions : Array(String), + tabs : Array(String) + +record AboutRelatedChannel, + ucid : String, + author : String, + author_url : String, + author_thumbnail : String + +def get_about_info(ucid, locale) : AboutChannel begin # "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"} initdata = YoutubeAPI.browse(browse_id: ucid, params: "EgVhYm91dA==") @@ -59,12 +52,10 @@ def get_about_info(ucid, locale) banner = banners.try &.[-1]?.try &.["url"].as_s? description = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"]["simpleText"].as_s - description_html = HTML.escape(description).gsub("\n", "<br>") + description_html = HTML.escape(description) is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool - allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map { |a| a.as_s } - - related_channels = [] of AboutRelatedChannel + allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map(&.as_s) else author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s @@ -81,42 +72,10 @@ def get_about_info(ucid, locale) # end description = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]?.try &.as_s? || "" - description_html = HTML.escape(description).gsub("\n", "<br>") + description_html = HTML.escape(description) is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool - allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map { |a| a.as_s } - - related_channels = initdata["contents"]["twoColumnBrowseResultsRenderer"] - .["secondaryContents"]?.try &.["browseSecondaryContentsRenderer"]["contents"][0]? - .try &.["verticalChannelSectionRenderer"]?.try &.["items"]?.try &.as_a.map do |node| - renderer = node["miniChannelRenderer"]? - related_id = renderer.try &.["channelId"]?.try &.as_s? - related_id ||= "" - - related_title = renderer.try &.["title"]?.try &.["simpleText"]?.try &.as_s? - related_title ||= "" - - related_author_url = renderer.try &.["navigationEndpoint"]?.try &.["commandMetadata"]?.try &.["webCommandMetadata"]? - .try &.["url"]?.try &.as_s? - related_author_url ||= "" - - related_author_thumbnails = renderer.try &.["thumbnail"]?.try &.["thumbnails"]?.try &.as_a? - related_author_thumbnails ||= [] of JSON::Any - - related_author_thumbnail = "" - if related_author_thumbnails.size > 0 - related_author_thumbnail = related_author_thumbnails[-1]["url"]?.try &.as_s? - related_author_thumbnail ||= "" - end - - AboutRelatedChannel.new({ - ucid: related_id, - author: related_title, - author_url: related_author_url, - author_thumbnail: related_author_thumbnail, - }) - end - related_channels ||= [] of AboutRelatedChannel + allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map(&.as_s) end total_views = 0_i64 @@ -149,26 +108,50 @@ def get_about_info(ucid, locale) end end end - tabs = tabs_json.reject { |node| node["tabRenderer"]?.nil? }.map { |node| node["tabRenderer"]["title"].as_s.downcase } + tabs = tabs_json.reject { |node| node["tabRenderer"]?.nil? }.map(&.["tabRenderer"]["title"].as_s.downcase) end sub_count = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s? .try { |text| short_text_to_number(text.split(" ")[0]) } || 0 - AboutChannel.new({ - ucid: ucid, - author: author, - auto_generated: auto_generated, - author_url: author_url, - author_thumbnail: author_thumbnail, - banner: banner, - description_html: description_html, - total_views: total_views, - sub_count: sub_count, - joined: joined, + AboutChannel.new( + ucid: ucid, + author: author, + auto_generated: auto_generated, + author_url: author_url, + author_thumbnail: author_thumbnail, + banner: banner, + description_html: description_html, + total_views: total_views, + sub_count: sub_count, + joined: joined, is_family_friendly: is_family_friendly, - allowed_regions: allowed_regions, - related_channels: related_channels, - tabs: tabs, - }) + allowed_regions: allowed_regions, + tabs: tabs, + ) +end + +def fetch_related_channels(about_channel : AboutChannel) : Array(AboutRelatedChannel) + # params is {"2:string":"channels"} encoded + channels = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D") + + tabs = channels.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs").try(&.as_a?) || [] of JSON::Any + tab = tabs.find { |tab| tab.dig?("tabRenderer", "title").try(&.as_s?) == "Channels" } + return [] of AboutRelatedChannel if tab.nil? + + items = tab.dig?("tabRenderer", "content", "sectionListRenderer", "contents", 0, "itemSectionRenderer", "contents", 0, "gridRenderer", "items").try(&.as_a?) || [] of JSON::Any + + items.map do |item| + related_id = item.dig("gridChannelRenderer", "channelId").as_s + related_title = item.dig("gridChannelRenderer", "title", "simpleText").as_s + related_author_url = item.dig("gridChannelRenderer", "navigationEndpoint", "browseEndpoint", "canonicalBaseUrl").as_s + related_author_thumbnail = item.dig("gridChannelRenderer", "thumbnail", "thumbnails", -1, "url").as_s + + AboutRelatedChannel.new( + ucid: related_id, + author: related_title, + author_url: related_author_url, + author_thumbnail: related_author_thumbnail, + ) + end end diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index 70623cc0..155ec559 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -101,7 +101,7 @@ struct ChannelVideo def to_tuple {% begin %} { - {{*@type.instance_vars.map { |var| var.name }}} + {{*@type.instance_vars.map(&.name)}} } {% end %} end @@ -114,7 +114,7 @@ class ChannelRedirect < Exception end end -def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, max_threads = 10) +def get_batch_channels(channels, refresh = false, pull_all_videos = true, max_threads = 10) finished_channel = Channel(String | Nil).new spawn do @@ -130,7 +130,7 @@ def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, ma active_threads += 1 spawn do begin - get_channel(ucid, db, refresh, pull_all_videos) + get_channel(ucid, refresh, pull_all_videos) finished_channel.send(ucid) rescue ex finished_channel.send(nil) @@ -151,28 +151,21 @@ def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, ma return final end -def get_channel(id, db, refresh = true, pull_all_videos = true) - if channel = db.query_one?("SELECT * FROM channels WHERE id = $1", id, as: InvidiousChannel) +def get_channel(id, refresh = true, pull_all_videos = true) + if channel = Invidious::Database::Channels.select(id) if refresh && Time.utc - channel.updated > 10.minutes - channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) - channel_array = channel.to_a - args = arg_array(channel_array) - - db.exec("INSERT INTO channels VALUES (#{args}) \ - ON CONFLICT (id) DO UPDATE SET author = $2, updated = $3", args: channel_array) + channel = fetch_channel(id, pull_all_videos: pull_all_videos) + Invidious::Database::Channels.insert(channel, update_on_conflict: true) end else - channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) - channel_array = channel.to_a - args = arg_array(channel_array) - - db.exec("INSERT INTO channels VALUES (#{args})", args: channel_array) + channel = fetch_channel(id, pull_all_videos: pull_all_videos) + Invidious::Database::Channels.insert(channel) end return channel end -def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) +def fetch_channel(ucid, pull_all_videos = true, locale = nil) LOGGER.debug("fetch_channel: #{ucid}") LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}, locale = #{locale}") @@ -241,15 +234,11 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) # We don't include the 'premiere_timestamp' here because channel pages don't include them, # meaning the above timestamp is always null - was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ - ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ - updated = $4, ucid = $5, author = $6, length_seconds = $7, \ - live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) + was_insert = Invidious::Database::ChannelVideos.insert(video) if was_insert LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions") - db.exec("UPDATE users SET notifications = array_append(notifications, $1), \ - feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) + Invidious::Database::Users.add_notification(video) else LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated") end @@ -284,13 +273,8 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) # We are notified of Red videos elsewhere (PubSub), which includes a correct published date, # so since they don't provide a published date here we can safely ignore them. if Time.utc - video.published > 1.minute - was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ - ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ - updated = $4, ucid = $5, author = $6, length_seconds = $7, \ - live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) - - db.exec("UPDATE users SET notifications = array_append(notifications, $1), \ - feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert + was_insert = Invidious::Database::ChannelVideos.insert(video) + Invidious::Database::Users.add_notification(video) if was_insert end end diff --git a/src/invidious/channels/community.cr b/src/invidious/channels/community.cr index 97ab30ec..4701ecbd 100644 --- a/src/invidious/channels/community.cr +++ b/src/invidious/channels/community.cr @@ -158,7 +158,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) view_count = attachment["viewCountText"]?.try &.["simpleText"].as_s.gsub(/\D/, "").to_i64? || 0_i64 json.field "viewCount", view_count - json.field "viewCountText", translate(locale, "`x` views", number_to_short_text(view_count)) + json.field "viewCountText", translate_count(locale, "generic_views_count", view_count, NumberFormatting::Short) end when .has_key?("backstageImageRenderer") attachment = attachment["backstageImageRenderer"] @@ -242,7 +242,7 @@ def produce_channel_community_continuation(ucid, cursor) }, } - continuation = object.try { |i| Protodec::Any.cast_json(object) } + continuation = object.try { |i| Protodec::Any.cast_json(i) } .try { |i| Protodec::Any.from_json(i) } .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } @@ -255,11 +255,11 @@ def extract_channel_community_cursor(continuation) .try { |i| Base64.decode(i) } .try { |i| IO::Memory.new(i) } .try { |i| Protodec::Any.parse(i) } - .try { |i| i["80226972:0:embedded"]["3:1:base64"].as_h } + .try(&.["80226972:0:embedded"]["3:1:base64"].as_h) if object["53:2:embedded"]?.try &.["3:0:embedded"]? object["53:2:embedded"]["3:0:embedded"]["2:0:string"] = object["53:2:embedded"]["3:0:embedded"] - .try { |i| i["2:0:base64"].as_h } + .try(&.["2:0:base64"].as_h) .try { |i| Protodec::Any.cast_json(i) } .try { |i| Protodec::Any.from_json(i) } .try { |i| Base64.urlsafe_encode(i, padding: false) } diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr index 393b055e..d5628f6a 100644 --- a/src/invidious/channels/playlists.cr +++ b/src/invidious/channels/playlists.cr @@ -1,17 +1,17 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by) if continuation response_json = YoutubeAPI.browse(continuation) - continuationItems = response_json["onResponseReceivedActions"]? + continuation_items = response_json["onResponseReceivedActions"]? .try &.[0]["appendContinuationItemsAction"]["continuationItems"] - return [] of SearchItem, nil if !continuationItems + return [] of SearchItem, nil if !continuation_items items = [] of SearchItem - continuationItems.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item| + continuation_items.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item| extract_item(item, author, ucid).try { |t| items << t } } - continuation = continuationItems.as_a.last["continuationItemRenderer"]? + continuation = continuation_items.as_a.last["continuationItemRenderer"]? .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s else url = "/channel/#{ucid}/playlists?flow=list&view=1" @@ -84,7 +84,7 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"]))) object["80226972:embedded"].delete("3:base64") - continuation = object.try { |i| Protodec::Any.cast_json(object) } + continuation = object.try { |i| Protodec::Any.cast_json(i) } .try { |i| Protodec::Any.from_json(i) } .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 2c43bf0b..48453bb7 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -49,7 +49,7 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"]))) object["80226972:embedded"].delete("3:base64") - continuation = object.try { |i| Protodec::Any.cast_json(object) } + continuation = object.try { |i| Protodec::Any.cast_json(i) } .try { |i| Protodec::Any.from_json(i) } .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index a5506b03..dda92440 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -60,8 +60,6 @@ def fetch_youtube_comments(id, cursor, format, locale, thin_mode, region, sort_b case cursor when nil, "" ctoken = produce_comment_continuation(id, cursor: "", sort_by: sort_by) - # when .starts_with? "Ug" - # ctoken = produce_comment_reply_continuation(id, video.ucid, cursor) when .starts_with? "ADSJ" ctoken = produce_comment_continuation(id, cursor: cursor, sort_by: sort_by) else @@ -72,10 +70,9 @@ def fetch_youtube_comments(id, cursor, format, locale, thin_mode, region, sort_b response = YoutubeAPI.next(continuation: ctoken, client_config: client_config) contents = nil - if response["onResponseReceivedEndpoints"]? - onResponseReceivedEndpoints = response["onResponseReceivedEndpoints"] + if on_response_received_endpoints = response["onResponseReceivedEndpoints"]? header = nil - onResponseReceivedEndpoints.as_a.each do |item| + on_response_received_endpoints.as_a.each do |item| if item["reloadContinuationItemsCommand"]? case item["reloadContinuationItemsCommand"]["slot"] when "RELOAD_CONTINUATION_SLOT_HEADER" @@ -97,7 +94,8 @@ def fetch_youtube_comments(id, cursor, format, locale, thin_mode, region, sort_b contents = body["contents"]? header = body["header"]? if body["continuations"]? - moreRepliesContinuation = body["continuations"][0]["nextContinuationData"]["continuation"].as_s + # Removable? Doesn't seem like this is used. + more_replies_continuation = body["continuations"][0]["nextContinuationData"]["continuation"].as_s end else raise InfoException.new("Could not fetch comments") @@ -111,10 +109,10 @@ def fetch_youtube_comments(id, cursor, format, locale, thin_mode, region, sort_b end end - continuationItemRenderer = nil + continuation_item_renderer = nil contents.as_a.reject! do |item| if item["continuationItemRenderer"]? - continuationItemRenderer = item["continuationItemRenderer"] + continuation_item_renderer = item["continuationItemRenderer"] true end end @@ -232,14 +230,14 @@ def fetch_youtube_comments(id, cursor, format, locale, thin_mode, region, sort_b end end - if continuationItemRenderer - if continuationItemRenderer["continuationEndpoint"]? - continuationEndpoint = continuationItemRenderer["continuationEndpoint"] - elsif continuationItemRenderer["button"]? - continuationEndpoint = continuationItemRenderer["button"]["buttonRenderer"]["command"] + if continuation_item_renderer + if continuation_item_renderer["continuationEndpoint"]? + continuation_endpoint = continuation_item_renderer["continuationEndpoint"] + elsif continuation_item_renderer["button"]? + continuation_endpoint = continuation_item_renderer["button"]["buttonRenderer"]["command"] end - if continuationEndpoint - json.field "continuation", continuationEndpoint["continuationCommand"]["token"].as_s + if continuation_endpoint + json.field "continuation", continuation_endpoint["continuationCommand"]["token"].as_s end end end @@ -270,18 +268,20 @@ def fetch_reddit_comments(id, sort_by = "confidence") headers = HTTP::Headers{"User-Agent" => "web:invidious:v#{CURRENT_VERSION} (by github.com/iv-org/invidious)"} # TODO: Use something like #479 for a static list of instances to use here - query = "(url:3D#{id}%20OR%20url:#{id})%20(site:invidio.us%20OR%20site:youtube.com%20OR%20site:youtu.be)" - search_results = client.get("/search.json?q=#{query}", headers) + query = URI::Params.encode({q: "(url:3D#{id} OR url:#{id}) AND (site:invidio.us OR site:youtube.com OR site:youtu.be)"}) + search_results = client.get("/search.json?#{query}", headers) if search_results.status_code == 200 search_results = RedditThing.from_json(search_results.body) # For videos that have more than one thread, choose the one with the highest score - thread = search_results.data.as(RedditListing).children.sort_by { |child| child.data.as(RedditLink).score }[-1] - thread = thread.data.as(RedditLink) - - result = client.get("/r/#{thread.subreddit}/comments/#{thread.id}.json?limit=100&sort=#{sort_by}", headers).body - result = Array(RedditThing).from_json(result) + threads = search_results.data.as(RedditListing).children + thread = threads.max_by?(&.data.as(RedditLink).score).try(&.data.as(RedditLink)) + result = thread.try do |t| + body = client.get("/r/#{t.subreddit}/comments/#{t.id}.json?limit=100&sort=#{sort_by}", headers).body + Array(RedditThing).from_json(body) + end + result ||= [] of RedditThing elsif search_results.status_code == 302 # Previously, if there was only one result then the API would redirect to that result. # Now, it appears it will still return a listing so this section is likely unnecessary. @@ -296,7 +296,8 @@ def fetch_reddit_comments(id, sort_by = "confidence") client.close - comments = result[1].data.as(RedditListing).children + comments = result[1]?.try(&.data.as(RedditListing).children) + comments ||= [] of RedditThing return comments, thread end @@ -305,13 +306,19 @@ def template_youtube_comments(comments, locale, thin_mode, is_replies = false) root = comments["comments"].as_a root.each do |child| if child["replies"]? + replies_count_text = translate_count(locale, + "comments_view_x_replies", + child["replies"]["replyCount"].as_i64 || 0, + NumberFormatting::Separator + ) + replies_html = <<-END_HTML <div id="replies" class="pure-g"> <div class="pure-u-1-24"></div> <div class="pure-u-23-24"> <p> <a href="javascript:void(0)" data-continuation="#{child["replies"]["continuation"]}" - data-onclick="get_youtube_replies" data-load-replies>#{translate(locale, "View `x` replies", number_with_separator(child["replies"]["replyCount"]))}</a> + data-onclick="get_youtube_replies" data-load-replies>#{replies_count_text}</a> </p> </div> </div> @@ -329,7 +336,7 @@ def template_youtube_comments(comments, locale, thin_mode, is_replies = false) html << <<-END_HTML <div class="pure-g" style="width:100%"> <div class="channel-profile pure-u-4-24 pure-u-md-2-24"> - <img style="margin-right:1em;margin-top:1em;width:90%" src="#{author_thumbnail}"> + <img loading="lazy" style="margin-right:1em;margin-top:1em;width:90%" src="#{author_thumbnail}"> </div> <div class="pure-u-20-24 pure-u-md-22-24"> <p> @@ -349,7 +356,7 @@ def template_youtube_comments(comments, locale, thin_mode, is_replies = false) html << <<-END_HTML <div class="pure-g"> <div class="pure-u-1 pure-u-md-1-2"> - <img style="width:100%" src="/ggpht#{URI.parse(attachment["url"].as_s).request_target}"> + <img loading="lazy" style="width:100%" src="/ggpht#{URI.parse(attachment["url"].as_s).request_target}"> </div> </div> END_HTML @@ -410,7 +417,7 @@ def template_youtube_comments(comments, locale, thin_mode, is_replies = false) html << <<-END_HTML <span class="creator-heart-container" title="#{translate(locale, "`x` marked it with a ❤", child["creatorHeart"]["creatorName"].as_s)}"> <div class="creator-heart"> - <img class="creator-heart-background-hearted" src="#{creator_thumbnail}"></img> + <img loading="lazy" class="creator-heart-background-hearted" src="#{creator_thumbnail}"></img> <div class="creator-heart-small-hearted"> <div class="icon ion-ios-heart creator-heart-small-container"></div> </div> @@ -473,7 +480,7 @@ def template_reddit_comments(root, locale) <p> <a href="javascript:void(0)" data-onclick="toggle_parent">[ - ]</a> <b><a href="https://www.reddit.com/user/#{child.author}">#{child.author}</a></b> - #{translate(locale, "`x` points", number_with_separator(child.score))} + #{translate_count(locale, "comments_points_count", child.score, NumberFormatting::Separator)} <span title="#{child.created_utc.to_s(translate(locale, "%a %B %-d %T %Y UTC"))}">#{translate(locale, "`x` ago", recode_date(child.created_utc, locale))}</span> <a href="https://www.reddit.com#{child.permalink}" title="#{translate(locale, "permalink")}">#{translate(locale, "permalink")}</a> </p> @@ -552,12 +559,12 @@ end def parse_content(content : JSON::Any) : String content["simpleText"]?.try &.as_s.rchop('\ufeff').try { |b| HTML.escape(b) }.to_s || - content["runs"]?.try &.as_a.try { |r| content_to_comment_html(r).try &.to_s } || "" + content["runs"]?.try &.as_a.try { |r| content_to_comment_html(r).try &.to_s.gsub("\n", "<br>") } || "" end def content_to_comment_html(content) comment_html = content.map do |run| - text = HTML.escape(run["text"].as_s).gsub("\n", "<br>") + text = HTML.escape(run["text"].as_s) if run["bold"]? text = "<b>#{text}</b>" @@ -575,7 +582,9 @@ def content_to_comment_html(content) url = "/watch?v=#{url.request_target.lstrip('/')}" elsif url.host.nil? || url.host.not_nil!.ends_with?("youtube.com") if url.path == "/redirect" - url = HTTP::Params.parse(url.query.not_nil!)["q"] + # Sometimes, links can be corrupted (why?) so make sure to fallback + # nicely. See https://github.com/iv-org/invidious/issues/2682 + url = HTTP::Params.parse(url.query.not_nil!)["q"]? || "" else url = url.request_target end @@ -638,42 +647,7 @@ def produce_comment_continuation(video_id, cursor = "", sort_by = "top") object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64 end - continuation = object.try { |i| Protodec::Any.cast_json(object) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return continuation -end - -def produce_comment_reply_continuation(video_id, ucid, comment_id) - object = { - "2:embedded" => { - "2:string" => video_id, - "24:varint" => 1_i64, - "25:varint" => 1_i64, - "28:varint" => 1_i64, - "36:embedded" => { - "5:varint" => -1_i64, - "8:varint" => 0_i64, - }, - }, - "3:varint" => 6_i64, - "6:embedded" => { - "3:embedded" => { - "2:string" => comment_id, - "4:embedded" => { - "1:varint" => 0_i64, - }, - "5:string" => ucid, - "6:string" => video_id, - "8:varint" => 1_i64, - "9:varint" => 10_i64, - }, - }, - } - - continuation = object.try { |i| Protodec::Any.cast_json(object) } + continuation = object.try { |i| Protodec::Any.cast_json(i) } .try { |i| Protodec::Any.from_json(i) } .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } diff --git a/src/invidious/config.cr b/src/invidious/config.cr new file mode 100644 index 00000000..c4a8bf83 --- /dev/null +++ b/src/invidious/config.cr @@ -0,0 +1,192 @@ +struct DBConfig + include YAML::Serializable + + property user : String + property password : String + property host : String + property port : Int32 + property dbname : String +end + +struct ConfigPreferences + include YAML::Serializable + + property annotations : Bool = false + property annotations_subscribed : Bool = false + property autoplay : Bool = false + property captions : Array(String) = ["", "", ""] + property comments : Array(String) = ["youtube", ""] + property continue : Bool = false + property continue_autoplay : Bool = true + property dark_mode : String = "" + property latest_only : Bool = false + property listen : Bool = false + property local : Bool = false + property locale : String = "en-US" + property max_results : Int32 = 40 + property notifications_only : Bool = false + property player_style : String = "invidious" + property quality : String = "hd720" + property quality_dash : String = "auto" + property default_home : String? = "Popular" + property feed_menu : Array(String) = ["Popular", "Trending", "Subscriptions", "Playlists"] + property automatic_instance_redirect : Bool = false + property region : String = "US" + property related_videos : Bool = true + property sort : String = "published" + property speed : Float32 = 1.0_f32 + property thin_mode : Bool = false + property unseen_only : Bool = false + property video_loop : Bool = false + property extend_desc : Bool = false + property volume : Int32 = 100 + property vr_mode : Bool = true + property show_nick : Bool = true + property save_player_pos : Bool = false + + def to_tuple + {% begin %} + { + {{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}} + } + {% end %} + end +end + +class Config + include YAML::Serializable + + property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions) + property feed_threads : Int32 = 1 # Number of threads to use for updating feeds + property output : String = "STDOUT" # Log file path or STDOUT + property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr + property db : DBConfig? = nil # Database configuration with separate parameters (username, hostname, etc) + + @[YAML::Field(converter: Preferences::URIConverter)] + property database_url : URI = URI.parse("") # Database configuration using 12-Factor "Database URL" syntax + property decrypt_polling : Bool = true # Use polling to keep decryption function up to date + property full_refresh : Bool = false # Used for crawling channels: threads should check all videos uploaded by a channel + property https_only : Bool? # Used to tell Invidious it is behind a proxy, so links to resources should be https:// + property hmac_key : String? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions + property domain : String? # Domain to be used for links to resources on the site where an absolute URL is required + property use_pubsub_feeds : Bool | Int32 = false # Subscribe to channels using PubSubHubbub (requires domain, hmac_key) + property popular_enabled : Bool = true + property captcha_enabled : Bool = true + property login_enabled : Bool = true + property registration_enabled : Bool = true + property statistics_enabled : Bool = false + property admins : Array(String) = [] of String + property external_port : Int32? = nil + property default_user_preferences : ConfigPreferences = ConfigPreferences.from_yaml("") + property dmca_content : Array(String) = [] of String # For compliance with DMCA, disables download widget using list of video IDs + property check_tables : Bool = false # Check table integrity, automatically try to add any missing columns, create tables, etc. + property cache_annotations : Bool = false # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards + property banner : String? = nil # Optional banner to be displayed along top of page for announcements, etc. + property hsts : Bool? = true # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely + property disable_proxy : Bool? | Array(String)? = false # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local' + + # URL to the modified source code to be easily AGPL compliant + # Will display in the footer, next to the main source code link + property modified_source_code_url : String? = nil + + @[YAML::Field(converter: Preferences::FamilyConverter)] + property force_resolve : Socket::Family = Socket::Family::UNSPEC # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729) + property port : Int32 = 3000 # Port to listen for connections (overrided by command line argument) + property host_binding : String = "0.0.0.0" # Host to bind (overrided by command line argument) + property pool_size : Int32 = 100 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) + property use_quic : Bool = false # Use quic transport for youtube api + + @[YAML::Field(converter: Preferences::StringToCookies)] + property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format + property captcha_key : String? = nil # Key for Anti-Captcha + property captcha_api_url : String = "https://api.anti-captcha.com" # API URL for Anti-Captcha + + def disabled?(option) + case disabled = CONFIG.disable_proxy + when Bool + return disabled + when Array + if disabled.includes? option + return true + else + return false + end + else + return false + end + end + + def self.load + # Load config from file or YAML string env var + env_config_file = "INVIDIOUS_CONFIG_FILE" + env_config_yaml = "INVIDIOUS_CONFIG" + + config_file = ENV.has_key?(env_config_file) ? ENV.fetch(env_config_file) : "config/config.yml" + config_yaml = ENV.has_key?(env_config_yaml) ? ENV.fetch(env_config_yaml) : File.read(config_file) + + config = Config.from_yaml(config_yaml) + + # Update config from env vars (upcased and prefixed with "INVIDIOUS_") + {% for ivar in Config.instance_vars %} + {% env_id = "INVIDIOUS_#{ivar.id.upcase}" %} + + if ENV.has_key?({{env_id}}) + # puts %(Config.{{ivar.id}} : Loading from env var {{env_id}}) + env_value = ENV.fetch({{env_id}}) + success = false + + # Use YAML converter if specified + {% ann = ivar.annotation(::YAML::Field) %} + {% if ann && ann[:converter] %} + puts %(Config.{{ivar.id}} : Parsing "#{env_value}" as {{ivar.type}} with {{ann[:converter]}} converter) + config.{{ivar.id}} = {{ann[:converter]}}.from_yaml(YAML::ParseContext.new, YAML::Nodes.parse(ENV.fetch({{env_id}})).nodes[0]) + puts %(Config.{{ivar.id}} : Set to #{config.{{ivar.id}}}) + success = true + + # Use regular YAML parser otherwise + {% else %} + {% ivar_types = ivar.type.union? ? ivar.type.union_types : [ivar.type] %} + # Sort types to avoid parsing nulls and numbers as strings + {% ivar_types = ivar_types.sort_by { |ivar_type| ivar_type == Nil ? 0 : ivar_type == Int32 ? 1 : 2 } %} + {{ivar_types}}.each do |ivar_type| + if !success + begin + # puts %(Config.{{ivar.id}} : Trying to parse "#{env_value}" as #{ivar_type}) + config.{{ivar.id}} = ivar_type.from_yaml(env_value) + puts %(Config.{{ivar.id}} : Set to #{config.{{ivar.id}}} (#{ivar_type})) + success = true + rescue + # nop + end + end + end + {% end %} + + # Exit on fail + if !success + puts %(Config.{{ivar.id}} failed to parse #{env_value} as {{ivar.type}}) + exit(1) + end + end + {% end %} + + # Build database_url from db.* if it's not set directly + if config.database_url.to_s.empty? + if db = config.db + config.database_url = URI.new( + scheme: "postgres", + user: db.user, + password: db.password, + host: db.host, + port: db.port, + path: db.dbname, + ) + else + puts "Config : Either database_url or db.* is required" + exit(1) + end + end + + return config + end +end diff --git a/src/invidious/database/annotations.cr b/src/invidious/database/annotations.cr new file mode 100644 index 00000000..03749473 --- /dev/null +++ b/src/invidious/database/annotations.cr @@ -0,0 +1,24 @@ +require "./base.cr" + +module Invidious::Database::Annotations + extend self + + def insert(id : String, annotations : String) + request = <<-SQL + INSERT INTO annotations + VALUES ($1, $2) + ON CONFLICT DO NOTHING + SQL + + PG_DB.exec(request, id, annotations) + end + + def select(id : String) : Annotation? + request = <<-SQL + SELECT * FROM annotations + WHERE id = $1 + SQL + + return PG_DB.query_one?(request, id, as: Annotation) + end +end diff --git a/src/invidious/database/base.cr b/src/invidious/database/base.cr new file mode 100644 index 00000000..6e49ea1a --- /dev/null +++ b/src/invidious/database/base.cr @@ -0,0 +1,110 @@ +require "pg" + +module Invidious::Database + extend self + + def check_enum(db, enum_name, struct_type = nil) + return # TODO + + if !db.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool) + LOGGER.info("check_enum: CREATE TYPE #{enum_name}") + + db.using_connection do |conn| + conn.as(PG::Connection).exec_all(File.read("config/sql/#{enum_name}.sql")) + end + end + end + + def check_table(db, table_name, struct_type = nil) + # Create table if it doesn't exist + begin + db.exec("SELECT * FROM #{table_name} LIMIT 0") + rescue ex + LOGGER.info("check_table: check_table: CREATE TABLE #{table_name}") + + db.using_connection do |conn| + conn.as(PG::Connection).exec_all(File.read("config/sql/#{table_name}.sql")) + end + end + + return if !struct_type + + struct_array = struct_type.type_array + column_array = get_column_array(db, table_name) + column_types = File.read("config/sql/#{table_name}.sql").match(/CREATE TABLE public\.#{table_name}\n\((?<types>[\d\D]*?)\);/) + .try &.["types"].split(",").map(&.strip).reject &.starts_with?("CONSTRAINT") + + return if !column_types + + struct_array.each_with_index do |name, i| + if name != column_array[i]? + if !column_array[i]? + new_column = column_types.select(&.starts_with?(name))[0] + LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}") + db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") + next + end + + # Column doesn't exist + if !column_array.includes? name + new_column = column_types.select(&.starts_with?(name))[0] + db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") + end + + # Column exists but in the wrong position, rotate + if struct_array.includes? column_array[i] + until name == column_array[i] + new_column = column_types.select(&.starts_with?(column_array[i]))[0]?.try &.gsub("#{column_array[i]}", "#{column_array[i]}_new") + + # There's a column we didn't expect + if !new_column + LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]}") + db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") + + column_array = get_column_array(db, table_name) + next + end + + LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}") + db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") + + LOGGER.info("check_table: UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}") + db.exec("UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}") + + LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") + db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") + + LOGGER.info("check_table: ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}") + db.exec("ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}") + + column_array = get_column_array(db, table_name) + end + else + LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") + db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") + end + end + end + + return if column_array.size <= struct_array.size + + column_array.each do |column| + if !struct_array.includes? column + LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE") + db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE") + end + end + end + + def get_column_array(db, table_name) + column_array = [] of String + db.query("SELECT * FROM #{table_name} LIMIT 0") do |rs| + rs.column_count.times do |i| + column = rs.as(PG::ResultSet).field(i) + column_array << column.name + end + end + + return column_array + end +end diff --git a/src/invidious/database/channels.cr b/src/invidious/database/channels.cr new file mode 100644 index 00000000..134cf59d --- /dev/null +++ b/src/invidious/database/channels.cr @@ -0,0 +1,149 @@ +require "./base.cr" + +# +# This module contains functions related to the "channels" table. +# +module Invidious::Database::Channels + extend self + + # ------------------- + # Insert / delete + # ------------------- + + def insert(channel : InvidiousChannel, update_on_conflict : Bool = false) + channel_array = channel.to_a + + request = <<-SQL + INSERT INTO channels + VALUES (#{arg_array(channel_array)}) + SQL + + if update_on_conflict + request += <<-SQL + ON CONFLICT (id) DO UPDATE + SET author = $2, updated = $3 + SQL + end + + PG_DB.exec(request, args: channel_array) + end + + # ------------------- + # Update + # ------------------- + + def update_author(id : String, author : String) + request = <<-SQL + UPDATE channels + SET updated = $1, author = $2, deleted = false + WHERE id = $3 + SQL + + PG_DB.exec(request, Time.utc, author, id) + end + + def update_mark_deleted(id : String) + request = <<-SQL + UPDATE channels + SET updated = $1, deleted = true + WHERE id = $2 + SQL + + PG_DB.exec(request, Time.utc, id) + end + + # ------------------- + # Select + # ------------------- + + def select(id : String) : InvidiousChannel? + request = <<-SQL + SELECT * FROM channels + WHERE id = $1 + SQL + + return PG_DB.query_one?(request, id, as: InvidiousChannel) + end + + def select(ids : Array(String)) : Array(InvidiousChannel)? + return [] of InvidiousChannel if ids.empty? + values = ids.map { |id| %(('#{id}')) }.join(",") + + request = <<-SQL + SELECT * FROM channels + WHERE id = ANY(VALUES #{values}) + SQL + + return PG_DB.query_all(request, as: InvidiousChannel) + end +end + +# +# This module contains functions related to the "channel_videos" table. +# +module Invidious::Database::ChannelVideos + extend self + + # ------------------- + # Insert + # ------------------- + + # This function returns the status of the query (i.e: success?) + def insert(video : ChannelVideo, with_premiere_timestamp : Bool = false) : Bool + if with_premiere_timestamp + last_items = "premiere_timestamp = $9, views = $10" + else + last_items = "views = $10" + end + + request = <<-SQL + INSERT INTO channel_videos + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (id) DO UPDATE + SET title = $2, published = $3, updated = $4, ucid = $5, + author = $6, length_seconds = $7, live_now = $8, #{last_items} + RETURNING (xmax=0) AS was_insert + SQL + + return PG_DB.query_one(request, *video.to_tuple, as: Bool) + end + + # ------------------- + # Select + # ------------------- + + def select(ids : Array(String)) : Array(ChannelVideo) + return [] of ChannelVideo if ids.empty? + + request = <<-SQL + SELECT * FROM channel_videos + WHERE id IN (#{arg_array(ids)}) + ORDER BY published DESC + SQL + + return PG_DB.query_all(request, args: ids, as: ChannelVideo) + end + + def select_notfications(ucid : String, since : Time) : Array(ChannelVideo) + request = <<-SQL + SELECT * FROM channel_videos + WHERE ucid = $1 AND published > $2 + ORDER BY published DESC + LIMIT 15 + SQL + + return PG_DB.query_all(request, ucid, since, as: ChannelVideo) + end + + def select_popular_videos : Array(ChannelVideo) + request = <<-SQL + SELECT DISTINCT ON (ucid) * + FROM channel_videos + WHERE ucid IN (SELECT channel FROM (SELECT UNNEST(subscriptions) AS channel FROM users) AS d + GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40) + ORDER BY ucid, published DESC + SQL + + PG_DB.query_all(request, as: ChannelVideo) + end +end diff --git a/src/invidious/database/nonces.cr b/src/invidious/database/nonces.cr new file mode 100644 index 00000000..469fcbd8 --- /dev/null +++ b/src/invidious/database/nonces.cr @@ -0,0 +1,46 @@ +require "./base.cr" + +module Invidious::Database::Nonces + extend self + + # ------------------- + # Insert + # ------------------- + + def insert(nonce : String, expire : Time) + request = <<-SQL + INSERT INTO nonces + VALUES ($1, $2) + ON CONFLICT DO NOTHING + SQL + + PG_DB.exec(request, nonce, expire) + end + + # ------------------- + # Update + # ------------------- + + def update_set_expired(nonce : String) + request = <<-SQL + UPDATE nonces + SET expire = $1 + WHERE nonce = $2 + SQL + + PG_DB.exec(request, Time.utc(1990, 1, 1), nonce) + end + + # ------------------- + # Select + # ------------------- + + def select(nonce : String) : Tuple(String, Time)? + request = <<-SQL + SELECT * FROM nonces + WHERE nonce = $1 + SQL + + return PG_DB.query_one?(request, nonce, as: {String, Time}) + end +end diff --git a/src/invidious/database/playlists.cr b/src/invidious/database/playlists.cr new file mode 100644 index 00000000..7a5f61dc --- /dev/null +++ b/src/invidious/database/playlists.cr @@ -0,0 +1,265 @@ +require "./base.cr" + +# +# This module contains functions related to the "playlists" table. +# +module Invidious::Database::Playlists + extend self + + # ------------------- + # Insert / delete + # ------------------- + + def insert(playlist : InvidiousPlaylist) + playlist_array = playlist.to_a + + request = <<-SQL + INSERT INTO playlists + VALUES (#{arg_array(playlist_array)}) + SQL + + PG_DB.exec(request, args: playlist_array) + end + + # deletes the given playlist and connected playlist videos + def delete(id : String) + PlaylistVideos.delete_by_playlist(id) + request = <<-SQL + DELETE FROM playlists * + WHERE id = $1 + SQL + + PG_DB.exec(request, id) + end + + # ------------------- + # Update + # ------------------- + + def update(id : String, title : String, privacy, description, updated) + request = <<-SQL + UPDATE playlists + SET title = $1, privacy = $2, description = $3, updated = $4 + WHERE id = $5 + SQL + + PG_DB.exec(request, title, privacy, description, updated, id) + end + + def update_description(id : String, description) + request = <<-SQL + UPDATE playlists + SET description = $1 + WHERE id = $2 + SQL + + PG_DB.exec(request, description, id) + end + + def update_subscription_time(id : String) + request = <<-SQL + UPDATE playlists + SET subscribed = $1 + WHERE id = $2 + SQL + + PG_DB.exec(request, Time.utc, id) + end + + def update_video_added(id : String, index : String | Int64) + request = <<-SQL + UPDATE playlists + SET index = array_append(index, $1), + video_count = cardinality(index) + 1, + updated = $2 + WHERE id = $3 + SQL + + PG_DB.exec(request, index, Time.utc, id) + end + + def update_video_removed(id : String, index : String | Int64) + request = <<-SQL + UPDATE playlists + SET index = array_remove(index, $1), + video_count = cardinality(index) - 1, + updated = $2 + WHERE id = $3 + SQL + + PG_DB.exec(request, index, Time.utc, id) + end + + # ------------------- + # Salect + # ------------------- + + def select(*, id : String, raise_on_fail : Bool = false) : InvidiousPlaylist? + request = <<-SQL + SELECT * FROM playlists + WHERE id = $1 + SQL + + if raise_on_fail + return PG_DB.query_one(request, id, as: InvidiousPlaylist) + else + return PG_DB.query_one?(request, id, as: InvidiousPlaylist) + end + end + + def select_all(*, author : String) : Array(InvidiousPlaylist) + request = <<-SQL + SELECT * FROM playlists + WHERE author = $1 + SQL + + return PG_DB.query_all(request, author, as: InvidiousPlaylist) + end + + # ------------------- + # Salect (filtered) + # ------------------- + + def select_like_iv(email : String) : Array(InvidiousPlaylist) + request = <<-SQL + SELECT * FROM playlists + WHERE author = $1 AND id LIKE 'IV%' + ORDER BY created + SQL + + PG_DB.query_all(request, email, as: InvidiousPlaylist) + end + + def select_not_like_iv(email : String) : Array(InvidiousPlaylist) + request = <<-SQL + SELECT * FROM playlists + WHERE author = $1 AND id NOT LIKE 'IV%' + ORDER BY created + SQL + + PG_DB.query_all(request, email, as: InvidiousPlaylist) + end + + def select_user_created_playlists(email : String) : Array({String, String}) + request = <<-SQL + SELECT id,title FROM playlists + WHERE author = $1 AND id LIKE 'IV%' + SQL + + PG_DB.query_all(request, email, as: {String, String}) + end + + # ------------------- + # Misc checks + # ------------------- + + # Check if given playlist ID exists + def exists?(id : String) : Bool + request = <<-SQL + SELECT id FROM playlists + WHERE id = $1 + SQL + + return PG_DB.query_one?(request, id, as: String).nil? + end + + # Count how many playlist a user has created. + def count_owned_by(author : String) : Int64 + request = <<-SQL + SELECT count(*) FROM playlists + WHERE author = $1 + SQL + + return PG_DB.query_one?(request, author, as: Int64) || 0_i64 + end +end + +# +# This module contains functions related to the "playlist_videos" table. +# +module Invidious::Database::PlaylistVideos + extend self + + private alias VideoIndex = Int64 | Array(Int64) + + # ------------------- + # Insert / Delete + # ------------------- + + def insert(video : PlaylistVideo) + video_array = video.to_a + + request = <<-SQL + INSERT INTO playlist_videos + VALUES (#{arg_array(video_array)}) + SQL + + PG_DB.exec(request, args: video_array) + end + + def delete(index) + request = <<-SQL + DELETE FROM playlist_videos * + WHERE index = $1 + SQL + + PG_DB.exec(request, index) + end + + def delete_by_playlist(plid : String) + request = <<-SQL + DELETE FROM playlist_videos * + WHERE plid = $1 + SQL + + PG_DB.exec(request, plid) + end + + # ------------------- + # Salect + # ------------------- + + def select(plid : String, index : VideoIndex, offset, limit = 100) : Array(PlaylistVideo) + request = <<-SQL + SELECT * FROM playlist_videos + WHERE plid = $1 + ORDER BY array_position($2, index) + LIMIT $3 + OFFSET $4 + SQL + + return PG_DB.query_all(request, plid, index, limit, offset, as: PlaylistVideo) + end + + def select_index(plid : String, vid : String) : Int64? + request = <<-SQL + SELECT index FROM playlist_videos + WHERE plid = $1 AND id = $2 + LIMIT 1 + SQL + + return PG_DB.query_one?(request, plid, vid, as: Int64) + end + + def select_one_id(plid : String, index : VideoIndex) : String? + request = <<-SQL + SELECT id FROM playlist_videos + WHERE plid = $1 + ORDER BY array_position($2, index) + LIMIT 1 + SQL + + return PG_DB.query_one?(request, plid, index, as: String) + end + + def select_ids(plid : String, index : VideoIndex, limit = 500) : Array(String) + request = <<-SQL + SELECT id FROM playlist_videos + WHERE plid = $1 + ORDER BY array_position($2, index) + LIMIT $3 + SQL + + return PG_DB.query_all(request, plid, index, limit, as: String) + end +end diff --git a/src/invidious/database/sessions.cr b/src/invidious/database/sessions.cr new file mode 100644 index 00000000..d5f85dd6 --- /dev/null +++ b/src/invidious/database/sessions.cr @@ -0,0 +1,74 @@ +require "./base.cr" + +module Invidious::Database::SessionIDs + extend self + + # ------------------- + # Insert + # ------------------- + + def insert(sid : String, email : String, handle_conflicts : Bool = false) + request = <<-SQL + INSERT INTO session_ids + VALUES ($1, $2, $3) + SQL + + request += " ON CONFLICT (id) DO NOTHING" if handle_conflicts + + PG_DB.exec(request, sid, email, Time.utc) + end + + # ------------------- + # Delete + # ------------------- + + def delete(*, sid : String) + request = <<-SQL + DELETE FROM session_ids * + WHERE id = $1 + SQL + + PG_DB.exec(request, sid) + end + + def delete(*, email : String) + request = <<-SQL + DELETE FROM session_ids * + WHERE email = $1 + SQL + + PG_DB.exec(request, email) + end + + def delete(*, sid : String, email : String) + request = <<-SQL + DELETE FROM session_ids * + WHERE id = $1 AND email = $2 + SQL + + PG_DB.exec(request, sid, email) + end + + # ------------------- + # Select + # ------------------- + + def select_email(sid : String) : String? + request = <<-SQL + SELECT email FROM session_ids + WHERE id = $1 + SQL + + PG_DB.query_one?(request, sid, as: String) + end + + def select_all(email : String) : Array({session: String, issued: Time}) + request = <<-SQL + SELECT id, issued FROM session_ids + WHERE email = $1 + ORDER BY issued DESC + SQL + + PG_DB.query_all(request, email, as: {session: String, issued: Time}) + end +end diff --git a/src/invidious/database/statistics.cr b/src/invidious/database/statistics.cr new file mode 100644 index 00000000..1df549e2 --- /dev/null +++ b/src/invidious/database/statistics.cr @@ -0,0 +1,49 @@ +require "./base.cr" + +module Invidious::Database::Statistics + extend self + + # ------------------- + # User stats + # ------------------- + + def count_users_total : Int64 + request = <<-SQL + SELECT count(*) FROM users + SQL + + PG_DB.query_one(request, as: Int64) + end + + def count_users_active_1m : Int64 + request = <<-SQL + SELECT count(*) FROM users + WHERE CURRENT_TIMESTAMP - updated < '6 months' + SQL + + PG_DB.query_one(request, as: Int64) + end + + def count_users_active_6m : Int64 + request = <<-SQL + SELECT count(*) FROM users + WHERE CURRENT_TIMESTAMP - updated < '1 month' + SQL + + PG_DB.query_one(request, as: Int64) + end + + # ------------------- + # Channel stats + # ------------------- + + def channel_last_update : Time? + request = <<-SQL + SELECT updated FROM channels + ORDER BY updated DESC + LIMIT 1 + SQL + + PG_DB.query_one?(request, as: Time) + end +end diff --git a/src/invidious/database/users.cr b/src/invidious/database/users.cr new file mode 100644 index 00000000..53724dbf --- /dev/null +++ b/src/invidious/database/users.cr @@ -0,0 +1,218 @@ +require "./base.cr" + +module Invidious::Database::Users + extend self + + # ------------------- + # Insert / delete + # ------------------- + + def insert(user : User, update_on_conflict : Bool = false) + user_array = user.to_a + user_array[4] = user_array[4].to_json # User preferences + + request = <<-SQL + INSERT INTO users + VALUES (#{arg_array(user_array)}) + SQL + + if update_on_conflict + request += <<-SQL + ON CONFLICT (email) DO UPDATE + SET updated = $1, subscriptions = $3 + SQL + end + + PG_DB.exec(request, args: user_array) + end + + def delete(user : User) + request = <<-SQL + DELETE FROM users * + WHERE email = $1 + SQL + + PG_DB.exec(request, user.email) + end + + # ------------------- + # Update (history) + # ------------------- + + def update_watch_history(user : User) + request = <<-SQL + UPDATE users + SET watched = $1 + WHERE email = $2 + SQL + + PG_DB.exec(request, user.watched, user.email) + end + + def mark_watched(user : User, vid : String) + request = <<-SQL + UPDATE users + SET watched = array_append(watched, $1) + WHERE email = $2 + SQL + + PG_DB.exec(request, vid, user.email) + end + + def mark_unwatched(user : User, vid : String) + request = <<-SQL + UPDATE users + SET watched = array_remove(watched, $1) + WHERE email = $2 + SQL + + PG_DB.exec(request, vid, user.email) + end + + def clear_watch_history(user : User) + request = <<-SQL + UPDATE users + SET watched = '{}' + WHERE email = $1 + SQL + + PG_DB.exec(request, user.email) + end + + # ------------------- + # Update (channels) + # ------------------- + + def update_subscriptions(user : User) + request = <<-SQL + UPDATE users + SET feed_needs_update = true, subscriptions = $1 + WHERE email = $2 + SQL + + PG_DB.exec(request, user.subscriptions, user.email) + end + + def subscribe_channel(user : User, ucid : String) + request = <<-SQL + UPDATE users + SET feed_needs_update = true, + subscriptions = array_append(subscriptions,$1) + WHERE email = $2 + SQL + + PG_DB.exec(request, ucid, user.email) + end + + def unsubscribe_channel(user : User, ucid : String) + request = <<-SQL + UPDATE users + SET feed_needs_update = true, + subscriptions = array_remove(subscriptions, $1) + WHERE email = $2 + SQL + + PG_DB.exec(request, ucid, user.email) + end + + # ------------------- + # Update (notifs) + # ------------------- + + def add_notification(video : ChannelVideo) + request = <<-SQL + UPDATE users + SET notifications = array_append(notifications, $1), + feed_needs_update = true + WHERE $2 = ANY(subscriptions) + SQL + + PG_DB.exec(request, video.id, video.ucid) + end + + def remove_notification(user : User, vid : String) + request = <<-SQL + UPDATE users + SET notifications = array_remove(notifications, $1) + WHERE email = $2 + SQL + + PG_DB.exec(request, vid, user.email) + end + + def clear_notifications(user : User) + request = <<-SQL + UPDATE users + SET notifications = '{}', updated = $1 + WHERE email = $2 + SQL + + PG_DB.exec(request, Time.utc, user.email) + end + + # ------------------- + # Update (misc) + # ------------------- + + def update_preferences(user : User) + request = <<-SQL + UPDATE users + SET preferences = $1 + WHERE email = $2 + SQL + + PG_DB.exec(request, user.preferences.to_json, user.email) + end + + def update_password(user : User, pass : String) + request = <<-SQL + UPDATE users + SET password = $1 + WHERE email = $2 + SQL + + PG_DB.exec(request, user.email, pass) + end + + # ------------------- + # Select + # ------------------- + + def select(*, email : String) : User? + request = <<-SQL + SELECT * FROM users + WHERE email = $1 + SQL + + return PG_DB.query_one?(request, email, as: User) + end + + # Same as select, but can raise an exception + def select!(*, email : String) : User + request = <<-SQL + SELECT * FROM users + WHERE email = $1 + SQL + + return PG_DB.query_one(request, email, as: User) + end + + def select(*, token : String) : User? + request = <<-SQL + SELECT * FROM users + WHERE token = $1 + SQL + + return PG_DB.query_one?(request, token, as: User) + end + + def select_notifications(user : User) : Array(String) + request = <<-SQL + SELECT notifications + FROM users + WHERE email = $1 + SQL + + return PG_DB.query_one(request, user.email, as: Array(String)) + end +end diff --git a/src/invidious/database/videos.cr b/src/invidious/database/videos.cr new file mode 100644 index 00000000..e1fa01c3 --- /dev/null +++ b/src/invidious/database/videos.cr @@ -0,0 +1,43 @@ +require "./base.cr" + +module Invidious::Database::Videos + extend self + + def insert(video : Video) + request = <<-SQL + INSERT INTO videos + VALUES ($1, $2, $3) + ON CONFLICT (id) DO NOTHING + SQL + + PG_DB.exec(request, video.id, video.info.to_json, video.updated) + end + + def delete(id) + request = <<-SQL + DELETE FROM videos * + WHERE id = $1 + SQL + + PG_DB.exec(request, id) + end + + def update(video : Video) + request = <<-SQL + UPDATE videos + SET (id, info, updated) = ($1, $2, $3) + WHERE id = $1 + SQL + + PG_DB.exec(request, video.id, video.info.to_json, video.updated) + end + + def select(id : String) : Video? + request = <<-SQL + SELECT * FROM videos + WHERE id = $1 + SQL + + return PG_DB.query_one?(request, id, as: Video) + end +end diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr index e1d02563..26c38669 100644 --- a/src/invidious/helpers/errors.cr +++ b/src/invidious/helpers/errors.cr @@ -22,31 +22,62 @@ def github_details(summary : String, content : String) return HTML.escape(details) end -def error_template_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception) +def error_template_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, exception : Exception) if exception.is_a?(InfoException) return error_template_helper(env, locale, status_code, exception.message || "") end + env.response.content_type = "text/html" env.response.status_code = status_code - issue_template = %(Title: `#{exception.message} (#{exception.class})`) + + issue_title = "#{exception.message} (#{exception.class})" + + issue_template = %(Title: `#{issue_title}`) issue_template += %(\nDate: `#{Time::Format::ISO_8601_DATE_TIME.format(Time.utc)}`) issue_template += %(\nRoute: `#{env.request.resource}`) issue_template += %(\nVersion: `#{SOFTWARE["version"]} @ #{SOFTWARE["branch"]}`) # issue_template += github_details("Preferences", env.get("preferences").as(Preferences).to_pretty_json) issue_template += github_details("Backtrace", exception.inspect_with_backtrace) + + # URLs for the error message below + url_faq = "https://github.com/iv-org/documentation/blob/master/FAQ.md" + url_search_issues = "https://github.com/iv-org/invidious/issues" + + url_switch = "https://redirect.invidious.io" + env.request.resource + + url_new_issue = "https://github.com/iv-org/invidious/issues/new" + url_new_issue += "?labels=bug&template=bug_report.md&title=" + url_new_issue += URI.encode_www_form("[Bug] " + issue_title) + error_message = <<-END_HTML - Looks like you've found a bug in Invidious. Please open a new issue - <a href="https://github.com/iv-org/invidious/issues">on GitHub</a> - and include the following text in your message: - <pre style="padding: 20px; background: rgba(0, 0, 0, 0.12345);">#{issue_template}</pre> + <div class="error_message"> + <h2>#{translate(locale, "crash_page_you_found_a_bug")}</h2> + <br/><br/> + + <p><b>#{translate(locale, "crash_page_before_reporting")}</b></p> + <ul> + <li>#{translate(locale, "crash_page_refresh", env.request.resource)}</li> + <li>#{translate(locale, "crash_page_switch_instance", url_switch)}</li> + <li>#{translate(locale, "crash_page_read_the_faq", url_faq)}</li> + <li>#{translate(locale, "crash_page_search_issue", url_search_issues)}</li> + </ul> + + <br/> + <p>#{translate(locale, "crash_page_report_issue", url_new_issue)}</p> + + <!-- TODO: Add a "copy to clipboard" button --> + <pre style="padding: 20px; background: rgba(0, 0, 0, 0.12345);">#{issue_template}</pre> + </div> END_HTML - next_steps = error_redirect_helper(env, locale) + # Don't show the usual "next steps" widget. The same options are + # proposed above the error message, just worded differently. + next_steps = "" return templated "error" end -def error_template_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String) +def error_template_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, message : String) env.response.content_type = "text/html" env.response.status_code = status_code error_message = translate(locale, message) @@ -58,7 +89,7 @@ macro error_atom(*args) error_atom_helper(env, locale, {{*args}}) end -def error_atom_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception) +def error_atom_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, exception : Exception) if exception.is_a?(InfoException) return error_atom_helper(env, locale, status_code, exception.message || "") end @@ -67,7 +98,7 @@ def error_atom_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::A return "<error>#{exception.inspect_with_backtrace}</error>" end -def error_atom_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String) +def error_atom_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, message : String) env.response.content_type = "application/atom+xml" env.response.status_code = status_code return "<error>#{message}</error>" @@ -77,7 +108,7 @@ macro error_json(*args) error_json_helper(env, locale, {{*args}}) end -def error_json_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception, additional_fields : Hash(String, Object) | Nil) +def error_json_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, exception : Exception, additional_fields : Hash(String, Object) | Nil) if exception.is_a?(InfoException) return error_json_helper(env, locale, status_code, exception.message || "", additional_fields) end @@ -90,11 +121,11 @@ def error_json_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::A return error_message.to_json end -def error_json_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception) +def error_json_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, exception : Exception) return error_json_helper(env, locale, status_code, exception, nil) end -def error_json_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String, additional_fields : Hash(String, Object) | Nil) +def error_json_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, message : String, additional_fields : Hash(String, Object) | Nil) env.response.content_type = "application/json" env.response.status_code = status_code error_message = {"error" => message} @@ -104,11 +135,11 @@ def error_json_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::A return error_message.to_json end -def error_json_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String) +def error_json_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, message : String) error_json_helper(env, locale, status_code, message, nil) end -def error_redirect_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil) +def error_redirect_helper(env : HTTP::Server::Context, locale : String?) request_path = env.request.path if request_path.starts_with?("/search") || request_path.starts_with?("/watch") || @@ -132,8 +163,6 @@ def error_redirect_helper(env : HTTP::Server::Context, locale : Hash(String, JSO </li> </ul> END_HTML - - return next_step_html else return "" end diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index 045b6701..d140a858 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -97,18 +97,18 @@ class AuthHandler < Kemal::Handler if token = env.request.headers["Authorization"]? token = JSON.parse(URI.decode_www_form(token.lchop("Bearer "))) session = URI.decode_www_form(token["session"].as_s) - scopes, expire, signature = validate_request(token, session, env.request, HMAC_KEY, PG_DB, nil) + scopes, expire, signature = validate_request(token, session, env.request, HMAC_KEY, nil) - if email = PG_DB.query_one?("SELECT email FROM session_ids WHERE id = $1", session, as: String) - user = PG_DB.query_one("SELECT * FROM users WHERE email = $1", email, as: User) + if email = Invidious::Database::SessionIDs.select_email(session) + user = Invidious::Database::Users.select!(email: email) end elsif sid = env.request.cookies["SID"]?.try &.value if sid.starts_with? "v1:" raise "Cannot use token as SID" end - if email = PG_DB.query_one?("SELECT email FROM session_ids WHERE id = $1", sid, as: String) - user = PG_DB.query_one("SELECT * FROM users WHERE email = $1", email, as: User) + if email = Invidious::Database::SessionIDs.select_email(sid) + user = Invidious::Database::Users.select!(email: email) end scopes = [":*"] diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index fb33df1c..c3b53339 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -22,193 +22,6 @@ struct Annotation property annotations : String end -struct ConfigPreferences - include YAML::Serializable - - property annotations : Bool = false - property annotations_subscribed : Bool = false - property autoplay : Bool = false - property captions : Array(String) = ["", "", ""] - property comments : Array(String) = ["youtube", ""] - property continue : Bool = false - property continue_autoplay : Bool = true - property dark_mode : String = "" - property latest_only : Bool = false - property listen : Bool = false - property local : Bool = false - property locale : String = "en-US" - property max_results : Int32 = 40 - property notifications_only : Bool = false - property player_style : String = "invidious" - property quality : String = "hd720" - property quality_dash : String = "auto" - property default_home : String? = "Popular" - property feed_menu : Array(String) = ["Popular", "Trending", "Subscriptions", "Playlists"] - property automatic_instance_redirect : Bool = false - property related_videos : Bool = true - property sort : String = "published" - property speed : Float32 = 1.0_f32 - property thin_mode : Bool = false - property unseen_only : Bool = false - property video_loop : Bool = false - property extend_desc : Bool = false - property volume : Int32 = 100 - property vr_mode : Bool = true - property show_nick : Bool = true - - def to_tuple - {% begin %} - { - {{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}} - } - {% end %} - end -end - -class Config - include YAML::Serializable - - property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions) - property feed_threads : Int32 = 1 # Number of threads to use for updating feeds - property output : String = "STDOUT" # Log file path or STDOUT - property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr - property db : DBConfig? = nil # Database configuration with separate parameters (username, hostname, etc) - - @[YAML::Field(converter: Preferences::URIConverter)] - property database_url : URI = URI.parse("") # Database configuration using 12-Factor "Database URL" syntax - property decrypt_polling : Bool = true # Use polling to keep decryption function up to date - property full_refresh : Bool = false # Used for crawling channels: threads should check all videos uploaded by a channel - property https_only : Bool? # Used to tell Invidious it is behind a proxy, so links to resources should be https:// - property hmac_key : String? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions - property domain : String? # Domain to be used for links to resources on the site where an absolute URL is required - property use_pubsub_feeds : Bool | Int32 = false # Subscribe to channels using PubSubHubbub (requires domain, hmac_key) - property popular_enabled : Bool = true - property captcha_enabled : Bool = true - property login_enabled : Bool = true - property registration_enabled : Bool = true - property statistics_enabled : Bool = false - property admins : Array(String) = [] of String - property external_port : Int32? = nil - property default_user_preferences : ConfigPreferences = ConfigPreferences.from_yaml("") - property dmca_content : Array(String) = [] of String # For compliance with DMCA, disables download widget using list of video IDs - property check_tables : Bool = false # Check table integrity, automatically try to add any missing columns, create tables, etc. - property cache_annotations : Bool = false # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards - property banner : String? = nil # Optional banner to be displayed along top of page for announcements, etc. - property hsts : Bool? = true # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely - property disable_proxy : Bool? | Array(String)? = false # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local' - - @[YAML::Field(converter: Preferences::FamilyConverter)] - property force_resolve : Socket::Family = Socket::Family::UNSPEC # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729) - property port : Int32 = 3000 # Port to listen for connections (overrided by command line argument) - property host_binding : String = "0.0.0.0" # Host to bind (overrided by command line argument) - property pool_size : Int32 = 100 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) - property use_quic : Bool = true # Use quic transport for youtube api - - @[YAML::Field(converter: Preferences::StringToCookies)] - property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format - property captcha_key : String? = nil # Key for Anti-Captcha - property captcha_api_url : String = "https://api.anti-captcha.com" # API URL for Anti-Captcha - - def disabled?(option) - case disabled = CONFIG.disable_proxy - when Bool - return disabled - when Array - if disabled.includes? option - return true - else - return false - end - else - return false - end - end - - def self.load - # Load config from file or YAML string env var - env_config_file = "INVIDIOUS_CONFIG_FILE" - env_config_yaml = "INVIDIOUS_CONFIG" - - config_file = ENV.has_key?(env_config_file) ? ENV.fetch(env_config_file) : "config/config.yml" - config_yaml = ENV.has_key?(env_config_yaml) ? ENV.fetch(env_config_yaml) : File.read(config_file) - - config = Config.from_yaml(config_yaml) - - # Update config from env vars (upcased and prefixed with "INVIDIOUS_") - {% for ivar in Config.instance_vars %} - {% env_id = "INVIDIOUS_#{ivar.id.upcase}" %} - - if ENV.has_key?({{env_id}}) - # puts %(Config.{{ivar.id}} : Loading from env var {{env_id}}) - env_value = ENV.fetch({{env_id}}) - success = false - - # Use YAML converter if specified - {% ann = ivar.annotation(::YAML::Field) %} - {% if ann && ann[:converter] %} - puts %(Config.{{ivar.id}} : Parsing "#{env_value}" as {{ivar.type}} with {{ann[:converter]}} converter) - config.{{ivar.id}} = {{ann[:converter]}}.from_yaml(YAML::ParseContext.new, YAML::Nodes.parse(ENV.fetch({{env_id}})).nodes[0]) - puts %(Config.{{ivar.id}} : Set to #{config.{{ivar.id}}}) - success = true - - # Use regular YAML parser otherwise - {% else %} - {% ivar_types = ivar.type.union? ? ivar.type.union_types : [ivar.type] %} - # Sort types to avoid parsing nulls and numbers as strings - {% ivar_types = ivar_types.sort_by { |ivar_type| ivar_type == Nil ? 0 : ivar_type == Int32 ? 1 : 2 } %} - {{ivar_types}}.each do |ivar_type| - if !success - begin - # puts %(Config.{{ivar.id}} : Trying to parse "#{env_value}" as #{ivar_type}) - config.{{ivar.id}} = ivar_type.from_yaml(env_value) - puts %(Config.{{ivar.id}} : Set to #{config.{{ivar.id}}} (#{ivar_type})) - success = true - rescue - # nop - end - end - end - {% end %} - - # Exit on fail - if !success - puts %(Config.{{ivar.id}} failed to parse #{env_value} as {{ivar.type}}) - exit(1) - end - end - {% end %} - - # Build database_url from db.* if it's not set directly - if config.database_url.to_s.empty? - if db = config.db - config.database_url = URI.new( - scheme: "postgres", - user: db.user, - password: db.password, - host: db.host, - port: db.port, - path: db.dbname, - ) - else - puts "Config : Either database_url or db.* is required" - exit(1) - end - end - - return config - end -end - -struct DBConfig - include YAML::Serializable - - property user : String - property password : String - property host : String - property port : Int32 - property dbname : String -end - def login_req(f_req) data = { # Unfortunately there's not much information available on `bgRequest`; part of Google's BotGuard @@ -247,277 +60,7 @@ def html_to_content(description_html : String) return description end -def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) - extract_items(initial_data, author_fallback, author_id_fallback).select(&.is_a?(SearchVideo)).map(&.as(SearchVideo)) -end - -def extract_item(item : JSON::Any, author_fallback : String? = nil, author_id_fallback : String? = nil) - if i = (item["videoRenderer"]? || item["gridVideoRenderer"]?) - video_id = i["videoId"].as_s - title = i["title"].try { |t| t["simpleText"]?.try &.as_s || t["runs"]?.try &.as_a.map(&.["text"].as_s).join("") } || "" - - author_info = i["ownerText"]?.try &.["runs"]?.try &.as_a?.try &.[0]? - author = author_info.try &.["text"].as_s || author_fallback || "" - author_id = author_info.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]["browseId"].as_s || author_id_fallback || "" - - published = i["publishedTimeText"]?.try &.["simpleText"]?.try { |t| decode_date(t.as_s) } || Time.local - view_count = i["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64 - description_html = i["descriptionSnippet"]?.try { |t| parse_content(t) } || "" - length_seconds = i["lengthText"]?.try &.["simpleText"]?.try &.as_s.try { |t| decode_length_seconds(t) } || - i["thumbnailOverlays"]?.try &.as_a.find(&.["thumbnailOverlayTimeStatusRenderer"]?).try &.["thumbnailOverlayTimeStatusRenderer"]? - .try &.["text"]?.try &.["simpleText"]?.try &.as_s.try { |t| decode_length_seconds(t) } || 0 - - live_now = false - premium = false - - premiere_timestamp = i["upcomingEventData"]?.try &.["startTime"]?.try { |t| Time.unix(t.as_s.to_i64) } - - i["badges"]?.try &.as_a.each do |badge| - b = badge["metadataBadgeRenderer"] - case b["label"].as_s - when "LIVE NOW" - live_now = true - when "New", "4K", "CC" - # TODO - when "Premium" - # TODO: Potentially available as i["topStandaloneBadge"]["metadataBadgeRenderer"] - premium = true - else nil # Ignore - end - end - - SearchVideo.new({ - title: title, - id: video_id, - author: author, - ucid: author_id, - published: published, - views: view_count, - description_html: description_html, - length_seconds: length_seconds, - live_now: live_now, - premium: premium, - premiere_timestamp: premiere_timestamp, - }) - elsif i = item["channelRenderer"]? - author = i["title"]["simpleText"]?.try &.as_s || author_fallback || "" - author_id = i["channelId"]?.try &.as_s || author_id_fallback || "" - - author_thumbnail = i["thumbnail"]["thumbnails"]?.try &.as_a[0]?.try &.["url"]?.try &.as_s || "" - subscriber_count = i["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s.try { |s| short_text_to_number(s.split(" ")[0]) } || 0 - - auto_generated = false - auto_generated = true if !i["videoCountText"]? - video_count = i["videoCountText"]?.try &.["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0 - description_html = i["descriptionSnippet"]?.try { |t| parse_content(t) } || "" - - SearchChannel.new({ - author: author, - ucid: author_id, - author_thumbnail: author_thumbnail, - subscriber_count: subscriber_count, - video_count: video_count, - description_html: description_html, - auto_generated: auto_generated, - }) - elsif i = item["gridPlaylistRenderer"]? - title = i["title"]["runs"].as_a[0]?.try &.["text"].as_s || "" - plid = i["playlistId"]?.try &.as_s || "" - - video_count = i["videoCountText"]["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0 - playlist_thumbnail = i["thumbnail"]["thumbnails"][0]?.try &.["url"]?.try &.as_s || "" - - SearchPlaylist.new({ - title: title, - id: plid, - author: author_fallback || "", - ucid: author_id_fallback || "", - video_count: video_count, - videos: [] of SearchPlaylistVideo, - thumbnail: playlist_thumbnail, - }) - elsif i = item["playlistRenderer"]? - title = i["title"]["simpleText"]?.try &.as_s || "" - plid = i["playlistId"]?.try &.as_s || "" - - video_count = i["videoCount"]?.try &.as_s.to_i || 0 - playlist_thumbnail = i["thumbnails"].as_a[0]?.try &.["thumbnails"]?.try &.as_a[0]?.try &.["url"].as_s || "" - - author_info = i["shortBylineText"]?.try &.["runs"]?.try &.as_a?.try &.[0]? - author = author_info.try &.["text"].as_s || author_fallback || "" - author_id = author_info.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]["browseId"].as_s || author_id_fallback || "" - - videos = i["videos"]?.try &.as_a.map do |v| - v = v["childVideoRenderer"] - v_title = v["title"]["simpleText"]?.try &.as_s || "" - v_id = v["videoId"]?.try &.as_s || "" - v_length_seconds = v["lengthText"]?.try &.["simpleText"]?.try { |t| decode_length_seconds(t.as_s) } || 0 - SearchPlaylistVideo.new({ - title: v_title, - id: v_id, - length_seconds: v_length_seconds, - }) - end || [] of SearchPlaylistVideo - - # TODO: i["publishedTimeText"]? - - SearchPlaylist.new({ - title: title, - id: plid, - author: author, - ucid: author_id, - video_count: video_count, - videos: videos, - thumbnail: playlist_thumbnail, - }) - elsif i = item["radioRenderer"]? # Mix - # TODO - elsif i = item["showRenderer"]? # Show - # TODO - elsif i = item["shelfRenderer"]? - elsif i = item["horizontalCardListRenderer"]? - elsif i = item["searchPyvRenderer"]? # Ad - end -end - -def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) - items = [] of SearchItem - - channel_v2_response = initial_data - .try &.["continuationContents"]? - .try &.["gridContinuation"]? - .try &.["items"]? - - if channel_v2_response - channel_v2_response.try &.as_a.each { |item| - extract_item(item, author_fallback, author_id_fallback) - .try { |t| items << t } - } - else - initial_data.try { |t| t["contents"]? || t["response"]? } - .try { |t| t["twoColumnBrowseResultsRenderer"]?.try &.["tabs"].as_a.select(&.["tabRenderer"]?.try &.["selected"].as_bool)[0]?.try &.["tabRenderer"]["content"] || - t["twoColumnSearchResultsRenderer"]?.try &.["primaryContents"] || - t["continuationContents"]? } - .try { |t| t["sectionListRenderer"]? || t["sectionListContinuation"]? } - .try &.["contents"].as_a - .each { |c| c.try &.["itemSectionRenderer"]?.try &.["contents"].as_a - .try { |t| t[0]?.try &.["shelfRenderer"]?.try &.["content"]["expandedShelfContentsRenderer"]?.try &.["items"].as_a || - t[0]?.try &.["gridRenderer"]?.try &.["items"].as_a || t } - .each { |item| - extract_item(item, author_fallback, author_id_fallback) - .try { |t| items << t } - } } - end - - items -end - -def check_enum(db, enum_name, struct_type = nil) - return # TODO - - if !db.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool) - LOGGER.info("check_enum: CREATE TYPE #{enum_name}") - - db.using_connection do |conn| - conn.as(PG::Connection).exec_all(File.read("config/sql/#{enum_name}.sql")) - end - end -end - -def check_table(db, table_name, struct_type = nil) - # Create table if it doesn't exist - begin - db.exec("SELECT * FROM #{table_name} LIMIT 0") - rescue ex - LOGGER.info("check_table: check_table: CREATE TABLE #{table_name}") - - db.using_connection do |conn| - conn.as(PG::Connection).exec_all(File.read("config/sql/#{table_name}.sql")) - end - end - - return if !struct_type - - struct_array = struct_type.type_array - column_array = get_column_array(db, table_name) - column_types = File.read("config/sql/#{table_name}.sql").match(/CREATE TABLE public\.#{table_name}\n\((?<types>[\d\D]*?)\);/) - .try &.["types"].split(",").map { |line| line.strip }.reject &.starts_with?("CONSTRAINT") - - return if !column_types - - struct_array.each_with_index do |name, i| - if name != column_array[i]? - if !column_array[i]? - new_column = column_types.select { |line| line.starts_with? name }[0] - LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}") - db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") - next - end - - # Column doesn't exist - if !column_array.includes? name - new_column = column_types.select { |line| line.starts_with? name }[0] - db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") - end - - # Column exists but in the wrong position, rotate - if struct_array.includes? column_array[i] - until name == column_array[i] - new_column = column_types.select { |line| line.starts_with? column_array[i] }[0]?.try &.gsub("#{column_array[i]}", "#{column_array[i]}_new") - - # There's a column we didn't expect - if !new_column - LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]}") - db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") - - column_array = get_column_array(db, table_name) - next - end - - LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}") - db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") - - LOGGER.info("check_table: UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}") - db.exec("UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}") - - LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") - db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") - - LOGGER.info("check_table: ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}") - db.exec("ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}") - - column_array = get_column_array(db, table_name) - end - else - LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") - db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") - end - end - end - - return if column_array.size <= struct_array.size - - column_array.each do |column| - if !struct_array.includes? column - LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE") - db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE") - end - end -end - -def get_column_array(db, table_name) - column_array = [] of String - db.query("SELECT * FROM #{table_name} LIMIT 0") do |rs| - rs.column_count.times do |i| - column = rs.as(PG::ResultSet).field(i) - column_array << column.name - end - end - - return column_array -end - -def cache_annotation(db, id, annotations) +def cache_annotation(id, annotations) if !CONFIG.cache_annotations return end @@ -535,14 +78,14 @@ def cache_annotation(db, id, annotations) end end - db.exec("INSERT INTO annotations VALUES ($1, $2) ON CONFLICT DO NOTHING", id, annotations) if has_legacy_annotations + Invidious::Database::Annotations.insert(id, annotations) if has_legacy_annotations end def create_notification_stream(env, topics, connection_channel) connection = Channel(PQ::Notification).new(8) connection_channel.send({true, connection}) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale since = env.params.query["since"]?.try &.to_i? id = 0 @@ -556,9 +99,9 @@ def create_notification_stream(env, topics, connection_channel) published = Time.utc - Time::Span.new(days: time_span[0], hours: time_span[1], minutes: time_span[2], seconds: time_span[3]) video_id = TEST_IDS[rand(TEST_IDS.size)] - video = get_video(video_id, PG_DB) + video = get_video(video_id) video.published = published - response = JSON.parse(video.to_json(locale)) + response = JSON.parse(video.to_json(locale, nil)) if fields_text = env.params.query["fields"]? begin @@ -587,11 +130,12 @@ def create_notification_stream(env, topics, connection_channel) spawn do begin if since + since_unix = Time.unix(since.not_nil!) + topics.try &.each do |topic| case topic when .match(/UC[A-Za-z0-9_-]{22}/) - PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid = $1 AND published > $2 ORDER BY published DESC LIMIT 15", - topic, Time.unix(since.not_nil!), as: ChannelVideo).each do |video| + Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video| response = JSON.parse(video.to_json(locale)) if fields_text = env.params.query["fields"]? @@ -632,9 +176,9 @@ def create_notification_stream(env, topics, connection_channel) next end - video = get_video(video_id, PG_DB) + video = get_video(video_id) video.published = Time.unix(published) - response = JSON.parse(video.to_json(locale)) + response = JSON.parse(video.to_json(locale, nil)) if fields_text = env.params.query["fields"]? begin diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index 7ffdfdcc..e88e4491 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -1,74 +1,111 @@ -LOCALES = { - "ar" => load_locale("ar"), # Arabic - "bn_BD" => load_locale("bn_BD"), # Bengali (Bangladesh) - "cs" => load_locale("cs"), # Czech - "da" => load_locale("da"), # Danish - "de" => load_locale("de"), # German - "el" => load_locale("el"), # Greek - "en-US" => load_locale("en-US"), # English (US) - "eo" => load_locale("eo"), # Esperanto - "es" => load_locale("es"), # Spanish - "eu" => load_locale("eu"), # Basque - "fa" => load_locale("fa"), # Persian - "fi" => load_locale("fi"), # Finnish - "fr" => load_locale("fr"), # French - "he" => load_locale("he"), # Hebrew - "hr" => load_locale("hr"), # Croatian - "hu-HU" => load_locale("hu-HU"), # Hungarian - "id" => load_locale("id"), # Indonesian - "is" => load_locale("is"), # Icelandic - "it" => load_locale("it"), # Italian - "ja" => load_locale("ja"), # Japanese - "ko" => load_locale("ko"), # Korean - "lt" => load_locale("lt"), # Lithuanian - "nb-NO" => load_locale("nb-NO"), # Norwegian Bokmål - "nl" => load_locale("nl"), # Dutch - "pl" => load_locale("pl"), # Polish - "pt-BR" => load_locale("pt-BR"), # Portuguese (Brazil) - "pt-PT" => load_locale("pt-PT"), # Portuguese (Portugal) - "ro" => load_locale("ro"), # Romanian - "ru" => load_locale("ru"), # Russian - "si" => load_locale("si"), # Sinhala - "sk" => load_locale("sk"), # Slovak - "sr" => load_locale("sr"), # Serbian - "sr_Cyrl" => load_locale("sr_Cyrl"), # Serbian (cyrillic) - "sv-SE" => load_locale("sv-SE"), # Swedish - "tr" => load_locale("tr"), # Turkish - "uk" => load_locale("uk"), # Ukrainian - "vi" => load_locale("vi"), # Vietnamese - "zh-CN" => load_locale("zh-CN"), # Chinese (Simplified) - "zh-TW" => load_locale("zh-TW"), # Chinese (Traditional) +# "bn_BD" => load_locale("bn_BD"), # Bengali (Bangladesh) [Incomplete] +# "eu" => load_locale("eu"), # Basque [Incomplete] +# "sk" => load_locale("sk"), # Slovak [Incomplete] +LOCALES_LIST = { + "ar" => "العربية", # Arabic + "cs" => "Čeština", # Czech + "da" => "Dansk", # Danish + "de" => "Deutsch", # German + "el" => "Ελληνικά", # Greek + "en-US" => "English", # English + "eo" => "Esperanto", # Esperanto + "es" => "Español", # Spanish + "fa" => "فارسی", # Persian + "fi" => "Suomi", # Finnish + "fr" => "Français", # French + "he" => "עברית", # Hebrew + "hr" => "Hrvatski", # Croatian + "hu-HU" => "Magyar Nyelv", # Hungarian + "id" => "Bahasa Indonesia", # Indonesian + "is" => "Íslenska", # Icelandic + "it" => "Italiano", # Italian + "ja" => "日本語", # Japanese + "ko" => "한국어", # Korean + "lt" => "Lietuvių", # Lithuanian + "nb-NO" => "Norsk bokmål", # Norwegian Bokmål + "nl" => "Nederlands", # Dutch + "pl" => "Polski", # Polish + "pt" => "Português", # Portuguese + "pt-BR" => "Português Brasileiro", # Portuguese (Brazil) + "pt-PT" => "Português de Portugal", # Portuguese (Portugal) + "ro" => "Română", # Romanian + "ru" => "русский", # Russian + "sr" => "srpski (latinica)", # Serbian (Latin) + "sr_Cyrl" => "српски (ћирилица)", # Serbian (Cyrillic) + "sv-SE" => "Svenska", # Swedish + "tr" => "Türkçe", # Turkish + "uk" => "Українська", # Ukrainian + "vi" => "Tiếng Việt", # Vietnamese + "zh-CN" => "汉语", # Chinese (Simplified) + "zh-TW" => "漢語", # Chinese (Traditional) } -def load_locale(name) - return JSON.parse(File.read("locales/#{name}.json")).as_h +LOCALES = load_all_locales() + +CONTENT_REGIONS = { + "AE", "AR", "AT", "AU", "AZ", "BA", "BD", "BE", "BG", "BH", "BO", "BR", "BY", + "CA", "CH", "CL", "CO", "CR", "CY", "CZ", "DE", "DK", "DO", "DZ", "EC", "EE", + "EG", "ES", "FI", "FR", "GB", "GE", "GH", "GR", "GT", "HK", "HN", "HR", "HU", + "ID", "IE", "IL", "IN", "IQ", "IS", "IT", "JM", "JO", "JP", "KE", "KR", "KW", + "KZ", "LB", "LI", "LK", "LT", "LU", "LV", "LY", "MA", "ME", "MK", "MT", "MX", + "MY", "NG", "NI", "NL", "NO", "NP", "NZ", "OM", "PA", "PE", "PG", "PH", "PK", + "PL", "PR", "PT", "PY", "QA", "RO", "RS", "RU", "SA", "SE", "SG", "SI", "SK", + "SN", "SV", "TH", "TN", "TR", "TW", "TZ", "UA", "UG", "US", "UY", "VE", "VN", + "YE", "ZA", "ZW", +} + +# Enum for the different types of number formats +enum NumberFormatting + None # Print the number as-is + Separator # Use a separator for thousands + Short # Use short notation (k/M/B) + HtmlSpan # Surround with <span id="count"></span> end -def translate(locale : Hash(String, JSON::Any) | Nil, translation : String, text : String | Nil = nil) - # if locale && !locale[translation]? - # puts "Could not find translation for #{translation.dump}" - # end - - if locale && locale[translation]? - case locale[translation] - when .as_h? - match_length = 0 - - locale[translation].as_h.each do |key, value| - if md = text.try &.match(/#{key}/) - if md[0].size >= match_length - translation = value.as_s - match_length = md[0].size - end +def load_all_locales + locales = {} of String => Hash(String, JSON::Any) + + LOCALES_LIST.each_key do |name| + locales[name] = JSON.parse(File.read("locales/#{name}.json")).as_h + end + + return locales +end + +def translate(locale : String?, key : String, text : String | Nil = nil) : String + # Log a warning if "key" doesn't exist in en-US locale and return + # that key as the text, so this is more or less transparent to the user. + if !LOCALES["en-US"].has_key?(key) + LOGGER.warn("i18n: Missing translation key \"#{key}\"") + return key + end + + # Default to english, whenever the locale doesn't exist, + # or the key requested has not been translated + if locale && LOCALES.has_key?(locale) && LOCALES[locale].has_key?(key) + raw_data = LOCALES[locale][key] + else + raw_data = LOCALES["en-US"][key] + end + + case raw_data + when .as_h? + # Init + translation = "" + match_length = 0 + + raw_data.as_h.each do |key, value| + if md = text.try &.match(/#{key}/) + if md[0].size >= match_length + translation = value.as_s + match_length = md[0].size end end - when .as_s? - if !locale[translation].as_s.empty? - translation = locale[translation].as_s - end - else - raise "Invalid translation #{translation}" end + when .as_s? + translation = raw_data.as_s + else + raise "Invalid translation \"#{raw_data}\"" end if text @@ -78,7 +115,43 @@ def translate(locale : Hash(String, JSON::Any) | Nil, translation : String, text return translation end -def translate_bool(locale : Hash(String, JSON::Any) | Nil, translation : Bool) +def translate_count(locale : String, key : String, count : Int, format = NumberFormatting::None) : String + # Fallback on english if locale doesn't exist + locale = "en-US" if !LOCALES.has_key?(locale) + + # Retrieve suffix + suffix = I18next::Plurals::RESOLVER.get_suffix(locale, count) + plural_key = key + suffix + + if LOCALES[locale].has_key?(plural_key) + translation = LOCALES[locale][plural_key].as_s + else + # Try #1: Fallback to singular in the same locale + singular_suffix = I18next::Plurals::RESOLVER.get_suffix(locale, 1) + + if LOCALES[locale].has_key?(key + singular_suffix) + translation = LOCALES[locale][key + singular_suffix].as_s + elsif locale != "en-US" + # Try #2: Fallback to english + translation = translate_count("en-US", key, count) + else + # Return key if we're already in english, as the tranlation is missing + LOGGER.warn("i18n: Missing translation key \"#{key}\"") + return key + end + end + + case format + when .separator? then count_txt = number_with_separator(count) + when .short? then count_txt = number_to_short_text(count) + when .html_span? then count_txt = "<span id=\"count\">" + count.to_s + "</span>" + else count_txt = count.to_s + end + + return translation.gsub("{{count}}", count_txt) +end + +def translate_bool(locale : String?, translation : Bool) case translation when true return translate(locale, "Yes") diff --git a/src/invidious/helpers/i18next.cr b/src/invidious/helpers/i18next.cr new file mode 100644 index 00000000..e84f88fb --- /dev/null +++ b/src/invidious/helpers/i18next.cr @@ -0,0 +1,511 @@ +# I18next-compatible implementation of plural forms +# +module I18next::Plurals + # ----------------------------------- + # I18next plural forms definition + # ----------------------------------- + + enum PluralForms + # One singular, one plural forms + Single_gt_one = 1 # E.g: French + Single_not_one = 2 # E.g: English + + # No plural forms (E.g: Azerbaijani) + None = 3 + + # One singular, two plural forms + Dual_Slavic = 4 # E.g: Russian + + # Special cases (rules used by only one or two language(s)) + Special_Arabic = 5 + Special_Czech_Slovak = 6 + Special_Polish_Kashubian = 7 + Special_Welsh = 8 + Special_Irish = 10 + Special_Scottish_Gaelic = 11 + Special_Icelandic = 12 + Special_Javanese = 13 + Special_Cornish = 14 + Special_Lithuanian = 15 + Special_Latvian = 16 + Special_Macedonian = 17 + Special_Mandinka = 18 + Special_Maltese = 19 + Special_Romanian = 20 + Special_Slovenian = 21 + Special_Hebrew = 22 + Special_Odia = 23 + end + + private PLURAL_SETS = { + PluralForms::Single_gt_one => [ + "ach", "ak", "am", "arn", "br", "fil", "fr", "gun", "ln", "mfe", "mg", + "mi", "oc", "pt", "pt-BR", "tg", "tl", "ti", "tr", "uz", "wa", + ], + PluralForms::Single_not_one => [ + "af", "an", "ast", "az", "bg", "bn", "ca", "da", "de", "dev", "el", "en", + "eo", "es", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi", + "hu", "hy", "ia", "it", "kk", "kn", "ku", "lb", "mai", "ml", "mn", "mr", + "nah", "nap", "nb", "ne", "nl", "nn", "no", "nso", "pa", "pap", "pms", + "ps", "pt-PT", "rm", "sco", "se", "si", "so", "son", "sq", "sv", "sw", + "ta", "te", "tk", "ur", "yo", + ], + PluralForms::None => [ + "ay", "bo", "cgg", "fa", "ht", "id", "ja", "jbo", "ka", "km", "ko", "ky", + "lo", "ms", "sah", "su", "th", "tt", "ug", "vi", "wo", "zh", + ], + PluralForms::Dual_Slavic => [ + "be", "bs", "cnr", "dz", "hr", "ru", "sr", "uk", + ], + } + + private PLURAL_SINGLES = { + "ar" => PluralForms::Special_Arabic, + "cs" => PluralForms::Special_Czech_Slovak, + "csb" => PluralForms::Special_Polish_Kashubian, + "cy" => PluralForms::Special_Welsh, + "ga" => PluralForms::Special_Irish, + "gd" => PluralForms::Special_Scottish_Gaelic, + "he" => PluralForms::Special_Hebrew, + "is" => PluralForms::Special_Icelandic, + "iw" => PluralForms::Special_Hebrew, + "jv" => PluralForms::Special_Javanese, + "kw" => PluralForms::Special_Cornish, + "lt" => PluralForms::Special_Lithuanian, + "lv" => PluralForms::Special_Latvian, + "mk" => PluralForms::Special_Macedonian, + "mnk" => PluralForms::Special_Mandinka, + "mt" => PluralForms::Special_Maltese, + "or" => PluralForms::Special_Odia, + "pl" => PluralForms::Special_Polish_Kashubian, + "ro" => PluralForms::Special_Romanian, + "sk" => PluralForms::Special_Czech_Slovak, + "sl" => PluralForms::Special_Slovenian, + } + + # These are the v1 and v2 compatible suffixes. + # The array indices matches the PluralForms enum above. + private NUMBERS = [ + [1, 2], # 1 + [1, 2], # 2 + [1], # 3 + [1, 2, 5], # 4 + [0, 1, 2, 3, 11, 100], # 5 + [1, 2, 5], # 6 + [1, 2, 5], # 7 + [1, 2, 3, 8], # 8 + [1, 2], # 9 (not used) + [1, 2, 3, 7, 11], # 10 + [1, 2, 3, 20], # 11 + [1, 2], # 12 + [0, 1], # 13 + [1, 2, 3, 4], # 14 + [1, 2, 10], # 15 + [1, 2, 0], # 16 + [1, 2], # 17 + [0, 1, 2], # 18 + [1, 2, 11, 20], # 19 + [1, 2, 20], # 20 + [5, 1, 2, 3], # 21 + [1, 2, 20, 21], # 22 + [2, 1], # 23 (Odia) + ] + + # ----------------------------------- + # I18next plural resolver class + # ----------------------------------- + + RESOLVER = Resolver.new + + class Resolver + private property forms = {} of String => PluralForms + property version : UInt8 = 3 + + # Options + property simplify_plural_suffix : Bool = true + + def initialize(version : Int = 3) + # Sanity checks + # V4 isn't supported, as it requires a full CLDR database. + if version > 4 || version == 0 + raise "Invalid i18next version: v#{version}." + elsif version == 4 + # Logger.error("Unsupported i18next version: v4. Falling back to v3") + @version = 3_u8 + else + @version = version.to_u8 + end + + self.init_rules + end + + def init_rules + # Look into sets + PLURAL_SETS.each do |form, langs| + langs.each { |lang| self.forms[lang] = form } + end + + # Add plurals from the "singles" set + self.forms.merge!(PLURAL_SINGLES) + end + + def get_plural_form(locale : String) : PluralForms + # Extract the ISO 639-1 or 639-2 code from an RFC 5646 language code, + # except for pt-BR and pt-PT which needs to be kept as-is. + if !locale.matches?(/^pt-(BR|PT)$/) + locale = locale.split('-')[0] + end + + return self.forms[locale] if self.forms[locale]? + + # If nothing was found, then use the most common form, i.e + # one singular and one plural, as in english. Not perfect, + # but better than yielding an exception at the user. + return PluralForms::Single_not_one + end + + def get_suffix(locale : String, count : Int) : String + # Checked count must be absolute. In i18next, `rule.noAbs` is used to + # determine if comparison should be done on a signed or unsigned integer, + # but this variable is never set, resulting in the comparison always + # being done on absolute numbers. + return get_suffix_retrocompat(locale, count.abs) + end + + # Emulate the `rule.numbers.size == 2 && rule.numbers[0] == 1` check + # from original i18next code + private def is_simple_plural(form : PluralForms) : Bool + case form + when .single_gt_one? then return true + when .single_not_one? then return true + when .special_icelandic? then return true + when .special_macedonian? then return true + else + return false + end + end + + private def get_suffix_retrocompat(locale : String, count : Int) : String + # Get plural form + plural_form = get_plural_form(locale) + + # Languages with no plural have the "_0" suffix + return "_0" if plural_form.none? + + # Get the index and suffix for this number + idx = SuffixIndex.get_index(plural_form, count) + + # Simple plurals are handled differently in all versions (but v4) + if @simplify_plural_suffix && is_simple_plural(plural_form) + return (idx == 1) ? "_plural" : "" + end + + # More complex plurals + # TODO: support v1 and v2 + # TODO: support `options.prepend` (v2 and v3) + # this.options.prepend && suffix.toString() ? this.options.prepend + suffix.toString() : suffix.toString() + # + # case @version + # when 1 + # suffix = SUFFIXES_V1_V2[plural_form.to_i][idx] + # return (suffix == 1) ? "" : return "_plural_#{suffix}" + # when 2 + # return "_#{suffix}" + # else # v3 + return "_#{idx}" + # end + end + end + + # ----------------------------- + # Plural functions + # ----------------------------- + + module SuffixIndex + def self.get_index(plural_form : PluralForms, count : Int) : UInt8 + case plural_form + when .single_gt_one? then return (count > 1) ? 1_u8 : 0_u8 + when .single_not_one? then return (count != 1) ? 1_u8 : 0_u8 + when .none? then return 0_u8 + when .dual_slavic? then return dual_slavic(count) + when .special_arabic? then return special_arabic(count) + when .special_czech_slovak? then return special_czech_slovak(count) + when .special_polish_kashubian? then return special_polish_kashubian(count) + when .special_welsh? then return special_welsh(count) + when .special_irish? then return special_irish(count) + when .special_scottish_gaelic? then return special_scottish_gaelic(count) + when .special_icelandic? then return special_icelandic(count) + when .special_javanese? then return special_javanese(count) + when .special_cornish? then return special_cornish(count) + when .special_lithuanian? then return special_lithuanian(count) + when .special_latvian? then return special_latvian(count) + when .special_macedonian? then return special_macedonian(count) + when .special_mandinka? then return special_mandinka(count) + when .special_maltese? then return special_maltese(count) + when .special_romanian? then return special_romanian(count) + when .special_slovenian? then return special_slovenian(count) + when .special_hebrew? then return special_hebrew(count) + when .special_odia? then return special_odia(count) + else + # default, if nothing matched above + return 0_u8 + end + end + + # Plural form of Slavic languages (E.g: Russian) + # + # Corresponds to i18next rule #4 + # Rule: (n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2) + # + def self.dual_slavic(count : Int) : UInt8 + n_mod_10 = count % 10 + n_mod_100 = count % 100 + + if n_mod_10 == 1 && n_mod_100 != 11 + return 0_u8 + elsif n_mod_10 >= 2 && n_mod_10 <= 4 && (n_mod_100 < 10 || n_mod_100 >= 20) + return 1_u8 + else + return 2_u8 + end + end + + # Plural form for Arabic language + # + # Corresponds to i18next rule #5 + # Rule: (n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 && n%100<=10 ? 3 : n%100>=11 ? 4 : 5) + # + def self.special_arabic(count : Int) : UInt8 + return count.to_u8 if (count == 0 || count == 1 || count == 2) + + n_mod_100 = count % 100 + + return 3_u8 if (n_mod_100 >= 3 && n_mod_100 <= 10) + return 4_u8 if (n_mod_100 >= 11) + return 5_u8 + end + + # Plural form for Czech and Slovak languages + # + # Corresponds to i18next rule #6 + # Rule: ((n==1) ? 0 : (n>=2 && n<=4) ? 1 : 2) + # + def self.special_czech_slovak(count : Int) : UInt8 + return 0_u8 if (count == 1) + return 1_u8 if (count >= 2 && count <= 4) + return 2_u8 + end + + # Plural form for Polish and Kashubian languages + # + # Corresponds to i18next rule #7 + # Rule: (n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2) + # + def self.special_polish_kashubian(count : Int) : UInt8 + return 0_u8 if (count == 1) + + n_mod_10 = count % 10 + n_mod_100 = count % 100 + + if n_mod_10 >= 2 && n_mod_10 <= 4 && (n_mod_100 < 10 || n_mod_100 >= 20) + return 1_u8 + else + return 2_u8 + end + end + + # Plural form for Welsh language + # + # Corresponds to i18next rule #8 + # Rule: ((n==1) ? 0 : (n==2) ? 1 : (n != 8 && n != 11) ? 2 : 3) + # + def self.special_welsh(count : Int) : UInt8 + return 0_u8 if (count == 1) + return 1_u8 if (count == 2) + return 2_u8 if (count != 8 && count != 11) + return 3_u8 + end + + # Plural form for Irish language + # + # Corresponds to i18next rule #10 + # Rule: (n==1 ? 0 : n==2 ? 1 : n<7 ? 2 : n<11 ? 3 : 4) + # + def self.special_irish(count : Int) : UInt8 + return 0_u8 if (count == 1) + return 1_u8 if (count == 2) + return 2_u8 if (count < 7) + return 3_u8 if (count < 11) + return 4_u8 + end + + # Plural form for Gaelic language + # + # Corresponds to i18next rule #11 + # Rule: ((n==1 || n==11) ? 0 : (n==2 || n==12) ? 1 : (n > 2 && n < 20) ? 2 : 3) + # + def self.special_scottish_gaelic(count : Int) : UInt8 + return 0_u8 if (count == 1 || count == 11) + return 1_u8 if (count == 2 || count == 12) + return 2_u8 if (count > 2 && count < 20) + return 3_u8 + end + + # Plural form for Icelandic language + # + # Corresponds to i18next rule #12 + # Rule: (n%10!=1 || n%100==11) + # + def self.special_icelandic(count : Int) : UInt8 + if (count % 10) != 1 || (count % 100) == 11 + return 1_u8 + else + return 0_u8 + end + end + + # Plural form for Javanese language + # + # Corresponds to i18next rule #13 + # Rule: (n !== 0) + # + def self.special_javanese(count : Int) : UInt8 + return (count != 0) ? 1_u8 : 0_u8 + end + + # Plural form for Cornish language + # + # Corresponds to i18next rule #14 + # Rule: ((n==1) ? 0 : (n==2) ? 1 : (n == 3) ? 2 : 3) + # + def self.special_cornish(count : Int) : UInt8 + return 0_u8 if count == 1 + return 1_u8 if count == 2 + return 2_u8 if count == 3 + return 3_u8 + end + + # Plural form for Lithuanian language + # + # Corresponds to i18next rule #15 + # Rule: (n%10==1 && n%100!=11 ? 0 : n%10>=2 && (n%100<10 || n%100>=20) ? 1 : 2) + # + def self.special_lithuanian(count : Int) : UInt8 + n_mod_10 = count % 10 + n_mod_100 = count % 100 + + if n_mod_10 == 1 && n_mod_100 != 11 + return 0_u8 + elsif n_mod_10 >= 2 && (n_mod_100 < 10 || n_mod_100 >= 20) + return 1_u8 + else + return 2_u8 + end + end + + # Plural form for Latvian language + # + # Corresponds to i18next rule #16 + # Rule: (n%10==1 && n%100!=11 ? 0 : n !== 0 ? 1 : 2) + # + def self.special_latvian(count : Int) : UInt8 + if (count % 10) == 1 && (count % 100) != 11 + return 0_u8 + elsif count != 0 + return 1_u8 + else + return 2_u8 + end + end + + # Plural form for Macedonian language + # + # Corresponds to i18next rule #17 + # Rule: (n==1 || n%10==1 && n%100!=11 ? 0 : 1) + # + def self.special_macedonian(count : Int) : UInt8 + if count == 1 || ((count % 10) == 1 && (count % 100) != 11) + return 0_u8 + else + return 1_u8 + end + end + + # Plural form for Mandinka language + # + # Corresponds to i18next rule #18 + # Rule: (n==0 ? 0 : n==1 ? 1 : 2) + # + def self.special_mandinka(count : Int) : UInt8 + return (count == 0 || count == 1) ? count.to_u8 : 2_u8 + end + + # Plural form for Maltese language + # + # Corresponds to i18next rule #19 + # Rule: (n==1 ? 0 : n==0 || ( n%100>1 && n%100<11) ? 1 : (n%100>10 && n%100<20 ) ? 2 : 3) + # + def self.special_maltese(count : Int) : UInt8 + return 0_u8 if count == 1 + return 1_u8 if count == 0 + + n_mod_100 = count % 100 + return 1_u8 if (n_mod_100 > 1 && n_mod_100 < 11) + return 2_u8 if (n_mod_100 > 10 && n_mod_100 < 20) + return 3_u8 + end + + # Plural form for Romanian language + # + # Corresponds to i18next rule #20 + # Rule: (n==1 ? 0 : (n==0 || (n%100 > 0 && n%100 < 20)) ? 1 : 2) + # + def self.special_romanian(count : Int) : UInt8 + return 0_u8 if count == 1 + return 1_u8 if count == 0 + + n_mod_100 = count % 100 + return 1_u8 if (n_mod_100 > 0 && n_mod_100 < 20) + return 2_u8 + end + + # Plural form for Slovenian language + # + # Corresponds to i18next rule #21 + # Rule: (n%100==1 ? 1 : n%100==2 ? 2 : n%100==3 || n%100==4 ? 3 : 0) + # + def self.special_slovenian(count : Int) : UInt8 + n_mod_100 = count % 100 + return 1_u8 if (n_mod_100 == 1) + return 2_u8 if (n_mod_100 == 2) + return 3_u8 if (n_mod_100 == 3 || n_mod_100 == 4) + return 0_u8 + end + + # Plural form for Hebrew language + # + # Corresponds to i18next rule #22 + # Rule: (n==1 ? 0 : n==2 ? 1 : (n<0 || n>10) && n%10==0 ? 2 : 3) + # + def self.special_hebrew(count : Int) : UInt8 + return 0_u8 if (count == 1) + return 1_u8 if (count == 2) + + if (count < 0 || count > 10) && (count % 10) == 0 + return 2_u8 + else + return 3_u8 + end + end + + # Plural form for Odia ("or") language + # + # This one is a bit special. It should use rule #2 (like english) + # but the "numbers" (suffixes?) it has are inverted, so we'll make a + # special rule for it. + # + def self.special_odia(count : Int) : UInt8 + return (count == 1) ? 0_u8 : 1_u8 + end + end +end diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr index 5d91a258..e2e50905 100644 --- a/src/invidious/helpers/logger.cr +++ b/src/invidious/helpers/logger.cr @@ -17,7 +17,19 @@ class Invidious::LogHandler < Kemal::BaseLogHandler elapsed_time = Time.measure { call_next(context) } elapsed_text = elapsed_text(elapsed_time) - info("#{context.response.status_code} #{context.request.method} #{context.request.resource} #{elapsed_text}") + # Default: full path with parameters + requested_url = context.request.resource + + # Try not to log search queries passed as GET parameters during normal use + # (They will still be logged if log level is 'Debug' or 'Trace') + if @level > LogLevel::Debug && ( + requested_url.downcase.includes?("search") || requested_url.downcase.includes?("q=") + ) + # Log only the path + requested_url = context.request.path + end + + info("#{context.response.status_code} #{context.request.method} #{requested_url} #{elapsed_text}") context end diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr new file mode 100644 index 00000000..bfbc237c --- /dev/null +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -0,0 +1,263 @@ +struct SearchVideo + include DB::Serializable + + property title : String + property id : String + property author : String + property ucid : String + property published : Time + property views : Int64 + property description_html : String + property length_seconds : Int32 + property live_now : Bool + property premium : Bool + property premiere_timestamp : Time? + + def to_xml(auto_generated, query_params, xml : XML::Builder) + query_params["v"] = self.id + + xml.element("entry") do + xml.element("id") { xml.text "yt:video:#{self.id}" } + xml.element("yt:videoId") { xml.text self.id } + xml.element("yt:channelId") { xml.text self.ucid } + xml.element("title") { xml.text self.title } + xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}") + + xml.element("author") do + if auto_generated + xml.element("name") { xml.text self.author } + xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } + else + xml.element("name") { xml.text author } + xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" } + end + end + + xml.element("content", type: "xhtml") do + xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do + xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do + xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg") + end + + xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text html_to_content(self.description_html) } + end + end + + xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") } + + xml.element("media:group") do + xml.element("media:title") { xml.text self.title } + xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg", + width: "320", height: "180") + xml.element("media:description") { xml.text html_to_content(self.description_html) } + end + + xml.element("media:community") do + xml.element("media:statistics", views: self.views) + end + end + end + + def to_xml(auto_generated, query_params, _xml : Nil) + XML.build do |xml| + to_xml(auto_generated, query_params, xml) + end + end + + def to_json(locale : String?, json : JSON::Builder) + json.object do + json.field "type", "video" + json.field "title", self.title + json.field "videoId", self.id + + json.field "author", self.author + json.field "authorId", self.ucid + json.field "authorUrl", "/channel/#{self.ucid}" + + json.field "videoThumbnails" do + generate_thumbnails(json, self.id) + end + + json.field "description", html_to_content(self.description_html) + json.field "descriptionHtml", self.description_html + + json.field "viewCount", self.views + json.field "published", self.published.to_unix + json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) + json.field "lengthSeconds", self.length_seconds + json.field "liveNow", self.live_now + json.field "premium", self.premium + json.field "isUpcoming", self.is_upcoming + + if self.premiere_timestamp + json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix + end + end + end + + # TODO: remove the locale and follow the crystal convention + def to_json(locale : String?, _json : Nil) + JSON.build do |json| + to_json(locale, json) + end + end + + def to_json(json : JSON::Builder) + to_json(nil, json) + end + + def is_upcoming + premiere_timestamp ? true : false + end +end + +struct SearchPlaylistVideo + include DB::Serializable + + property title : String + property id : String + property length_seconds : Int32 +end + +struct SearchPlaylist + include DB::Serializable + + property title : String + property id : String + property author : String + property ucid : String + property video_count : Int32 + property videos : Array(SearchPlaylistVideo) + property thumbnail : String? + + def to_json(locale : String?, json : JSON::Builder) + json.object do + json.field "type", "playlist" + json.field "title", self.title + json.field "playlistId", self.id + json.field "playlistThumbnail", self.thumbnail + + json.field "author", self.author + json.field "authorId", self.ucid + json.field "authorUrl", "/channel/#{self.ucid}" + + json.field "videoCount", self.video_count + json.field "videos" do + json.array do + self.videos.each do |video| + json.object do + json.field "title", video.title + json.field "videoId", video.id + json.field "lengthSeconds", video.length_seconds + + json.field "videoThumbnails" do + generate_thumbnails(json, video.id) + end + end + end + end + end + end + end + + # TODO: remove the locale and follow the crystal convention + def to_json(locale : String?, _json : Nil) + JSON.build do |json| + to_json(locale, json) + end + end + + def to_json(json : JSON::Builder) + to_json(nil, json) + end +end + +struct SearchChannel + include DB::Serializable + + property author : String + property ucid : String + property author_thumbnail : String + property subscriber_count : Int32 + property video_count : Int32 + property description_html : String + property auto_generated : Bool + + def to_json(locale : String?, json : JSON::Builder) + json.object do + json.field "type", "channel" + json.field "author", self.author + json.field "authorId", self.ucid + json.field "authorUrl", "/channel/#{self.ucid}" + + json.field "authorThumbnails" do + json.array do + qualities = {32, 48, 76, 100, 176, 512} + + qualities.each do |quality| + json.object do + json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}") + json.field "width", quality + json.field "height", quality + end + end + end + end + + json.field "autoGenerated", self.auto_generated + json.field "subCount", self.subscriber_count + json.field "videoCount", self.video_count + + json.field "description", html_to_content(self.description_html) + json.field "descriptionHtml", self.description_html + end + end + + # TODO: remove the locale and follow the crystal convention + def to_json(locale : String?, _json : Nil) + JSON.build do |json| + to_json(locale, json) + end + end + + def to_json(json : JSON::Builder) + to_json(nil, json) + end +end + +class Category + include DB::Serializable + + property title : String + property contents : Array(SearchItem) | Array(Video) + property url : String? + property description_html : String + property badges : Array(Tuple(String, String))? + + def to_json(locale : String?, json : JSON::Builder) + json.object do + json.field "type", "category" + json.field "title", self.title + json.field "contents" do + json.array do + self.contents.each do |item| + item.to_json(locale, json) + end + end + end + end + end + + # TODO: remove the locale and follow the crystal convention + def to_json(locale : String?, _json : Nil) + JSON.build do |json| + to_json(locale, json) + end + end + + def to_json(json : JSON::Builder) + to_json(nil, json) + end +end + +alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category diff --git a/src/invidious/helpers/signatures.cr b/src/invidious/helpers/signatures.cr index d8b1de65..ee09415b 100644 --- a/src/invidious/helpers/signatures.cr +++ b/src/invidious/helpers/signatures.cr @@ -30,7 +30,7 @@ struct DecryptFunction case op_body when "{a.reverse()" - operations[op_name] = ->(a : Array(String), b : Int32) { a.reverse } + operations[op_name] = ->(a : Array(String), _b : Int32) { a.reverse } when "{a.splice(0,b)" operations[op_name] = ->(a : Array(String), b : Int32) { a.delete_at(0..(b - 1)); a } else diff --git a/src/invidious/helpers/static_file_handler.cr b/src/invidious/helpers/static_file_handler.cr index be9d36ab..630c2fd2 100644 --- a/src/invidious/helpers/static_file_handler.cr +++ b/src/invidious/helpers/static_file_handler.cr @@ -173,7 +173,7 @@ module Kemal return end - if @cached_files.sum { |element| element[1][:data].bytesize } + (size = File.size(file_path)) < CACHE_LIMIT + if @cached_files.sum(&.[1][:data].bytesize) + (size = File.size(file_path)) < CACHE_LIMIT data = Bytes.new(size) File.open(file_path) do |file| file.read(data) diff --git a/src/invidious/helpers/tokens.cr b/src/invidious/helpers/tokens.cr index a09ce90b..8b076e39 100644 --- a/src/invidious/helpers/tokens.cr +++ b/src/invidious/helpers/tokens.cr @@ -1,8 +1,8 @@ require "crypto/subtle" -def generate_token(email, scopes, expire, key, db) +def generate_token(email, scopes, expire, key) session = "v1:#{Base64.urlsafe_encode(Random::Secure.random_bytes(32))}" - PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", session, email, Time.utc) + Invidious::Database::SessionIDs.insert(session, email) token = { "session" => session, @@ -19,7 +19,7 @@ def generate_token(email, scopes, expire, key, db) return token.to_json end -def generate_response(session, scopes, key, db, expire = 6.hours, use_nonce = false) +def generate_response(session, scopes, key, expire = 6.hours, use_nonce = false) expire = Time.utc + expire token = { @@ -30,7 +30,7 @@ def generate_response(session, scopes, key, db, expire = 6.hours, use_nonce = fa if use_nonce nonce = Random::Secure.hex(16) - db.exec("INSERT INTO nonces VALUES ($1, $2) ON CONFLICT DO NOTHING", nonce, expire) + Invidious::Database::Nonces.insert(nonce, expire) token["nonce"] = nonce end @@ -46,7 +46,7 @@ def sign_token(key, hash) next if key == "signature" if value.is_a?(JSON::Any) && value.as_a? - value = value.as_a.map { |i| i.as_s } + value = value.as_a.map(&.as_s) end case value @@ -63,7 +63,7 @@ def sign_token(key, hash) return Base64.urlsafe_encode(OpenSSL::HMAC.digest(:sha256, key, string_to_sign)).strip end -def validate_request(token, session, request, key, db, locale = nil) +def validate_request(token, session, request, key, locale = nil) case token when String token = JSON.parse(URI.decode_www_form(token)).as_h @@ -82,7 +82,7 @@ def validate_request(token, session, request, key, db, locale = nil) raise InfoException.new("Erroneous token") end - scopes = token["scopes"].as_a.map { |v| v.as_s } + scopes = token["scopes"].as_a.map(&.as_s) scope = "#{request.method}:#{request.path.lchop("/api/v1/auth/").lstrip("/")}" if !scopes_include_scope(scopes, scope) raise InfoException.new("Invalid scope") @@ -92,9 +92,9 @@ def validate_request(token, session, request, key, db, locale = nil) raise InfoException.new("Invalid signature") end - if token["nonce"]? && (nonce = db.query_one?("SELECT * FROM nonces WHERE nonce = $1", token["nonce"], as: {String, Time})) + if token["nonce"]? && (nonce = Invidious::Database::Nonces.select(token["nonce"].as_s)) if nonce[1] > Time.utc - db.exec("UPDATE nonces SET expire = $1 WHERE nonce = $2", Time.utc(1990, 1, 1), nonce[0]) + Invidious::Database::Nonces.update_set_expired(nonce[0]) else raise InfoException.new("Erroneous token") end @@ -105,11 +105,11 @@ end def scope_includes_scope(scope, subset) methods, endpoint = scope.split(":") - methods = methods.split(";").map { |method| method.upcase }.reject { |method| method.empty? }.sort + methods = methods.split(";").map(&.upcase).reject(&.empty?).sort! endpoint = endpoint.downcase subset_methods, subset_endpoint = subset.split(":") - subset_methods = subset_methods.split(";").map { |method| method.upcase }.sort + subset_methods = subset_methods.split(";").map(&.upcase).sort! subset_endpoint = subset_endpoint.downcase if methods.empty? diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 6ee07d7a..09181c10 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -1,70 +1,3 @@ -require "lsquic" -require "db" - -def add_yt_headers(request) - request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" - request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" - request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" - request.headers["accept-language"] ||= "en-us,en;q=0.5" - return if request.resource.starts_with? "/sorry/index" - request.headers["x-youtube-client-name"] ||= "1" - request.headers["x-youtube-client-version"] ||= "2.20200609" - # Preserve original cookies and add new YT consent cookie for EU servers - request.headers["cookie"] = "#{request.headers["cookie"]?}; CONSENT=YES+" - if !CONFIG.cookies.empty? - request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" - end -end - -struct YoutubeConnectionPool - property! url : URI - property! capacity : Int32 - property! timeout : Float64 - property pool : DB::Pool(QUIC::Client | HTTP::Client) - - def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true) - @url = url - @pool = build_pool(use_quic) - end - - def client(region = nil, &block) - if region - conn = make_client(url, region) - response = yield conn - else - conn = pool.checkout - begin - response = yield conn - rescue ex - conn.close - conn = QUIC::Client.new(url) - conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET - conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC - conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" - response = yield conn - ensure - pool.release(conn) - end - end - - response - end - - private def build_pool(use_quic) - DB::Pool(QUIC::Client | HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do - if use_quic - conn = QUIC::Client.new(url) - else - conn = HTTP::Client.new(url) - end - conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET - conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC - conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" - conn - end - end -end - # See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html def ci_lower_bound(pos, n) if n == 0 @@ -85,42 +18,18 @@ def elapsed_text(elapsed) "#{(millis * 1000).round(2)}µs" end -def make_client(url : URI, region = nil) - # TODO: Migrate any applicable endpoints to QUIC - client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure) - client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC - client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" - client.read_timeout = 10.seconds - client.connect_timeout = 10.seconds - - if region - PROXY_LIST[region]?.try &.sample(40).each do |proxy| - begin - proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) - client.set_proxy(proxy) - break - rescue ex - end - end - end - - return client -end - -def make_client(url : URI, region = nil, &block) - client = make_client(url, region) - begin - yield client - ensure - client.close - end -end - def decode_length_seconds(string) - length_seconds = string.gsub(/[^0-9:]/, "").split(":").map &.to_i + length_seconds = string.gsub(/[^0-9:]/, "") + return 0_i32 if length_seconds.empty? + + length_seconds = length_seconds.split(":").map { |x| x.to_i? || 0 } length_seconds = [0] * (3 - length_seconds.size) + length_seconds - length_seconds = Time::Span.new hours: length_seconds[0], minutes: length_seconds[1], seconds: length_seconds[2] - length_seconds = length_seconds.total_seconds.to_i + + length_seconds = Time::Span.new( + hours: length_seconds[0], + minutes: length_seconds[1], + seconds: length_seconds[2] + ).total_seconds.to_i32 return length_seconds end @@ -214,22 +123,20 @@ def recode_date(time : Time, locale) span = Time.utc - time if span.total_days > 365.0 - span = translate(locale, "`x` years", (span.total_days.to_i // 365).to_s) + return translate_count(locale, "generic_count_years", span.total_days.to_i // 365) elsif span.total_days > 30.0 - span = translate(locale, "`x` months", (span.total_days.to_i // 30).to_s) + return translate_count(locale, "generic_count_months", span.total_days.to_i // 30) elsif span.total_days > 7.0 - span = translate(locale, "`x` weeks", (span.total_days.to_i // 7).to_s) + return translate_count(locale, "generic_count_weeks", span.total_days.to_i // 7) elsif span.total_hours > 24.0 - span = translate(locale, "`x` days", (span.total_days.to_i).to_s) + return translate_count(locale, "generic_count_days", span.total_days.to_i) elsif span.total_minutes > 60.0 - span = translate(locale, "`x` hours", (span.total_hours.to_i).to_s) + return translate_count(locale, "generic_count_hours", span.total_hours.to_i) elsif span.total_seconds > 60.0 - span = translate(locale, "`x` minutes", (span.total_minutes.to_i).to_s) + return translate_count(locale, "generic_count_minutes", span.total_minutes.to_i) else - span = translate(locale, "`x` seconds", (span.total_seconds.to_i).to_s) + return translate_count(locale, "generic_count_seconds", span.total_seconds.to_i) end - - return span end def number_with_separator(number) @@ -397,22 +304,9 @@ def parse_range(range) return 0_i64, nil end -def convert_theme(theme) - case theme - when "true" - "dark" - when "false" - "light" - when "", nil - nil - else - theme - end -end - def fetch_random_instance begin - instance_api_client = HTTP::Client.new(URI.parse("https://api.invidious.io")) + instance_api_client = make_client(URI.parse("https://api.invidious.io")) # Timeouts instance_api_client.connect_timeout = 10.seconds diff --git a/src/invidious/jobs/pull_popular_videos_job.cr b/src/invidious/jobs/pull_popular_videos_job.cr index 7a8ab84e..dc785bae 100644 --- a/src/invidious/jobs/pull_popular_videos_job.cr +++ b/src/invidious/jobs/pull_popular_videos_job.cr @@ -1,11 +1,4 @@ class Invidious::Jobs::PullPopularVideosJob < Invidious::Jobs::BaseJob - QUERY = <<-SQL - SELECT DISTINCT ON (ucid) * - FROM channel_videos - WHERE ucid IN (SELECT channel FROM (SELECT UNNEST(subscriptions) AS channel FROM users) AS d - GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40) - ORDER BY ucid, published DESC - SQL POPULAR_VIDEOS = Atomic.new([] of ChannelVideo) private getter db : DB::Database @@ -14,9 +7,9 @@ class Invidious::Jobs::PullPopularVideosJob < Invidious::Jobs::BaseJob def begin loop do - videos = db.query_all(QUERY, as: ChannelVideo) - .sort_by(&.published) - .reverse + videos = Invidious::Database::ChannelVideos.select_popular_videos + .sort_by!(&.published) + .reverse! POPULAR_VIDEOS.set(videos) diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr index fbe6d381..941089c1 100644 --- a/src/invidious/jobs/refresh_channels_job.cr +++ b/src/invidious/jobs/refresh_channels_job.cr @@ -9,11 +9,11 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob lim_fibers = max_fibers active_fibers = 0 active_channel = Channel(Bool).new - backoff = 1.seconds + backoff = 2.minutes loop do LOGGER.debug("RefreshChannelsJob: Refreshing all channels") - db.query("SELECT id FROM channels ORDER BY updated") do |rs| + PG_DB.query("SELECT id FROM channels ORDER BY updated") do |rs| rs.each do id = rs.read(String) @@ -30,16 +30,16 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob spawn do begin LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel") - channel = fetch_channel(id, db, CONFIG.full_refresh) + channel = fetch_channel(id, CONFIG.full_refresh) lim_fibers = max_fibers LOGGER.trace("RefreshChannelsJob: #{id} fiber : Updating DB") - db.exec("UPDATE channels SET updated = $1, author = $2, deleted = false WHERE id = $3", Time.utc, channel.author, id) + Invidious::Database::Channels.update_author(id, channel.author) rescue ex LOGGER.error("RefreshChannelsJob: #{id} : #{ex.message}") if ex.message == "Deleted or invalid channel" - db.exec("UPDATE channels SET updated = $1, deleted = true WHERE id = $2", Time.utc, id) + Invidious::Database::Channels.update_mark_deleted(id) else lim_fibers = 1 LOGGER.error("RefreshChannelsJob: #{id} fiber : backing off for #{backoff}s") @@ -58,8 +58,9 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob end end - LOGGER.debug("RefreshChannelsJob: Done, sleeping for one minute") - sleep 1.minute + # TODO: make this configurable + LOGGER.debug("RefreshChannelsJob: Done, sleeping for thirty minutes") + sleep 30.minutes Fiber.yield end end diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr index 926c27fa..4b52c959 100644 --- a/src/invidious/jobs/refresh_feeds_job.cr +++ b/src/invidious/jobs/refresh_feeds_job.cr @@ -25,7 +25,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob spawn do begin # Drop outdated views - column_array = get_column_array(db, view_name) + column_array = Invidious::Database.get_column_array(db, view_name) ChannelVideo.type_array.each_with_index do |name, i| if name != column_array[i]? LOGGER.info("RefreshFeedsJob: DROP MATERIALIZED VIEW #{view_name}") diff --git a/src/invidious/jobs/statistics_refresh_job.cr b/src/invidious/jobs/statistics_refresh_job.cr index 6569c0a1..a113bd77 100644 --- a/src/invidious/jobs/statistics_refresh_job.cr +++ b/src/invidious/jobs/statistics_refresh_job.cr @@ -47,12 +47,14 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob private def refresh_stats users = STATISTICS.dig("usage", "users").as(Hash(String, Int64)) - users["total"] = db.query_one("SELECT count(*) FROM users", as: Int64) - users["activeHalfyear"] = db.query_one("SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '6 months'", as: Int64) - users["activeMonth"] = db.query_one("SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '1 month'", as: Int64) + + users["total"] = Invidious::Database::Statistics.count_users_total + users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_1m + users["activeMonth"] = Invidious::Database::Statistics.count_users_active_6m + STATISTICS["metadata"] = { "updatedAt" => Time.utc.to_unix, - "lastChannelRefreshedAt" => db.query_one?("SELECT updated FROM channels ORDER BY updated DESC LIMIT 1", as: Time).try &.to_unix || 0_i64, + "lastChannelRefreshedAt" => Invidious::Database::Statistics.channel_last_update.try &.to_unix || 0_i64, } end end diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr index 55b01174..3f342b92 100644 --- a/src/invidious/mixes.cr +++ b/src/invidious/mixes.cr @@ -72,7 +72,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil) videos += next_page.videos end - videos.uniq! { |video| video.id } + videos.uniq!(&.id) videos = videos.first(50) return Mix.new({ title: mix_title, @@ -97,7 +97,7 @@ def template_mix(mix) <li class="pure-menu-item"> <a href="/watch?v=#{video["videoId"]}&list=#{mix["mixId"]}"> <div class="thumbnail"> - <img class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg"> + <img loading="lazy" class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg"> <p class="length">#{recode_length_seconds(video["lengthSeconds"].as_i)}</p> </div> <p style="width:100%">#{video["title"]}</p> diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index f56cc2ea..a09e6cdb 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -11,7 +11,7 @@ struct PlaylistVideo property index : Int64 property live_now : Bool - def to_xml(auto_generated, xml : XML::Builder) + def to_xml(xml : XML::Builder) xml.element("entry") do xml.element("id") { xml.text "yt:video:#{self.id}" } xml.element("yt:videoId") { xml.text self.id } @@ -20,13 +20,8 @@ struct PlaylistVideo xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?v=#{self.id}") xml.element("author") do - if auto_generated - xml.element("name") { xml.text self.author } - xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } - else - xml.element("name") { xml.text author } - xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" } - end + xml.element("name") { xml.text self.author } + xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } end xml.element("content", type: "xhtml") do @@ -47,17 +42,11 @@ struct PlaylistVideo end end - def to_xml(auto_generated, xml : XML::Builder? = nil) - if xml - to_xml(auto_generated, xml) - else - XML.build do |json| - to_xml(auto_generated, xml) - end - end + def to_xml(_xml : Nil = nil) + XML.build { |xml| to_xml(xml) } end - def to_json(locale, json : JSON::Builder, index : Int32?) + def to_json(json : JSON::Builder, index : Int32? = nil) json.object do json.field "title", self.title json.field "videoId", self.id @@ -81,14 +70,8 @@ struct PlaylistVideo end end - def to_json(locale, json : JSON::Builder? = nil, index : Int32? = nil) - if json - to_json(locale, json, index: index) - else - JSON.build do |json| - to_json(locale, json, index: index) - end - end + def to_json(_json : Nil, index : Int32? = nil) + JSON.build { |json| to_json(json, index: index) } end end @@ -107,7 +90,7 @@ struct Playlist property updated : Time property thumbnail : String? - def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil) + def to_json(offset, locale, json : JSON::Builder, video_id : String? = nil) json.object do json.field "type", "playlist" json.field "title", self.title @@ -142,21 +125,21 @@ struct Playlist json.field "videos" do json.array do - videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation) - videos.each_with_index do |video, index| - video.to_json(locale, json) + videos = get_playlist_videos(self, offset: offset, locale: locale, video_id: video_id) + videos.each do |video| + video.to_json(json) end end end end end - def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil) + def to_json(offset, locale, json : JSON::Builder? = nil, video_id : String? = nil) if json - to_json(offset, locale, json, continuation: continuation) + to_json(offset, locale, json, video_id: video_id) else JSON.build do |json| - to_json(offset, locale, json, continuation: continuation) + to_json(offset, locale, json, video_id: video_id) end end end @@ -196,7 +179,7 @@ struct InvidiousPlaylist end end - def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil) + def to_json(offset, locale, json : JSON::Builder, video_id : String? = nil) json.object do json.field "type", "invidiousPlaylist" json.field "title", self.title @@ -217,32 +200,33 @@ struct InvidiousPlaylist json.field "videos" do json.array do - if !offset || offset == 0 - index = PG_DB.query_one?("SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1", self.id, continuation, as: Int64) + if (!offset || offset == 0) && !video_id.nil? + index = Invidious::Database::PlaylistVideos.select_index(self.id, video_id) offset = self.index.index(index) || 0 end - videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation) + videos = get_playlist_videos(self, offset: offset, locale: locale, video_id: video_id) videos.each_with_index do |video, index| - video.to_json(locale, json, offset + index) + video.to_json(json, offset + index) end end end end end - def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil) + def to_json(offset, locale, json : JSON::Builder? = nil, video_id : String? = nil) if json - to_json(offset, locale, json, continuation: continuation) + to_json(offset, locale, json, video_id: video_id) else JSON.build do |json| - to_json(offset, locale, json, continuation: continuation) + to_json(offset, locale, json, video_id: video_id) end end end def thumbnail - @thumbnail_id ||= PG_DB.query_one?("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 1", self.id, self.index, as: String) || "-----------" + # TODO: Get playlist thumbnail from playlist data rather than first video + @thumbnail_id ||= Invidious::Database::PlaylistVideos.select_one_id(self.id, self.index) || "-----------" "/vi/#{@thumbnail_id}/mqdefault.jpg" end @@ -259,11 +243,11 @@ struct InvidiousPlaylist end def description_html - HTML.escape(self.description).gsub("\n", "<br>") + HTML.escape(self.description) end end -def create_playlist(db, title, privacy, user) +def create_playlist(title, privacy, user) plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}" playlist = InvidiousPlaylist.new({ @@ -278,15 +262,12 @@ def create_playlist(db, title, privacy, user) index: [] of Int64, }) - playlist_array = playlist.to_a - args = arg_array(playlist_array) - - db.exec("INSERT INTO playlists VALUES (#{args})", args: playlist_array) + Invidious::Database::Playlists.insert(playlist) return playlist end -def subscribe_playlist(db, user, playlist) +def subscribe_playlist(user, playlist) playlist = InvidiousPlaylist.new({ title: playlist.title.byte_slice(0, 150), id: playlist.id, @@ -299,10 +280,7 @@ def subscribe_playlist(db, user, playlist) index: [] of Int64, }) - playlist_array = playlist.to_a - args = arg_array(playlist_array) - - db.exec("INSERT INTO playlists VALUES (#{args})", args: playlist_array) + Invidious::Database::Playlists.insert(playlist) return playlist end @@ -322,21 +300,19 @@ def produce_playlist_continuation(id, index) .try { |i| Protodec::Any.from_json(i) } .try { |i| Base64.urlsafe_encode(i, padding: false) } - data_wrapper = {"1:varint" => request_count, "15:string" => "PT:#{data}"} - .try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - object = { "80226972:embedded" => { - "2:string" => plid, - "3:string" => data_wrapper, + "2:string" => plid, + "3:base64" => { + "1:varint" => request_count, + "15:string" => "PT:#{data}", + "104:embedded" => {"1:0:varint" => 0_i64}, + }, "35:string" => id, }, } - continuation = object.try { |i| Protodec::Any.cast_json(object) } + continuation = object.try { |i| Protodec::Any.cast_json(i) } .try { |i| Protodec::Any.from_json(i) } .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } @@ -344,9 +320,9 @@ def produce_playlist_continuation(id, index) return continuation end -def get_playlist(db, plid, locale, refresh = true, force_refresh = false) +def get_playlist(plid, locale, refresh = true, force_refresh = false) if plid.starts_with? "IV" - if playlist = db.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if playlist = Invidious::Database::Playlists.select(id: plid) return playlist else raise InfoException.new("Playlist does not exist.") @@ -369,7 +345,7 @@ def fetch_playlist(plid, locale) playlist_info = playlist_sidebar_renderer[0]["playlistSidebarPrimaryInfoRenderer"]? raise InfoException.new("Could not extract playlist info") if !playlist_info - title = playlist_info["title"]?.try &.["runs"][0]?.try &.["text"]?.try &.as_s || "" + title = playlist_info.dig?("title", "runs", 0, "text").try &.as_s || "" desc_item = playlist_info["description"]? @@ -426,7 +402,7 @@ def fetch_playlist(plid, locale) }) end -def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil) +def get_playlist_videos(playlist, offset, locale = nil, video_id = nil) # Show empy playlist if requested page is out of range # (e.g, when a new playlist has been created, offset will be negative) if offset >= playlist.video_count || offset < 0 @@ -434,20 +410,28 @@ def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil) end if playlist.is_a? InvidiousPlaylist - db.query_all("SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3", - playlist.id, playlist.index, offset, as: PlaylistVideo) + Invidious::Database::PlaylistVideos.select(playlist.id, playlist.index, offset, limit: 100) else - if offset >= 100 - # Normalize offset to match youtube's behavior (100 videos chunck per request) - offset = (offset / 100).to_i64 * 100_i64 + if video_id + initial_data = YoutubeAPI.next({ + "videoId" => video_id, + "playlistId" => playlist.id, + }) + offset = initial_data.dig?("contents", "twoColumnWatchNextResults", "playlist", "playlist", "currentIndex").try &.as_i || offset + end + + videos = [] of PlaylistVideo + until videos.size >= 200 || videos.size == playlist.video_count || offset >= playlist.video_count + # 100 videos per request ctoken = produce_playlist_continuation(playlist.id, offset) initial_data = YoutubeAPI.browse(ctoken) - else - initial_data = YoutubeAPI.browse("VL" + playlist.id, params: "") + videos += extract_playlist_videos(initial_data) + + offset += 100 end - return extract_playlist_videos(initial_data) + return videos end end @@ -523,10 +507,10 @@ def template_playlist(playlist) playlist["videos"].as_a.each do |video| html += <<-END_HTML - <li class="pure-menu-item"> - <a href="/watch?v=#{video["videoId"]}&list=#{playlist["playlistId"]}"> + <li class="pure-menu-item" id="#{video["videoId"]}"> + <a href="/watch?v=#{video["videoId"]}&list=#{playlist["playlistId"]}&index=#{video["index"]}"> <div class="thumbnail"> - <img class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg"> + <img loading="lazy" class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg"> <p class="length">#{recode_length_seconds(video["lengthSeconds"].as_i)}</p> </div> <p style="width:100%">#{video["title"]}</p> diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index 93bee55c..b6183001 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -13,7 +13,7 @@ module Invidious::Routes::API::Manifest unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe } begin - video = get_video(id, PG_DB, region: region) + video = get_video(id, region: region) rescue ex : VideoRedirect return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex @@ -47,7 +47,7 @@ module Invidious::Routes::API::Manifest end audio_streams = video.audio_streams - video_streams = video.video_streams.sort_by { |stream| {stream["width"].as_i, stream["fps"].as_i} }.reverse + video_streams = video.video_streams.sort_by { |stream| {stream["width"].as_i, stream["fps"].as_i} }.reverse! manifest = XML.build(indent: " ", encoding: "UTF-8") do |xml| xml.element("MPD", "xmlns": "urn:mpeg:dash:schema:mpd:2011", @@ -160,7 +160,7 @@ module Invidious::Routes::API::Manifest manifest = response.body if local - manifest = manifest.gsub(/^https:\/\/r\d---.{11}\.c\.youtube\.com[^\n]*/m) do |match| + manifest = manifest.gsub(/^https:\/\/\w+---.{11}\.c\.youtube\.com[^\n]*/m) do |match| path = URI.parse(match).path path = path.lchop("/videoplayback/") diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index b4e9e9c8..fda655ef 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -22,12 +22,11 @@ module Invidious::Routes::API::V1::Authenticated user = env.get("user").as(User) begin - preferences = Preferences.from_json(env.request.body || "{}") + user.preferences = Preferences.from_json(env.request.body || "{}") rescue - preferences = user.preferences end - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) + Invidious::Database::Users.update_preferences(user) env.response.status_code = 204 end @@ -36,7 +35,7 @@ module Invidious::Routes::API::V1::Authenticated env.response.content_type = "application/json" user = env.get("user").as(User) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale max_results = env.params.query["max_results"]?.try &.to_i? max_results ||= user.preferences.max_results @@ -45,7 +44,7 @@ module Invidious::Routes::API::V1::Authenticated page = env.params.query["page"]?.try &.to_i? page ||= 1 - videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) + videos, notifications = get_subscription_feed(user, max_results, page) JSON.build do |json| json.object do @@ -72,13 +71,7 @@ module Invidious::Routes::API::V1::Authenticated env.response.content_type = "application/json" user = env.get("user").as(User) - if user.subscriptions.empty? - values = "'{}'" - else - values = "VALUES #{user.subscriptions.map { |id| %(('#{id}')) }.join(",")}" - end - - subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousChannel) + subscriptions = Invidious::Database::Channels.select(user.subscriptions) JSON.build do |json| json.array do @@ -99,8 +92,8 @@ module Invidious::Routes::API::V1::Authenticated ucid = env.params.url["ucid"] if !user.subscriptions.includes? ucid - get_channel(ucid, PG_DB, false, false) - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email) + get_channel(ucid, false, false) + Invidious::Database::Users.subscribe_channel(user, ucid) end # For Google accounts, access tokens don't have enough information to @@ -116,18 +109,18 @@ module Invidious::Routes::API::V1::Authenticated ucid = env.params.url["ucid"] - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_remove(subscriptions, $1) WHERE email = $2", ucid, user.email) + Invidious::Database::Users.unsubscribe_channel(user, ucid) env.response.status_code = 204 end def self.list_playlists(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" user = env.get("user").as(User) - playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1", user.email, as: InvidiousPlaylist) + playlists = Invidious::Database::Playlists.select_all(author: user.email) JSON.build do |json| json.array do @@ -141,7 +134,7 @@ module Invidious::Routes::API::V1::Authenticated def self.create_playlist(env) env.response.content_type = "application/json" user = env.get("user").as(User) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale title = env.params.json["title"]?.try &.as(String).delete("<>").byte_slice(0, 150) if !title @@ -153,11 +146,11 @@ module Invidious::Routes::API::V1::Authenticated return error_json(400, "Invalid privacy setting.") end - if PG_DB.query_one("SELECT count(*) FROM playlists WHERE author = $1", user.email, as: Int64) >= 100 + if Invidious::Database::Playlists.count_owned_by(user.email) >= 100 return error_json(400, "User cannot have more than 100 playlists.") end - playlist = create_playlist(PG_DB, title, privacy, user) + playlist = create_playlist(title, privacy, user) env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{playlist.id}" env.response.status_code = 201 { @@ -167,14 +160,17 @@ module Invidious::Routes::API::V1::Authenticated end def self.update_playlist_attribute(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" user = env.get("user").as(User) - plid = env.params.url["plid"] + plid = env.params.url["plid"]? + if !plid || plid.empty? + return error_json(400, "A playlist ID is required") + end - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email && playlist.privacy.private? return error_json(404, "Playlist does not exist.") end @@ -195,19 +191,20 @@ module Invidious::Routes::API::V1::Authenticated updated = playlist.updated end - PG_DB.exec("UPDATE playlists SET title = $1, privacy = $2, description = $3, updated = $4 WHERE id = $5", title, privacy, description, updated, plid) + Invidious::Database::Playlists.update(plid, title, privacy, description, updated) + env.response.status_code = 204 end def self.delete_playlist(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" user = env.get("user").as(User) plid = env.params.url["plid"] - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email && playlist.privacy.private? return error_json(404, "Playlist does not exist.") end @@ -216,21 +213,20 @@ module Invidious::Routes::API::V1::Authenticated return error_json(403, "Invalid user") end - PG_DB.exec("DELETE FROM playlist_videos * WHERE plid = $1", plid) - PG_DB.exec("DELETE FROM playlists * WHERE id = $1", plid) + Invidious::Database::Playlists.delete(plid) env.response.status_code = 204 end def self.insert_video_into_playlist(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" user = env.get("user").as(User) plid = env.params.url["plid"] - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email && playlist.privacy.private? return error_json(404, "Playlist does not exist.") end @@ -249,7 +245,7 @@ module Invidious::Routes::API::V1::Authenticated end begin - video = get_video(video_id, PG_DB) + video = get_video(video_id) rescue ex return error_json(500, ex) end @@ -266,19 +262,19 @@ module Invidious::Routes::API::V1::Authenticated index: Random::Secure.rand(0_i64..Int64::MAX), }) - video_array = playlist_video.to_a - args = arg_array(video_array) - - PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array) - PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, plid) + Invidious::Database::PlaylistVideos.insert(playlist_video) + Invidious::Database::Playlists.update_video_added(plid, playlist_video.index) env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{plid}/videos/#{playlist_video.index.to_u64.to_s(16).upcase}" env.response.status_code = 201 - playlist_video.to_json(locale, index: playlist.index.size) + + JSON.build do |json| + playlist_video.to_json(json, index: playlist.index.size) + end end def self.delete_video_in_playlist(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" user = env.get("user").as(User) @@ -286,7 +282,7 @@ module Invidious::Routes::API::V1::Authenticated plid = env.params.url["plid"] index = env.params.url["index"].to_i64(16) - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email && playlist.privacy.private? return error_json(404, "Playlist does not exist.") end @@ -299,8 +295,8 @@ module Invidious::Routes::API::V1::Authenticated return error_json(404, "Playlist does not contain index") end - PG_DB.exec("DELETE FROM playlist_videos * WHERE index = $1", index) - PG_DB.exec("UPDATE playlists SET index = array_remove(index, $1), video_count = cardinality(index) - 1, updated = $2 WHERE id = $3", index, Time.utc, plid) + Invidious::Database::PlaylistVideos.delete(index) + Invidious::Database::Playlists.update_video_removed(plid, index) env.response.status_code = 204 end @@ -315,7 +311,7 @@ module Invidious::Routes::API::V1::Authenticated user = env.get("user").as(User) scopes = env.get("scopes").as(Array(String)) - tokens = PG_DB.query_all("SELECT id, issued FROM session_ids WHERE email = $1", user.email, as: {session: String, issued: Time}) + tokens = Invidious::Database::SessionIDs.select_all(user.email) JSON.build do |json| json.array do @@ -331,15 +327,15 @@ module Invidious::Routes::API::V1::Authenticated def self.register_token(env) user = env.get("user").as(User) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale case env.request.headers["Content-Type"]? when "application/x-www-form-urlencoded" - scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v } + scopes = env.params.body.select { |k, _| k.match(/^scopes\[\d+\]$/) }.map { |_, v| v } callback_url = env.params.body["callbackUrl"]? expire = env.params.body["expire"]?.try &.to_i? when "application/json" - scopes = env.params.json["scopes"].as(Array).map { |v| v.as_s } + scopes = env.params.json["scopes"].as(Array).map(&.as_s) callback_url = env.params.json["callbackUrl"]?.try &.as(String) expire = env.params.json["expire"]?.try &.as(Int64) else @@ -357,7 +353,7 @@ module Invidious::Routes::API::V1::Authenticated if sid = env.get?("sid").try &.as(String) env.response.content_type = "text/html" - csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, PG_DB, use_nonce: true) + csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, use_nonce: true) return templated "authorize_token" else env.response.content_type = "application/json" @@ -371,7 +367,7 @@ module Invidious::Routes::API::V1::Authenticated end end - access_token = generate_token(user.email, authorized_scopes, expire, HMAC_KEY, PG_DB) + access_token = generate_token(user.email, authorized_scopes, expire, HMAC_KEY) if callback_url access_token = URI.encode_www_form(access_token) @@ -393,7 +389,7 @@ module Invidious::Routes::API::V1::Authenticated end def self.unregister_token(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" user = env.get("user").as(User) scopes = env.get("scopes").as(Array(String)) @@ -403,9 +399,9 @@ module Invidious::Routes::API::V1::Authenticated # Allow tokens to revoke other tokens with correct scope if session == env.get("session").as(String) - PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", session) + Invidious::Database::SessionIDs.delete(sid: session) elsif scopes_include_scope(scopes, "GET:tokens") - PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", session) + Invidious::Database::SessionIDs.delete(sid: session) else return error_json(400, "Cannot revoke session #{session}") end diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index da39661c..322ac42e 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -1,6 +1,6 @@ module Invidious::Routes::API::V1::Channels def self.home(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" @@ -96,7 +96,7 @@ module Invidious::Routes::API::V1::Channels json.field "relatedChannels" do json.array do - channel.related_channels.each do |related_channel| + fetch_related_channels(channel).each do |related_channel| json.object do json.field "author", related_channel.author json.field "authorId", related_channel.ucid @@ -124,7 +124,7 @@ module Invidious::Routes::API::V1::Channels end def self.latest(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" @@ -146,7 +146,7 @@ module Invidious::Routes::API::V1::Channels end def self.videos(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" @@ -182,7 +182,7 @@ module Invidious::Routes::API::V1::Channels end def self.playlists(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" @@ -219,7 +219,7 @@ module Invidious::Routes::API::V1::Channels end def self.community(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" @@ -242,7 +242,7 @@ module Invidious::Routes::API::V1::Channels end def self.search(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" diff --git a/src/invidious/routes/api/v1/feeds.cr b/src/invidious/routes/api/v1/feeds.cr index bb8f661b..41865f34 100644 --- a/src/invidious/routes/api/v1/feeds.cr +++ b/src/invidious/routes/api/v1/feeds.cr @@ -1,6 +1,6 @@ module Invidious::Routes::API::V1::Feeds def self.trending(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" @@ -25,7 +25,7 @@ module Invidious::Routes::API::V1::Feeds end def self.popular(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index cf95bd9b..ac0576a0 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -1,7 +1,7 @@ module Invidious::Routes::API::V1::Misc # Stats API endpoint for Invidious def self.stats(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" if !CONFIG.statistics_enabled @@ -15,7 +15,7 @@ module Invidious::Routes::API::V1::Misc # user playlists and Invidious playlists. This means that we can't # reasonably split them yet. This should be addressed in APIv2 def self.get_playlist(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" plid = env.params.url["plid"] @@ -24,7 +24,7 @@ module Invidious::Routes::API::V1::Misc offset ||= env.params.query["page"]?.try &.to_i?.try { |page| (page - 1) * 100 } offset ||= 0 - continuation = env.params.query["continuation"]? + video_id = env.params.query["continuation"]? format = env.params.query["format"]? format ||= "json" @@ -34,7 +34,7 @@ module Invidious::Routes::API::V1::Misc end begin - playlist = get_playlist(PG_DB, plid, locale) + playlist = get_playlist(plid, locale) rescue ex : InfoException return error_json(404, ex) rescue ex @@ -46,12 +46,32 @@ module Invidious::Routes::API::V1::Misc return error_json(404, "Playlist does not exist.") end - response = playlist.to_json(offset, locale, continuation: continuation) + # includes into the playlist a maximum of 20 videos, before the offset + if offset > 0 + lookback = offset < 50 ? offset : 50 + response = playlist.to_json(offset - lookback, locale) + json_response = JSON.parse(response) + else + # Unless the continuation is really the offset 0, it becomes expensive. + # It happens when the offset is not set. + # First we find the actual offset, and then we lookback + # it shouldn't happen often though + + lookback = 0 + response = playlist.to_json(offset, locale, video_id: video_id) + json_response = JSON.parse(response) + + if json_response["videos"].as_a[0]["index"] != offset + offset = json_response["videos"].as_a[0]["index"].as_i + lookback = offset < 50 ? offset : 50 + response = playlist.to_json(offset - lookback, locale) + json_response = JSON.parse(response) + end + end if format == "html" - response = JSON.parse(response) - playlist_html = template_playlist(response) - index, next_video = response["videos"].as_a.skip(1).select { |video| !video["author"].as_s.empty? }[0]?.try { |v| {v["index"], v["videoId"]} } || {nil, nil} + playlist_html = template_playlist(json_response) + index, next_video = json_response["videos"].as_a.skip(1 + lookback).select { |video| !video["author"].as_s.empty? }[0]?.try { |v| {v["index"], v["videoId"]} } || {nil, nil} response = { "playlistHtml" => playlist_html, @@ -64,7 +84,7 @@ module Invidious::Routes::API::V1::Misc end def self.mixes(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index f3a6fa06..a3b6c795 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -1,6 +1,6 @@ module Invidious::Routes::API::V1::Search def self.search(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale region = env.params.query["region"]? env.response.content_type = "application/json" @@ -20,7 +20,7 @@ module Invidious::Routes::API::V1::Search duration = env.params.query["duration"]?.try &.downcase duration ||= "" - features = env.params.query["features"]?.try &.split(",").map { |feature| feature.downcase } + features = env.params.query["features"]?.try &.split(",").map(&.downcase) features ||= [] of String content_type = env.params.query["type"]?.try &.downcase @@ -43,7 +43,7 @@ module Invidious::Routes::API::V1::Search end def self.search_suggestions(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale region = env.params.query["region"]? env.response.content_type = "application/json" diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 575e6fdf..3a013ba0 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -1,6 +1,6 @@ module Invidious::Routes::API::V1::Videos def self.videos(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" @@ -8,7 +8,7 @@ module Invidious::Routes::API::V1::Videos region = env.params.query["region"]? begin - video = get_video(id, PG_DB, region: region) + video = get_video(id, region: region) rescue ex : VideoRedirect env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) @@ -16,11 +16,11 @@ module Invidious::Routes::API::V1::Videos return error_json(500, ex) end - video.to_json(locale) + video.to_json(locale, nil) end def self.captions(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" @@ -36,7 +36,7 @@ module Invidious::Routes::API::V1::Videos # getting video info. begin - video = get_video(id, PG_DB, region: region) + video = get_video(id, region: region) rescue ex : VideoRedirect env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) @@ -58,7 +58,7 @@ module Invidious::Routes::API::V1::Videos captions.each do |caption| json.object do json.field "label", caption.name - json.field "languageCode", caption.languageCode + json.field "languageCode", caption.language_code json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}" end end @@ -73,7 +73,7 @@ module Invidious::Routes::API::V1::Videos env.response.content_type = "text/vtt; charset=UTF-8" if lang - caption = captions.select { |caption| caption.languageCode == lang } + caption = captions.select { |caption| caption.language_code == lang } else caption = captions.select { |caption| caption.name == label } end @@ -84,7 +84,7 @@ module Invidious::Routes::API::V1::Videos caption = caption[0] end - url = URI.parse("#{caption.baseUrl}&tlang=#{tlang}").request_target + url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target # Auto-generated captions often have cues that aren't aligned properly with the video, # as well as some other markup that makes it cumbersome, so we try to fix that here @@ -96,7 +96,7 @@ module Invidious::Routes::API::V1::Videos str << <<-END_VTT WEBVTT Kind: captions - Language: #{tlang || caption.languageCode} + Language: #{tlang || caption.language_code} END_VTT @@ -149,7 +149,7 @@ module Invidious::Routes::API::V1::Videos # thumbnails for individual scenes in a video. # See https://support.jwplayer.com/articles/how-to-add-preview-thumbnails def self.storyboards(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" @@ -157,7 +157,7 @@ module Invidious::Routes::API::V1::Videos region = env.params.query["region"]? begin - video = get_video(id, PG_DB, region: region) + video = get_video(id, region: region) rescue ex : VideoRedirect env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) @@ -223,7 +223,7 @@ module Invidious::Routes::API::V1::Videos end def self.annotations(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "text/xml" @@ -239,7 +239,7 @@ module Invidious::Routes::API::V1::Videos case source when "archive" - if CONFIG.cache_annotations && (cached_annotation = PG_DB.query_one?("SELECT * FROM annotations WHERE id = $1", id, as: Annotation)) + if CONFIG.cache_annotations && (cached_annotation = Invidious::Database::Annotations.select(id)) annotations = cached_annotation.annotations else index = CHARS_SAFE.index(id[0]).not_nil!.to_s.rjust(2, '0') @@ -271,7 +271,7 @@ module Invidious::Routes::API::V1::Videos annotations = response.body - cache_annotation(PG_DB, id, annotations) + cache_annotation(id, annotations) end else # "youtube" response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}") @@ -293,7 +293,7 @@ module Invidious::Routes::API::V1::Videos end def self.comments(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale region = env.params.query["region"]? env.response.content_type = "application/json" @@ -330,18 +330,13 @@ module Invidious::Routes::API::V1::Videos begin comments, reddit_thread = fetch_reddit_comments(id, sort_by: sort_by) - content_html = template_reddit_comments(comments, locale) - - content_html = fill_links(content_html, "https", "www.reddit.com") - content_html = replace_links(content_html) rescue ex comments = nil reddit_thread = nil - content_html = "" end if !reddit_thread || !comments - haltf env, 404 + return error_json(404, "No reddit threads found") end if format == "json" @@ -350,6 +345,9 @@ module Invidious::Routes::API::V1::Videos return reddit_thread.to_json else + content_html = template_reddit_comments(comments, locale) + content_html = fill_links(content_html, "https", "www.reddit.com") + content_html = replace_links(content_html) response = { "title" => reddit_thread.title, "permalink" => reddit_thread.permalink, diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 6a32988e..6cb1e1f7 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -1,3 +1,5 @@ +{% skip_file if flag?(:api_only) %} + module Invidious::Routes::Channels def self.home(env) self.videos(env) @@ -27,8 +29,8 @@ module Invidious::Routes::Channels item.author end end - items = items.select(&.is_a?(SearchPlaylist)).map(&.as(SearchPlaylist)) - items.each { |item| item.author = "" } + items = items.select(SearchPlaylist).map(&.as(SearchPlaylist)) + items.each(&.author = "") else sort_options = {"newest", "oldest", "popular"} sort_by ||= "newest" @@ -55,8 +57,8 @@ module Invidious::Routes::Channels end items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) - items = items.select { |item| item.is_a?(SearchPlaylist) }.map { |item| item.as(SearchPlaylist) } - items.each { |item| item.author = "" } + items = items.select(SearchPlaylist).map(&.as(SearchPlaylist)) + items.each(&.author = "") templated "playlists" end @@ -102,7 +104,7 @@ module Invidious::Routes::Channels # Redirects brand url channels to a normal /channel/:ucid route def self.brand_redirect(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale # /attribution_link endpoint needs both the `a` and `u` parameter # and in order to avoid detection from YouTube we should only send the required ones @@ -146,7 +148,7 @@ module Invidious::Routes::Channels end private def self.fetch_basic_information(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" if user diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 5fc8a61f..ab722ae2 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -1,12 +1,14 @@ +{% skip_file if flag?(:api_only) %} + module Invidious::Routes::Embed def self.redirect(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") begin - playlist = get_playlist(PG_DB, plid, locale: locale) + playlist = get_playlist(plid, locale: locale) offset = env.params.query["index"]?.try &.to_i? || 0 - videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale) + videos = get_playlist_videos(playlist, offset: offset, locale: locale) rescue ex return error_template(500, ex) end @@ -24,11 +26,11 @@ module Invidious::Routes::Embed end def self.show(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale id = env.params.url["id"] plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") - continuation = process_continuation(PG_DB, env.params.query, plid, id) + continuation = process_continuation(env.params.query, plid, id) if md = env.params.query["playlist"]? .try &.match(/[a-zA-Z0-9_-]{11}(,[a-zA-Z0-9_-]{11})*/) @@ -58,9 +60,9 @@ module Invidious::Routes::Embed if plid begin - playlist = get_playlist(PG_DB, plid, locale: locale) + playlist = get_playlist(plid, locale: locale) offset = env.params.query["index"]?.try &.to_i? || 0 - videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale) + videos = get_playlist_videos(playlist, offset: offset, locale: locale) rescue ex return error_template(500, ex) end @@ -117,7 +119,7 @@ module Invidious::Routes::Embed subscriptions ||= [] of String begin - video = get_video(id, PG_DB, region: params.region) + video = get_video(id, region: params.region) rescue ex : VideoRedirect return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex @@ -135,7 +137,7 @@ module Invidious::Routes::Embed # end if notifications && notifications.includes? id - PG_DB.exec("UPDATE users SET notifications = array_remove(notifications, $1) WHERE email = $2", id, user.as(User).email) + Invidious::Database::Users.remove_notification(user.as(User), id) env.get("user").as(User).notifications.delete(id) notifications.delete(id) end @@ -166,11 +168,11 @@ module Invidious::Routes::Embed preferred_captions = captions.select { |caption| params.preferred_captions.includes?(caption.name) || - params.preferred_captions.includes?(caption.languageCode.split("-")[0]) + params.preferred_captions.includes?(caption.language_code.split("-")[0]) } preferred_captions.sort_by! { |caption| (params.preferred_captions.index(caption.name) || - params.preferred_captions.index(caption.languageCode.split("-")[0])).not_nil! + params.preferred_captions.index(caption.language_code.split("-")[0])).not_nil! } captions = captions - preferred_captions diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index c88e96cf..fd8c25ce 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -1,10 +1,12 @@ +{% skip_file if flag?(:api_only) %} + module Invidious::Routes::Feeds def self.view_all_playlists_redirect(env) env.redirect "/feed/playlists" end def self.playlists(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" referer = get_referer(env) @@ -13,13 +15,14 @@ module Invidious::Routes::Feeds user = user.as(User) - items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + # TODO: make a single DB call and separate the items here? + items_created = Invidious::Database::Playlists.select_like_iv(user.email) items_created.map! do |item| item.author = "" item end - items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + items_saved = Invidious::Database::Playlists.select_not_like_iv(user.email) items_saved.map! do |item| item.author = "" item @@ -29,7 +32,7 @@ module Invidious::Routes::Feeds end def self.popular(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale if CONFIG.popular_enabled templated "feeds/popular" @@ -40,13 +43,13 @@ module Invidious::Routes::Feeds end def self.trending(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale trending_type = env.params.query["type"]? trending_type ||= "Default" region = env.params.query["region"]? - region ||= "US" + region ||= env.get("preferences").as(Preferences).region begin trending, plid = fetch_trending(trending_type, region, locale) @@ -58,7 +61,7 @@ module Invidious::Routes::Feeds end def self.subscriptions(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" sid = env.get? "sid" @@ -81,7 +84,7 @@ module Invidious::Routes::Feeds headers["Cookie"] = env.request.headers["Cookie"] if !user.password - user, sid = get_user(sid, headers, PG_DB) + user, sid = get_user(sid, headers) end max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE) @@ -91,14 +94,13 @@ module Invidious::Routes::Feeds page = env.params.query["page"]?.try &.to_i? page ||= 1 - videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) + videos, notifications = get_subscription_feed(user, max_results, page) # "updated" here is used for delivering new notifications, so if # we know a user has looked at their feed e.g. in the past 10 minutes, # they've already seen a video posted 20 minutes ago, and don't need # to be notified. - PG_DB.exec("UPDATE users SET notifications = $1, updated = $2 WHERE email = $3", [] of String, Time.utc, - user.email) + Invidious::Database::Users.clear_notifications(user) user.notifications = [] of String env.set "user", user @@ -106,7 +108,7 @@ module Invidious::Routes::Feeds end def self.history(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" referer = get_referer(env) @@ -135,7 +137,7 @@ module Invidious::Routes::Feeds # RSS feeds def self.rss_channel(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.headers["Content-Type"] = "application/atom+xml" env.response.content_type = "application/atom+xml" @@ -207,7 +209,7 @@ module Invidious::Routes::Feeds end def self.rss_private(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.headers["Content-Type"] = "application/atom+xml" env.response.content_type = "application/atom+xml" @@ -218,7 +220,7 @@ module Invidious::Routes::Feeds haltf env, status_code: 403 end - user = PG_DB.query_one?("SELECT * FROM users WHERE token = $1", token.strip, as: User) + user = Invidious::Database::Users.select(token: token.strip) if !user haltf env, status_code: 403 end @@ -232,7 +234,7 @@ module Invidious::Routes::Feeds params = HTTP::Params.parse(env.params.query["params"]? || "") - videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) + videos, notifications = get_subscription_feed(user, max_results, page) XML.build(indent: " ", encoding: "UTF-8") do |xml| xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", @@ -251,7 +253,7 @@ module Invidious::Routes::Feeds end def self.rss_playlist(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.headers["Content-Type"] = "application/atom+xml" env.response.content_type = "application/atom+xml" @@ -262,8 +264,8 @@ module Invidious::Routes::Feeds path = env.request.path if plid.starts_with? "IV" - if playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) - videos = get_playlist_videos(PG_DB, playlist, offset: 0, locale: locale) + if playlist = Invidious::Database::Playlists.select(id: plid) + videos = get_playlist_videos(playlist, offset: 0, locale: locale) return XML.build(indent: " ", encoding: "UTF-8") do |xml| xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", @@ -279,9 +281,7 @@ module Invidious::Routes::Feeds xml.element("name") { xml.text playlist.author } end - videos.each do |video| - video.to_xml(false, xml) - end + videos.each &.to_xml(xml) end end else @@ -364,7 +364,7 @@ module Invidious::Routes::Feeds if ucid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["channel_id"]? PG_DB.exec("UPDATE channels SET subscribed = $1 WHERE id = $2", Time.utc, ucid) elsif plid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["playlist_id"]? - PG_DB.exec("UPDATE playlists SET subscribed = $1 WHERE id = $2", Time.utc, ucid) + Invidious::Database::Playlists.update_subscription_time(plid) else haltf env, status_code: 400 end @@ -374,7 +374,7 @@ module Invidious::Routes::Feeds end def self.push_notifications_post(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale token = env.params.url["token"] body = env.request.body.not_nil!.gets_to_end @@ -393,7 +393,7 @@ module Invidious::Routes::Feeds published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) - video = get_video(id, PG_DB, force_refresh: true) + video = get_video(id, force_refresh: true) # Deliver notifications to `/api/v1/auth/notifications` payload = { @@ -416,13 +416,8 @@ module Invidious::Routes::Feeds views: video.views, }) - was_insert = PG_DB.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) - ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, - updated = $4, ucid = $5, author = $6, length_seconds = $7, - live_now = $8, premiere_timestamp = $9, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) - - PG_DB.exec("UPDATE users SET notifications = array_append(notifications, $1), - feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert + was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true) + Invidious::Database::Users.add_notification(video) if was_insert end end diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr new file mode 100644 index 00000000..594a7869 --- /dev/null +++ b/src/invidious/routes/images.cr @@ -0,0 +1,309 @@ +module Invidious::Routes::Images + # Avatars, banners and other large image assets. + def self.ggpht(env) + url = env.request.path.lchop("/ggpht") + + headers = ( + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + HTTP::Headers{":authority" => "yt3.ggpht.com"} + else + HTTP::Headers.new + end + {% else %} + HTTP::Headers.new + {% end %} + ) + + REQUEST_HEADERS_WHITELIST.each do |header| + if env.request.headers[header]? + headers[header] = env.request.headers[header] + end + end + + # We're encapsulating this into a proc in order to easily reuse this + # portion of the code for each request block below. + request_proc = ->(response : HTTP::Client::Response) { + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value + end + end + + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if response.status_code >= 300 + env.response.headers.delete("Transfer-Encoding") + return + end + + proxy_file(response, env) + } + + begin + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + YT_POOL.client &.get(url, headers) do |resp| + return request_proc.call(resp) + end + else + HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| + return request_proc.call(resp) + end + end + {% else %} + # This can likely be optimized into a (small) pool sometime in the future. + HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| + return request_proc.call(resp) + end + {% end %} + rescue ex + end + end + + def self.options_storyboard(env) + env.response.headers["Access-Control-Allow-Origin"] = "*" + env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" + env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range" + end + + def self.get_storyboard(env) + authority = env.params.url["authority"] + id = env.params.url["id"] + storyboard = env.params.url["storyboard"] + index = env.params.url["index"] + + url = "/sb/#{id}/#{storyboard}/#{index}?#{env.params.query}" + + headers = HTTP::Headers.new + + {% unless flag?(:disable_quic) %} + headers[":authority"] = "#{authority}.ytimg.com" + {% end %} + + REQUEST_HEADERS_WHITELIST.each do |header| + if env.request.headers[header]? + headers[header] = env.request.headers[header] + end + end + + request_proc = ->(response : HTTP::Client::Response) { + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value + end + end + + env.response.headers["Connection"] = "close" + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if response.status_code >= 300 + return env.response.headers.delete("Transfer-Encoding") + end + + proxy_file(response, env) + } + + begin + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + YT_POOL.client &.get(url, headers) do |resp| + return request_proc.call(resp) + end + else + HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| + return request_proc.call(resp) + end + end + {% else %} + # This can likely be optimized into a (small) pool sometime in the future. + HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| + return request_proc.call(resp) + end + {% end %} + rescue ex + end + end + + # ??? maybe also for storyboards? + def self.s_p_image(env) + id = env.params.url["id"] + name = env.params.url["name"] + url = env.request.resource + + headers = ( + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + HTTP::Headers{":authority" => "i9.ytimg.com"} + else + HTTP::Headers.new + end + {% else %} + HTTP::Headers.new + {% end %} + ) + + REQUEST_HEADERS_WHITELIST.each do |header| + if env.request.headers[header]? + headers[header] = env.request.headers[header] + end + end + + request_proc = ->(response : HTTP::Client::Response) { + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value + end + end + + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if response.status_code >= 300 && response.status_code != 404 + return env.response.headers.delete("Transfer-Encoding") + end + + proxy_file(response, env) + } + + begin + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + YT_POOL.client &.get(url, headers) do |resp| + return request_proc.call(resp) + end + else + HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| + return request_proc.call(resp) + end + end + {% else %} + # This can likely be optimized into a (small) pool sometime in the future. + HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| + return request_proc.call(resp) + end + {% end %} + rescue ex + end + end + + def self.yts_image(env) + headers = HTTP::Headers.new + REQUEST_HEADERS_WHITELIST.each do |header| + if env.request.headers[header]? + headers[header] = env.request.headers[header] + end + end + + begin + YT_POOL.client &.get(env.request.resource, headers) do |response| + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value + end + end + + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if response.status_code >= 300 && response.status_code != 404 + env.response.headers.delete("Transfer-Encoding") + break + end + + proxy_file(response, env) + end + rescue ex + end + end + + def self.thumbnails(env) + id = env.params.url["id"] + name = env.params.url["name"] + + headers = ( + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + HTTP::Headers{":authority" => "i.ytimg.com"} + else + HTTP::Headers.new + end + {% else %} + HTTP::Headers.new + {% end %} + ) + + if name == "maxres.jpg" + build_thumbnails(id).each do |thumb| + thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" + # Logic here is short enough that manually typing them out should be fine. + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + if YT_POOL.client &.head(thumbnail_resource_path, headers).status_code == 200 + name = thumb[:url] + ".jpg" + break + end + else + if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 + name = thumb[:url] + ".jpg" + break + end + end + {% else %} + # This can likely be optimized into a (small) pool sometime in the future. + if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 + name = thumb[:url] + ".jpg" + break + end + {% end %} + end + end + + url = "/vi/#{id}/#{name}" + + REQUEST_HEADERS_WHITELIST.each do |header| + if env.request.headers[header]? + headers[header] = env.request.headers[header] + end + end + + request_proc = ->(response : HTTP::Client::Response) { + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value + end + end + + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if response.status_code >= 300 && response.status_code != 404 + return env.response.headers.delete("Transfer-Encoding") + end + + proxy_file(response, env) + } + + begin + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + YT_POOL.client &.get(url, headers) do |resp| + return request_proc.call(resp) + end + else + HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| + return request_proc.call(resp) + end + end + {% else %} + # This can likely be optimized into a (small) pool sometime in the future. + HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| + return request_proc.call(resp) + end + {% end %} + rescue ex + end + end +end diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index f052d3f4..64da3e4e 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -1,6 +1,8 @@ +{% skip_file if flag?(:api_only) %} + module Invidious::Routes::Login def self.login_page(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" @@ -29,7 +31,7 @@ module Invidious::Routes::Login end def self.login(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale referer = get_referer(env, "/feed/subscriptions") @@ -51,7 +53,13 @@ module Invidious::Routes::Login # See https://github.com/ytdl-org/youtube-dl/blob/2019.04.07/youtube_dl/extractor/youtube.py#L82 begin - client = QUIC::Client.new(LOGIN_URL) + client = nil # Declare variable + {% unless flag?(:disable_quic) %} + client = CONFIG.use_quic ? QUIC::Client.new(LOGIN_URL) : HTTP::Client.new(LOGIN_URL) + {% else %} + client = HTTP::Client.new(LOGIN_URL) + {% end %} + headers = HTTP::Headers.new login_page = client.get("/ServiceLogin") @@ -267,7 +275,7 @@ module Invidious::Routes::Login raise "Couldn't get SID." end - user, sid = get_user(sid, headers, PG_DB) + user, sid = get_user(sid, headers) # We are now logged in traceback << "done.<br/>" @@ -295,8 +303,8 @@ module Invidious::Routes::Login end if env.request.cookies["PREFS"]? - preferences = env.get("preferences").as(Preferences) - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) + user.preferences = env.get("preferences").as(Preferences) + Invidious::Database::Users.update_preferences(user) cookie = env.request.cookies["PREFS"] cookie.expires = Time.utc(1990, 1, 1) @@ -319,7 +327,7 @@ module Invidious::Routes::Login return error_template(401, "Password is a required field") end - user = PG_DB.query_one?("SELECT * FROM users WHERE email = $1", email, as: User) + user = Invidious::Database::Users.select(email: email) if user if !user.password @@ -328,7 +336,7 @@ module Invidious::Routes::Login if Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55)) sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) + Invidious::Database::SessionIDs.insert(sid, email) if Kemal.config.ssl || CONFIG.https_only secure = true @@ -385,15 +393,15 @@ module Invidious::Routes::Login prompt = "" if captcha_type == "image" - captcha = generate_captcha(HMAC_KEY, PG_DB) + captcha = generate_captcha(HMAC_KEY) else - captcha = generate_text_captcha(HMAC_KEY, PG_DB) + captcha = generate_text_captcha(HMAC_KEY) end return templated "login" end - tokens = env.params.body.select { |k, v| k.match(/^token\[\d+\]$/) }.map { |k, v| v } + tokens = env.params.body.select { |k, _| k.match(/^token\[\d+\]$/) }.map { |_, v| v } answer ||= "" captcha_type ||= "image" @@ -404,7 +412,7 @@ module Invidious::Routes::Login answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer) begin - validate_request(tokens[0], answer, env.request, HMAC_KEY, PG_DB, locale) + validate_request(tokens[0], answer, env.request, HMAC_KEY, locale) rescue ex return error_template(400, ex) end @@ -417,9 +425,9 @@ module Invidious::Routes::Login found_valid_captcha = false error_exception = Exception.new - tokens.each_with_index do |token, i| + tokens.each do |token| begin - validate_request(token, answer, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, answer, env.request, HMAC_KEY, locale) found_valid_captcha = true rescue ex error_exception = ex @@ -441,13 +449,8 @@ module Invidious::Routes::Login end end - user_array = user.to_a - user_array[4] = user_array[4].to_json # User preferences - - args = arg_array(user_array) - - PG_DB.exec("INSERT INTO users VALUES (#{args})", args: user_array) - PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) + Invidious::Database::Users.insert(user) + Invidious::Database::SessionIDs.insert(sid, email) view_name = "subscriptions_#{sha256(user.email)}" PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") @@ -467,8 +470,8 @@ module Invidious::Routes::Login end if env.request.cookies["PREFS"]? - preferences = env.get("preferences").as(Preferences) - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) + user.preferences = env.get("preferences").as(Preferences) + Invidious::Database::Users.update_preferences(user) cookie = env.request.cookies["PREFS"] cookie.expires = Time.utc(1990, 1, 1) @@ -483,7 +486,7 @@ module Invidious::Routes::Login end def self.signout(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" sid = env.get? "sid" @@ -498,12 +501,12 @@ module Invidious::Routes::Login token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex return error_template(400, ex) end - PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", sid) + Invidious::Database::SessionIDs.delete(sid: sid) env.request.cookies.each do |cookie| cookie.expires = Time.utc(1990, 1, 1) diff --git a/src/invidious/routes/misc.cr b/src/invidious/routes/misc.cr index 82c40a95..d6bd9571 100644 --- a/src/invidious/routes/misc.cr +++ b/src/invidious/routes/misc.cr @@ -1,7 +1,9 @@ +{% skip_file if flag?(:api_only) %} + module Invidious::Routes::Misc def self.home(env) preferences = env.get("preferences").as(Preferences) - locale = LOCALES[preferences.locale]? + locale = preferences.locale user = env.get? "user" case preferences.default_home @@ -27,22 +29,17 @@ module Invidious::Routes::Misc end def self.privacy(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale templated "privacy" end def self.licenses(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale rendered "licenses" end def self.cross_instance_redirect(env) referer = get_referer(env) - - if !env.get("preferences").as(Preferences).automatic_instance_redirect - return env.redirect("https://redirect.invidious.io#{referer}") - end - instance_url = fetch_random_instance env.redirect "https://#{instance_url}#{referer}" end diff --git a/src/invidious/routes/notifications.cr b/src/invidious/routes/notifications.cr new file mode 100644 index 00000000..272a3dc7 --- /dev/null +++ b/src/invidious/routes/notifications.cr @@ -0,0 +1,78 @@ +module Invidious::Routes::Notifications + # /modify_notifications + # will "ding" all subscriptions. + # /modify_notifications?receive_all_updates=false&receive_no_updates=false + # will "unding" all subscriptions. + def self.modify(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env, "/") + + redirect = env.params.query["redirect"]? + redirect ||= "false" + redirect = redirect == "true" + + if !user + if redirect + return env.redirect referer + else + return error_json(403, "No such user") + end + end + + user = user.as(User) + + if !user.password + channel_req = {} of String => String + + channel_req["receive_all_updates"] = env.params.query["receive_all_updates"]? || "true" + channel_req["receive_no_updates"] = env.params.query["receive_no_updates"]? || "" + channel_req["receive_post_updates"] = env.params.query["receive_post_updates"]? || "true" + + channel_req.reject! { |k, v| v != "true" && v != "false" } + + headers = HTTP::Headers.new + headers["Cookie"] = env.request.headers["Cookie"] + + html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers) + + cookies = HTTP::Cookies.from_client_headers(headers) + html.cookies.each do |cookie| + if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name + if cookies[cookie.name]? + cookies[cookie.name] = cookie + else + cookies << cookie + end + end + end + headers = cookies.add_request_headers(headers) + + if match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[^"]+)"/) + session_token = match["session_token"] + else + return env.redirect referer + end + + headers["content-type"] = "application/x-www-form-urlencoded" + channel_req["session_token"] = session_token + + subs = XML.parse_html(html.body) + subs.xpath_nodes(%q(//a[@class="subscription-title yt-uix-sessionlink"]/@href)).each do |channel| + channel_id = channel.content.lstrip("/channel/").not_nil! + channel_req["channel_id"] = channel_id + + YT_POOL.client &.post("/subscription_ajax?action_update_subscription_preferences=1", headers, form: channel_req) + end + end + + if redirect + env.redirect referer + else + env.response.content_type = "application/json" + "{}" + end + end +end diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index 05a198d8..d437b79c 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -1,6 +1,8 @@ +{% skip_file if flag?(:api_only) %} + module Invidious::Routes::Playlists def self.new(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" sid = env.get? "sid" @@ -10,13 +12,13 @@ module Invidious::Routes::Playlists user = user.as(User) sid = sid.as(String) - csrf_token = generate_response(sid, {":create_playlist"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":create_playlist"}, HMAC_KEY) templated "create_playlist" end def self.create(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" sid = env.get? "sid" @@ -29,7 +31,7 @@ module Invidious::Routes::Playlists token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex return error_template(400, ex) end @@ -44,17 +46,17 @@ module Invidious::Routes::Playlists return error_template(400, "Invalid privacy setting.") end - if PG_DB.query_one("SELECT count(*) FROM playlists WHERE author = $1", user.email, as: Int64) >= 100 + if Invidious::Database::Playlists.count_owned_by(user.email) >= 100 return error_template(400, "User cannot have more than 100 playlists.") end - playlist = create_playlist(PG_DB, title, privacy, user) + playlist = create_playlist(title, privacy, user) env.redirect "/playlist?list=#{playlist.id}" end def self.subscribe(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" referer = get_referer(env) @@ -64,14 +66,14 @@ module Invidious::Routes::Playlists user = user.as(User) playlist_id = env.params.query["list"] - playlist = get_playlist(PG_DB, playlist_id, locale) - subscribe_playlist(PG_DB, user, playlist) + playlist = get_playlist(playlist_id, locale) + subscribe_playlist(user, playlist) env.redirect "/playlist?list=#{playlist.id}" end def self.delete_page(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" sid = env.get? "sid" @@ -83,18 +85,22 @@ module Invidious::Routes::Playlists sid = sid.as(String) plid = env.params.query["list"]? - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if !plid || plid.empty? + return error_template(400, "A playlist ID is required") + end + + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email return env.redirect referer end - csrf_token = generate_response(sid, {":delete_playlist"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":delete_playlist"}, HMAC_KEY) templated "delete_playlist" end def self.delete(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" sid = env.get? "sid" @@ -110,24 +116,23 @@ module Invidious::Routes::Playlists token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex return error_template(400, ex) end - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email return env.redirect referer end - PG_DB.exec("DELETE FROM playlist_videos * WHERE plid = $1", plid) - PG_DB.exec("DELETE FROM playlists * WHERE id = $1", plid) + Invidious::Database::Playlists.delete(plid) env.redirect "/feed/playlists" end def self.edit(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" sid = env.get? "sid" @@ -147,7 +152,7 @@ module Invidious::Routes::Playlists page ||= 1 begin - playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid, raise_on_fail: true) if !playlist || playlist.author != user.email return env.redirect referer end @@ -156,18 +161,18 @@ module Invidious::Routes::Playlists end begin - videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale) + videos = get_playlist_videos(playlist, offset: (page - 1) * 100, locale: locale) rescue ex videos = [] of PlaylistVideo end - csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY) templated "edit_playlist" end def self.update(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" sid = env.get? "sid" @@ -183,12 +188,12 @@ module Invidious::Routes::Playlists token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex return error_template(400, ex) end - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email return env.redirect referer end @@ -205,13 +210,13 @@ module Invidious::Routes::Playlists updated = playlist.updated end - PG_DB.exec("UPDATE playlists SET title = $1, privacy = $2, description = $3, updated = $4 WHERE id = $5", title, privacy, description, updated, plid) + Invidious::Database::Playlists.update(plid, title, privacy, description, updated) env.redirect "/playlist?list=#{plid}" end def self.add_playlist_items_page(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" sid = env.get? "sid" @@ -231,7 +236,7 @@ module Invidious::Routes::Playlists page ||= 1 begin - playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid, raise_on_fail: true) if !playlist || playlist.author != user.email return env.redirect referer end @@ -243,7 +248,7 @@ module Invidious::Routes::Playlists if query begin search_query, count, items, operators = process_search_query(query, page, user, region: nil) - videos = items.select { |item| item.is_a? SearchVideo }.map { |item| item.as(SearchVideo) } + videos = items.select(SearchVideo).map(&.as(SearchVideo)) rescue ex videos = [] of SearchVideo count = 0 @@ -258,7 +263,7 @@ module Invidious::Routes::Playlists end def self.playlist_ajax(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get? "user" sid = env.get? "sid" @@ -281,7 +286,7 @@ module Invidious::Routes::Playlists token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex if redirect return error_template(400, ex) @@ -309,7 +314,7 @@ module Invidious::Routes::Playlists begin playlist_id = env.params.query["playlist_id"] - playlist = get_playlist(PG_DB, playlist_id, locale).as(InvidiousPlaylist) + playlist = get_playlist(playlist_id, locale).as(InvidiousPlaylist) raise "Invalid user" if playlist.author != user.email rescue ex if redirect @@ -340,7 +345,7 @@ module Invidious::Routes::Playlists video_id = env.params.query["video_id"] begin - video = get_video(video_id, PG_DB) + video = get_video(video_id) rescue ex if redirect return error_template(500, ex) @@ -361,15 +366,12 @@ module Invidious::Routes::Playlists index: Random::Secure.rand(0_i64..Int64::MAX), }) - video_array = playlist_video.to_a - args = arg_array(video_array) - - PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array) - PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, playlist_id) + Invidious::Database::PlaylistVideos.insert(playlist_video) + Invidious::Database::Playlists.update_video_added(playlist_id, playlist_video.index) when "action_remove_video" index = env.params.query["set_video_id"] - PG_DB.exec("DELETE FROM playlist_videos * WHERE index = $1", index) - PG_DB.exec("UPDATE playlists SET index = array_remove(index, $1), video_count = cardinality(index) - 1, updated = $2 WHERE id = $3", index, Time.utc, playlist_id) + Invidious::Database::PlaylistVideos.delete(index) + Invidious::Database::Playlists.update_video_removed(playlist_id, index) when "action_move_video_before" # TODO: Playlist stub else @@ -385,7 +387,7 @@ module Invidious::Routes::Playlists end def self.show(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale user = env.get?("user").try &.as(User) referer = get_referer(env) @@ -403,7 +405,7 @@ module Invidious::Routes::Playlists end begin - playlist = get_playlist(PG_DB, plid, locale) + playlist = get_playlist(plid, locale) rescue ex return error_template(500, ex) end @@ -420,7 +422,7 @@ module Invidious::Routes::Playlists end begin - videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale) + videos = get_playlist_videos(playlist, offset: (page - 1) * 100, locale: locale) rescue ex return error_template(500, "Error encountered while retrieving playlist videos.<br>#{ex.message}") end @@ -433,7 +435,7 @@ module Invidious::Routes::Playlists end def self.mix(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale rdid = env.params.query["list"]? if !rdid diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index 0f26ec15..faae03bc 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -1,6 +1,8 @@ +{% skip_file if flag?(:api_only) %} + module Invidious::Routes::PreferencesRoute def self.show(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale referer = get_referer(env) @@ -10,7 +12,7 @@ module Invidious::Routes::PreferencesRoute end def self.update(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale referer = get_referer(env) video_loop = env.params.body["video_loop"]?.try &.as(String) @@ -68,6 +70,10 @@ module Invidious::Routes::PreferencesRoute vr_mode ||= "off" vr_mode = vr_mode == "on" + save_player_pos = env.params.body["save_player_pos"]?.try &.as(String) + save_player_pos ||= "off" + save_player_pos = save_player_pos == "on" + show_nick = env.params.body["show_nick"]?.try &.as(String) show_nick ||= "off" show_nick = show_nick == "on" @@ -100,6 +106,8 @@ module Invidious::Routes::PreferencesRoute automatic_instance_redirect ||= "off" automatic_instance_redirect = automatic_instance_redirect == "on" + region = env.params.body["region"]?.try &.as(String) + locale = env.params.body["locale"]?.try &.as(String) locale ||= CONFIG.default_user_preferences.locale @@ -150,6 +158,7 @@ module Invidious::Routes::PreferencesRoute default_home: default_home, feed_menu: feed_menu, automatic_instance_redirect: automatic_instance_redirect, + region: region, related_videos: related_videos, sort: sort, speed: speed, @@ -160,11 +169,13 @@ module Invidious::Routes::PreferencesRoute extend_desc: extend_desc, vr_mode: vr_mode, show_nick: show_nick, - }.to_json).to_json + save_player_pos: save_player_pos, + }.to_json) if user = env.get? "user" user = user.as(User) - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences, user.email) + user.preferences = preferences + Invidious::Database::Users.update_preferences(user) if CONFIG.admins.includes? user.email CONFIG.default_user_preferences.default_home = env.params.body["admin_default_home"]?.try &.as(String) || CONFIG.default_user_preferences.default_home @@ -198,6 +209,8 @@ module Invidious::Routes::PreferencesRoute statistics_enabled ||= "off" CONFIG.statistics_enabled = statistics_enabled == "on" + CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.try &.as(String) + File.write("config/config.yml", CONFIG.to_yaml) end else @@ -208,10 +221,10 @@ module Invidious::Routes::PreferencesRoute end if CONFIG.domain - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: URI.encode_www_form(preferences.to_json), expires: Time.utc + 2.years, secure: secure, http_only: true) else - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: URI.encode_www_form(preferences.to_json), expires: Time.utc + 2.years, secure: secure, http_only: true) end end @@ -220,7 +233,7 @@ module Invidious::Routes::PreferencesRoute end def self.toggle_theme(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale referer = get_referer(env, unroll: false) redirect = env.params.query["redirect"]? @@ -229,18 +242,15 @@ module Invidious::Routes::PreferencesRoute if user = env.get? "user" user = user.as(User) - preferences = user.preferences - case preferences.dark_mode + case user.preferences.dark_mode when "dark" - preferences.dark_mode = "light" + user.preferences.dark_mode = "light" else - preferences.dark_mode = "dark" + user.preferences.dark_mode = "dark" end - preferences = preferences.to_json - - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences, user.email) + Invidious::Database::Users.update_preferences(user) else preferences = env.get("preferences").as(Preferences) @@ -275,4 +285,191 @@ module Invidious::Routes::PreferencesRoute "{}" end end + + def self.data_control(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + + templated "data_control" + end + + def self.update_data_control(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + referer = get_referer(env) + + if user + user = user.as(User) + + # TODO: Find a way to prevent browser timeout + + HTTP::FormData.parse(env.request) do |part| + body = part.body.gets_to_end + type = part.headers["Content-Type"] + + next if body.empty? + + # TODO: Unify into single import based on content-type + case part.name + when "import_invidious" + body = JSON.parse(body) + + if body["subscriptions"]? + user.subscriptions += body["subscriptions"].as_a.map(&.as_s) + user.subscriptions.uniq! + + user.subscriptions = get_batch_channels(user.subscriptions, false, false) + + Invidious::Database::Users.update_subscriptions(user) + end + + if body["watch_history"]? + user.watched += body["watch_history"].as_a.map(&.as_s) + user.watched.uniq! + Invidious::Database::Users.update_watch_history(user) + end + + if body["preferences"]? + user.preferences = Preferences.from_json(body["preferences"].to_json) + Invidious::Database::Users.update_preferences(user) + end + + if playlists = body["playlists"]?.try &.as_a? + playlists.each do |item| + title = item["title"]?.try &.as_s?.try &.delete("<>") + description = item["description"]?.try &.as_s?.try &.delete("\r") + privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy } + + next if !title + next if !description + next if !privacy + + playlist = create_playlist(title, privacy, user) + Invidious::Database::Playlists.update_description(playlist.id, description) + + videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| + raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500 + + video_id = video_id.try &.as_s? + next if !video_id + + begin + video = get_video(video_id) + rescue ex + next + end + + playlist_video = PlaylistVideo.new({ + title: video.title, + id: video.id, + author: video.author, + ucid: video.ucid, + length_seconds: video.length_seconds, + published: video.published, + plid: playlist.id, + live_now: video.live_now, + index: Random::Secure.rand(0_i64..Int64::MAX), + }) + + Invidious::Database::PlaylistVideos.insert(playlist_video) + Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index) + end + end + end + when "import_youtube" + filename = part.filename || "" + extension = filename.split(".").last + + if extension == "xml" || type == "application/xml" || type == "text/xml" + subscriptions = XML.parse(body) + user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| + channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] + end + elsif extension == "json" || type == "application/json" + subscriptions = JSON.parse(body) + user.subscriptions += subscriptions.as_a.compact_map do |entry| + entry["snippet"]["resourceId"]["channelId"].as_s + end + elsif extension == "csv" || type == "text/csv" + subscriptions = parse_subscription_export_csv(body) + user.subscriptions += subscriptions + else + haltf(env, status_code: 415, + response: error_template(415, "Invalid subscription file uploaded") + ) + end + + user.subscriptions.uniq! + user.subscriptions = get_batch_channels(user.subscriptions, false, false) + + Invidious::Database::Users.update_subscriptions(user) + when "import_freetube" + user.subscriptions += body.scan(/"channelId":"(?<channel_id>[a-zA-Z0-9_-]{24})"/).map do |md| + md["channel_id"] + end + user.subscriptions.uniq! + + user.subscriptions = get_batch_channels(user.subscriptions, false, false) + + Invidious::Database::Users.update_subscriptions(user) + when "import_newpipe_subscriptions" + body = JSON.parse(body) + user.subscriptions += body["subscriptions"].as_a.compact_map do |channel| + if match = channel["url"].as_s.match(/\/channel\/(?<channel>UC[a-zA-Z0-9_-]{22})/) + next match["channel"] + elsif match = channel["url"].as_s.match(/\/user\/(?<user>.+)/) + response = YT_POOL.client &.get("/user/#{match["user"]}?disable_polymer=1&hl=en&gl=US") + html = XML.parse_html(response.body) + ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1] + next ucid if ucid + end + + nil + end + user.subscriptions.uniq! + + user.subscriptions = get_batch_channels(user.subscriptions, false, false) + + Invidious::Database::Users.update_subscriptions(user) + when "import_newpipe" + Compress::Zip::Reader.open(IO::Memory.new(body)) do |file| + file.each_entry do |entry| + if entry.filename == "newpipe.db" + tempfile = File.tempfile(".db") + File.write(tempfile.path, entry.io.gets_to_end) + db = DB.open("sqlite3://" + tempfile.path) + + user.watched += db.query_all("SELECT url FROM streams", as: String).map(&.lchop("https://www.youtube.com/watch?v=")) + user.watched.uniq! + + Invidious::Database::Users.update_watch_history(user) + + user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map(&.lchop("https://www.youtube.com/channel/")) + user.subscriptions.uniq! + + user.subscriptions = get_batch_channels(user.subscriptions, false, false) + + Invidious::Database::Users.update_subscriptions(user) + + db.close + tempfile.delete + end + end + end + else nil # Ignore + end + end + end + + env.redirect referer + end end diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index 610d5031..5e606adf 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -1,6 +1,8 @@ +{% skip_file if flag?(:api_only) %} + module Invidious::Routes::Search def self.opensearch(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/opensearchdescription+xml" XML.build(indent: " ", encoding: "UTF-8") do |xml| @@ -16,7 +18,7 @@ module Invidious::Routes::Search end def self.results(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale query = env.params.query["search_query"]? query ||= env.params.query["q"]? @@ -35,7 +37,7 @@ module Invidious::Routes::Search end def self.search(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale region = env.params.query["region"]? query = env.params.query["search_query"]? @@ -53,6 +55,8 @@ module Invidious::Routes::Search begin search_query, count, videos, operators = process_search_query(query, page, user, region: region) + rescue ex : ChannelSearchException + return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.") rescue ex return error_template(500, ex) end diff --git a/src/invidious/routes/subscriptions.cr b/src/invidious/routes/subscriptions.cr new file mode 100644 index 00000000..29152afb --- /dev/null +++ b/src/invidious/routes/subscriptions.cr @@ -0,0 +1,168 @@ +module Invidious::Routes::Subscriptions + def self.toggle_subscription(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env, "/") + + redirect = env.params.query["redirect"]? + redirect ||= "true" + redirect = redirect == "true" + + if !user + if redirect + return env.redirect referer + else + return error_json(403, "No such user") + end + end + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + begin + validate_request(token, sid, env.request, HMAC_KEY, locale) + rescue ex + if redirect + return error_template(400, ex) + else + return error_json(400, ex) + end + end + + if env.params.query["action_create_subscription_to_channel"]?.try &.to_i?.try &.== 1 + action = "action_create_subscription_to_channel" + elsif env.params.query["action_remove_subscriptions"]?.try &.to_i?.try &.== 1 + action = "action_remove_subscriptions" + else + return env.redirect referer + end + + channel_id = env.params.query["c"]? + channel_id ||= "" + + if !user.password + # Sync subscriptions with YouTube + subscribe_ajax(channel_id, action, env.request.headers) + end + + case action + when "action_create_subscription_to_channel" + if !user.subscriptions.includes? channel_id + get_channel(channel_id, false, false) + Invidious::Database::Users.subscribe_channel(user, channel_id) + end + when "action_remove_subscriptions" + Invidious::Database::Users.unsubscribe_channel(user, channel_id) + else + return error_json(400, "Unsupported action #{action}") + end + + if redirect + env.redirect referer + else + env.response.content_type = "application/json" + "{}" + end + end + + def self.subscription_manager(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + sid = sid.as(String) + + if !user.password + # Refresh account + headers = HTTP::Headers.new + headers["Cookie"] = env.request.headers["Cookie"] + + user, sid = get_user(sid, headers) + end + + action_takeout = env.params.query["action_takeout"]?.try &.to_i? + action_takeout ||= 0 + action_takeout = action_takeout == 1 + + format = env.params.query["format"]? + format ||= "rss" + + subscriptions = Invidious::Database::Channels.select(user.subscriptions) + subscriptions.sort_by!(&.author.downcase) + + if action_takeout + if format == "json" + env.response.content_type = "application/json" + env.response.headers["content-disposition"] = "attachment" + playlists = Invidious::Database::Playlists.select_like_iv(user.email) + + return JSON.build do |json| + json.object do + json.field "subscriptions", user.subscriptions + json.field "watch_history", user.watched + json.field "preferences", user.preferences + json.field "playlists" do + json.array do + playlists.each do |playlist| + json.object do + json.field "title", playlist.title + json.field "description", html_to_content(playlist.description_html) + json.field "privacy", playlist.privacy.to_s + json.field "videos" do + json.array do + Invidious::Database::PlaylistVideos.select_ids(playlist.id, playlist.index, limit: 500).each do |video_id| + json.string video_id + end + end + end + end + end + end + end + end + end + else + env.response.content_type = "application/xml" + env.response.headers["content-disposition"] = "attachment" + export = XML.build do |xml| + xml.element("opml", version: "1.1") do + xml.element("body") do + if format == "newpipe" + title = "YouTube Subscriptions" + else + title = "Invidious Subscriptions" + end + + xml.element("outline", text: title, title: title) do + subscriptions.each do |channel| + if format == "newpipe" + xml_url = "https://www.youtube.com/feeds/videos.xml?channel_id=#{channel.id}" + else + xml_url = "#{HOST_URL}/feed/channel/#{channel.id}" + end + + xml.element("outline", text: channel.author, title: channel.author, + "type": "rss", xmlUrl: xml_url) + end + end + end + end + end + + return export.gsub(%(<?xml version="1.0"?>\n), "") + end + end + + templated "subscription_manager" + end +end diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index acbf62b4..8a58b034 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -1,7 +1,7 @@ module Invidious::Routes::VideoPlayback # /videoplayback def self.get_video_playback(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale query_params = env.params.query fvip = query_params["fvip"]? || "3" @@ -20,7 +20,7 @@ module Invidious::Routes::VideoPlayback host = "https://r#{fvip}---#{mns.pop}.googlevideo.com" end - url = "/videoplayback?#{query_params.to_s}" + url = "/videoplayback?#{query_params}" headers = HTTP::Headers.new REQUEST_HEADERS_WHITELIST.each do |header| @@ -240,7 +240,7 @@ module Invidious::Routes::VideoPlayback download_widget = JSON.parse(env.params.query["download_widget"]) id = download_widget["id"].as_s - title = download_widget["title"].as_s + title = URI.decode_www_form(download_widget["title"].as_s) if label = download_widget["label"]? return env.redirect "/api/v1/captions/#{id}?label=#{label}&title=#{title}" @@ -263,7 +263,7 @@ module Invidious::Routes::VideoPlayback haltf env, status_code: 400, response: "TESTING" end - video = get_video(id, PG_DB, region: region) + video = get_video(id, region: region) fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag } url = fmt.try &.["url"]?.try &.as_s diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index f07b1358..7d048ce8 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -1,6 +1,8 @@ +{% skip_file if flag?(:api_only) %} + module Invidious::Routes::Watch def self.handle(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? + locale = env.get("preferences").as(Preferences).locale region = env.params.query["region"]? if env.params.query.to_s.includes?("%20") || env.params.query.to_s.includes?("+") @@ -37,7 +39,7 @@ module Invidious::Routes::Watch end plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") - continuation = process_continuation(PG_DB, env.params.query, plid, id) + continuation = process_continuation(env.params.query, plid, id) nojs = env.params.query["nojs"]? @@ -58,7 +60,7 @@ module Invidious::Routes::Watch env.params.query.delete_all("listen") begin - video = get_video(id, PG_DB, region: params.region) + video = get_video(id, region: params.region) rescue ex : VideoRedirect return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex @@ -74,11 +76,11 @@ module Invidious::Routes::Watch env.params.query.delete_all("iv_load_policy") if watched && !watched.includes? id - PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.as(User).email) + Invidious::Database::Users.mark_watched(user.as(User), id) end if notifications && notifications.includes? id - PG_DB.exec("UPDATE users SET notifications = array_remove(notifications, $1) WHERE email = $2", id, user.as(User).email) + Invidious::Database::Users.remove_notification(user.as(User), id) env.get("user").as(User).notifications.delete(id) notifications.delete(id) end @@ -151,11 +153,11 @@ module Invidious::Routes::Watch preferred_captions = captions.select { |caption| params.preferred_captions.includes?(caption.name) || - params.preferred_captions.includes?(caption.languageCode.split("-")[0]) + params.preferred_captions.includes?(caption.language_code.split("-")[0]) } preferred_captions.sort_by! { |caption| (params.preferred_captions.index(caption.name) || - params.preferred_captions.index(caption.languageCode.split("-")[0])).not_nil! + params.preferred_captions.index(caption.language_code.split("-")[0])).not_nil! } captions = captions - preferred_captions @@ -198,4 +200,70 @@ module Invidious::Routes::Watch return env.redirect url end + + def self.mark_watched(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env, "/feed/subscriptions") + + redirect = env.params.query["redirect"]? + redirect ||= "true" + redirect = redirect == "true" + + if !user + if redirect + return env.redirect referer + else + return error_json(403, "No such user") + end + end + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + id = env.params.query["id"]? + if !id + env.response.status_code = 400 + return + end + + begin + validate_request(token, sid, env.request, HMAC_KEY, locale) + rescue ex + if redirect + return error_template(400, ex) + else + return error_json(400, ex) + end + end + + if env.params.query["action_mark_watched"]? + action = "action_mark_watched" + elsif env.params.query["action_mark_unwatched"]? + action = "action_mark_unwatched" + else + return env.redirect referer + end + + case action + when "action_mark_watched" + if !user.watched.includes? id + Invidious::Database::Users.mark_watched(user, id) + end + when "action_mark_unwatched" + Invidious::Database::Users.mark_unwatched(user, id) + else + return error_json(400, "Unsupported action #{action}") + end + + if redirect + env.redirect referer + else + env.response.content_type = "application/json" + "{}" + end + end end diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index e0cddeb5..7551f22d 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -73,7 +73,7 @@ macro define_v1_api_routes Invidious::Routing.get "/api/v1/stats", {{namespace}}::Misc, :stats Invidious::Routing.get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist Invidious::Routing.get "/api/v1/auth/playlists/:plid", {{namespace}}::Misc, :get_playlist - Invidious::Routing.get "/api/v1//mixes/:rdid", {{namespace}}::Misc, :mixes + Invidious::Routing.get "/api/v1/mixes/:rdid", {{namespace}}::Misc, :mixes end macro define_api_manifest_routes diff --git a/src/invidious/search.cr b/src/invidious/search.cr index a3fcc7a3..0f6dc6eb 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -1,233 +1,10 @@ -struct SearchVideo - include DB::Serializable - - property title : String - property id : String - property author : String - property ucid : String - property published : Time - property views : Int64 - property description_html : String - property length_seconds : Int32 - property live_now : Bool - property premium : Bool - property premiere_timestamp : Time? - - def to_xml(auto_generated, query_params, xml : XML::Builder) - query_params["v"] = self.id - - xml.element("entry") do - xml.element("id") { xml.text "yt:video:#{self.id}" } - xml.element("yt:videoId") { xml.text self.id } - xml.element("yt:channelId") { xml.text self.ucid } - xml.element("title") { xml.text self.title } - xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}") - - xml.element("author") do - if auto_generated - xml.element("name") { xml.text self.author } - xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } - else - xml.element("name") { xml.text author } - xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" } - end - end - - xml.element("content", type: "xhtml") do - xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do - xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do - xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg") - end - - xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text html_to_content(self.description_html) } - end - end - - xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") } - - xml.element("media:group") do - xml.element("media:title") { xml.text self.title } - xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg", - width: "320", height: "180") - xml.element("media:description") { xml.text html_to_content(self.description_html) } - end - - xml.element("media:community") do - xml.element("media:statistics", views: self.views) - end - end - end - - def to_xml(auto_generated, query_params, xml : XML::Builder | Nil = nil) - if xml - to_xml(HOST_URL, auto_generated, query_params, xml) - else - XML.build do |json| - to_xml(HOST_URL, auto_generated, query_params, xml) - end - end - end - - def to_json(locale, json : JSON::Builder) - json.object do - json.field "type", "video" - json.field "title", self.title - json.field "videoId", self.id - - json.field "author", self.author - json.field "authorId", self.ucid - json.field "authorUrl", "/channel/#{self.ucid}" - - json.field "videoThumbnails" do - generate_thumbnails(json, self.id) - end +class ChannelSearchException < InfoException + getter channel : String - json.field "description", html_to_content(self.description_html) - json.field "descriptionHtml", self.description_html - - json.field "viewCount", self.views - json.field "published", self.published.to_unix - json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) - json.field "lengthSeconds", self.length_seconds - json.field "liveNow", self.live_now - json.field "premium", self.premium - json.field "isUpcoming", self.is_upcoming - - if self.premiere_timestamp - json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix - end - end - end - - def to_json(locale, json : JSON::Builder | Nil = nil) - if json - to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end - end - end - - def is_upcoming - premiere_timestamp ? true : false + def initialize(@channel) end end -struct SearchPlaylistVideo - include DB::Serializable - - property title : String - property id : String - property length_seconds : Int32 -end - -struct SearchPlaylist - include DB::Serializable - - property title : String - property id : String - property author : String - property ucid : String - property video_count : Int32 - property videos : Array(SearchPlaylistVideo) - property thumbnail : String? - - def to_json(locale, json : JSON::Builder) - json.object do - json.field "type", "playlist" - json.field "title", self.title - json.field "playlistId", self.id - json.field "playlistThumbnail", self.thumbnail - - json.field "author", self.author - json.field "authorId", self.ucid - json.field "authorUrl", "/channel/#{self.ucid}" - - json.field "videoCount", self.video_count - json.field "videos" do - json.array do - self.videos.each do |video| - json.object do - json.field "title", video.title - json.field "videoId", video.id - json.field "lengthSeconds", video.length_seconds - - json.field "videoThumbnails" do - generate_thumbnails(json, video.id) - end - end - end - end - end - end - end - - def to_json(locale, json : JSON::Builder | Nil = nil) - if json - to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end - end - end -end - -struct SearchChannel - include DB::Serializable - - property author : String - property ucid : String - property author_thumbnail : String - property subscriber_count : Int32 - property video_count : Int32 - property description_html : String - property auto_generated : Bool - - def to_json(locale, json : JSON::Builder) - json.object do - json.field "type", "channel" - json.field "author", self.author - json.field "authorId", self.ucid - json.field "authorUrl", "/channel/#{self.ucid}" - - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - - qualities.each do |quality| - json.object do - json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", quality - end - end - end - end - - json.field "autoGenerated", self.auto_generated - json.field "subCount", self.subscriber_count - json.field "videoCount", self.video_count - - json.field "description", html_to_content(self.description_html) - json.field "descriptionHtml", self.description_html - end - end - - def to_json(locale, json : JSON::Builder | Nil = nil) - if json - to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end - end - end -end - -alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist - def channel_search(query, page, channel) response = YT_POOL.client &.get("/channel/#{channel}") @@ -235,8 +12,8 @@ def channel_search(query, page, channel) response = YT_POOL.client &.get("/user/#{channel}") response = YT_POOL.client &.get("/c/#{channel}") if response.status_code == 404 initial_data = extract_initial_data(response.body) - ucid = initial_data["header"]["c4TabbedHeaderRenderer"]?.try &.["channelId"].as_s? - raise InfoException.new("Impossible to extract channel ID from page") if !ucid + ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?) + raise ChannelSearchException.new(channel) if !ucid else ucid = channel end @@ -244,13 +21,13 @@ def channel_search(query, page, channel) continuation = produce_channel_search_continuation(ucid, query, page) response_json = YoutubeAPI.browse(continuation) - continuationItems = response_json["onResponseReceivedActions"]? + continuation_items = response_json["onResponseReceivedActions"]? .try &.[0]["appendContinuationItemsAction"]["continuationItems"] - return 0, [] of SearchItem if !continuationItems + return 0, [] of SearchItem if !continuation_items items = [] of SearchItem - continuationItems.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each { |item| + continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each { |item| extract_item(item["itemSectionRenderer"]["contents"].as_a[0]) .try { |t| items << t } } @@ -358,7 +135,7 @@ def produce_search_params(page = 1, sort : String = "relevance", date : String = object.delete("2:embedded") end - params = object.try { |i| Protodec::Any.cast_json(object) } + params = object.try { |i| Protodec::Any.cast_json(i) } .try { |i| Protodec::Any.from_json(i) } .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } @@ -391,7 +168,7 @@ def produce_channel_search_continuation(ucid, query, page) }, } - continuation = object.try { |i| Protodec::Any.cast_json(object) } + continuation = object.try { |i| Protodec::Any.cast_json(i) } .try { |i| Protodec::Any.from_json(i) } .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } @@ -413,7 +190,7 @@ def process_search_query(query, page, user, region) sort = "relevance" subscriptions = nil - operators = query.split(" ").select { |a| a.match(/\w+:[\w,]+/) } + operators = query.split(" ").select(&.match(/\w+:[\w,]+/)) operators.each do |operator| key, value = operator.downcase.split(":") @@ -462,5 +239,20 @@ def process_search_query(query, page, user, region) count, items = search(search_query, search_params, region).as(Tuple) end - {search_query, count, items, operators} + # Light processing to flatten search results out of Categories. + # They should ideally be supported in the future. + items_without_category = [] of SearchItem | ChannelVideo + items.each do |i| + if i.is_a? Category + i.contents.each do |nest_i| + if !nest_i.is_a? Video + items_without_category << nest_i + end + end + else + items_without_category << i + end + end + + {search_query, items_without_category.size, items_without_category, operators} end diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index 25bab4d2..1f957081 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr @@ -20,13 +20,3 @@ def fetch_trending(trending_type, region, locale) return {trending, plid} end - -def extract_plid(url) - return url.try { |i| URI.parse(i).query } - .try { |i| HTTP::Params.parse(i)["bp"] } - .try { |i| URI.decode_www_form(i) } - .try { |i| Base64.decode(i) } - .try { |i| IO::Memory.new(i) } - .try { |i| Protodec::Any.parse(i) } - .try &.["44:0:embedded"]?.try &.["2:1:string"]?.try &.as_s -end diff --git a/src/invidious/user/converters.cr b/src/invidious/user/converters.cr new file mode 100644 index 00000000..dcbf8c53 --- /dev/null +++ b/src/invidious/user/converters.cr @@ -0,0 +1,12 @@ +def convert_theme(theme) + case theme + when "true" + "dark" + when "false" + "light" + when "", nil + nil + else + theme + end +end diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr new file mode 100644 index 00000000..2ae1dcb1 --- /dev/null +++ b/src/invidious/user/imports.cr @@ -0,0 +1,27 @@ +require "csv" + +def parse_subscription_export_csv(csv_content : String) + rows = CSV.new(csv_content, headers: true) + subscriptions = Array(String).new + + # Counter to limit the amount of imports. + # This is intended to prevent DoS. + row_counter = 0 + + rows.each do |row| + # Limit to 1200 + row_counter += 1 + break if row_counter > 1_200 + + # Channel ID is the first column in the csv export we can't use the header + # name, because the header name is localized depending on the + # language the user has set on their account + channel_id = row[0].strip + + next if channel_id.empty? + + subscriptions << channel_id + end + + return subscriptions +end diff --git a/src/invidious/user/preferences.cr b/src/invidious/user/preferences.cr new file mode 100644 index 00000000..bf7ea401 --- /dev/null +++ b/src/invidious/user/preferences.cr @@ -0,0 +1,259 @@ +struct Preferences + include JSON::Serializable + include YAML::Serializable + + property annotations : Bool = CONFIG.default_user_preferences.annotations + property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed + property autoplay : Bool = CONFIG.default_user_preferences.autoplay + property automatic_instance_redirect : Bool = CONFIG.default_user_preferences.automatic_instance_redirect + + @[JSON::Field(converter: Preferences::StringToArray)] + @[YAML::Field(converter: Preferences::StringToArray)] + property captions : Array(String) = CONFIG.default_user_preferences.captions + + @[JSON::Field(converter: Preferences::StringToArray)] + @[YAML::Field(converter: Preferences::StringToArray)] + property comments : Array(String) = CONFIG.default_user_preferences.comments + property continue : Bool = CONFIG.default_user_preferences.continue + property continue_autoplay : Bool = CONFIG.default_user_preferences.continue_autoplay + + @[JSON::Field(converter: Preferences::BoolToString)] + @[YAML::Field(converter: Preferences::BoolToString)] + property dark_mode : String = CONFIG.default_user_preferences.dark_mode + property latest_only : Bool = CONFIG.default_user_preferences.latest_only + property listen : Bool = CONFIG.default_user_preferences.listen + property local : Bool = CONFIG.default_user_preferences.local + property vr_mode : Bool = CONFIG.default_user_preferences.vr_mode + property show_nick : Bool = CONFIG.default_user_preferences.show_nick + + @[JSON::Field(converter: Preferences::ProcessString)] + property locale : String = CONFIG.default_user_preferences.locale + property region : String? = CONFIG.default_user_preferences.region + + @[JSON::Field(converter: Preferences::ClampInt)] + property max_results : Int32 = CONFIG.default_user_preferences.max_results + property notifications_only : Bool = CONFIG.default_user_preferences.notifications_only + + @[JSON::Field(converter: Preferences::ProcessString)] + property player_style : String = CONFIG.default_user_preferences.player_style + + @[JSON::Field(converter: Preferences::ProcessString)] + property quality : String = CONFIG.default_user_preferences.quality + @[JSON::Field(converter: Preferences::ProcessString)] + property quality_dash : String = CONFIG.default_user_preferences.quality_dash + property default_home : String? = CONFIG.default_user_preferences.default_home + property feed_menu : Array(String) = CONFIG.default_user_preferences.feed_menu + property related_videos : Bool = CONFIG.default_user_preferences.related_videos + + @[JSON::Field(converter: Preferences::ProcessString)] + property sort : String = CONFIG.default_user_preferences.sort + property speed : Float32 = CONFIG.default_user_preferences.speed + property thin_mode : Bool = CONFIG.default_user_preferences.thin_mode + property unseen_only : Bool = CONFIG.default_user_preferences.unseen_only + property video_loop : Bool = CONFIG.default_user_preferences.video_loop + property extend_desc : Bool = CONFIG.default_user_preferences.extend_desc + property volume : Int32 = CONFIG.default_user_preferences.volume + property save_player_pos : Bool = CONFIG.default_user_preferences.save_player_pos + + module BoolToString + def self.to_json(value : String, json : JSON::Builder) + json.string value + end + + def self.from_json(value : JSON::PullParser) : String + begin + result = value.read_string + + if result.empty? + CONFIG.default_user_preferences.dark_mode + else + result + end + rescue ex + if value.read_bool + "dark" + else + "light" + end + end + end + + def self.to_yaml(value : String, yaml : YAML::Nodes::Builder) + yaml.scalar value + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String + unless node.is_a?(YAML::Nodes::Scalar) + node.raise "Expected scalar, not #{node.class}" + end + + case node.value + when "true" + "dark" + when "false" + "light" + when "" + CONFIG.default_user_preferences.dark_mode + else + node.value + end + end + end + + module ClampInt + def self.to_json(value : Int32, json : JSON::Builder) + json.number value + end + + def self.from_json(value : JSON::PullParser) : Int32 + value.read_int.clamp(0, MAX_ITEMS_PER_PAGE).to_i32 + end + + def self.to_yaml(value : Int32, yaml : YAML::Nodes::Builder) + yaml.scalar value + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Int32 + node.value.clamp(0, MAX_ITEMS_PER_PAGE) + end + end + + module FamilyConverter + def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder) + case value + when Socket::Family::UNSPEC + yaml.scalar nil + when Socket::Family::INET + yaml.scalar "ipv4" + when Socket::Family::INET6 + yaml.scalar "ipv6" + when Socket::Family::UNIX + raise "Invalid socket family #{value}" + end + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family + if node.is_a?(YAML::Nodes::Scalar) + case node.value.downcase + when "ipv4" + Socket::Family::INET + when "ipv6" + Socket::Family::INET6 + else + Socket::Family::UNSPEC + end + else + node.raise "Expected scalar, not #{node.class}" + end + end + end + + module URIConverter + def self.to_yaml(value : URI, yaml : YAML::Nodes::Builder) + yaml.scalar value.normalize! + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : URI + if node.is_a?(YAML::Nodes::Scalar) + URI.parse node.value + else + node.raise "Expected scalar, not #{node.class}" + end + end + end + + module ProcessString + def self.to_json(value : String, json : JSON::Builder) + json.string value + end + + def self.from_json(value : JSON::PullParser) : String + HTML.escape(value.read_string[0, 100]) + end + + def self.to_yaml(value : String, yaml : YAML::Nodes::Builder) + yaml.scalar value + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String + HTML.escape(node.value[0, 100]) + end + end + + module StringToArray + def self.to_json(value : Array(String), json : JSON::Builder) + json.array do + value.each do |element| + json.string element + end + end + end + + def self.from_json(value : JSON::PullParser) : Array(String) + begin + result = [] of String + value.read_array do + result << HTML.escape(value.read_string[0, 100]) + end + rescue ex + result = [HTML.escape(value.read_string[0, 100]), ""] + end + + result + end + + def self.to_yaml(value : Array(String), yaml : YAML::Nodes::Builder) + yaml.sequence do + value.each do |element| + yaml.scalar element + end + end + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(String) + begin + unless node.is_a?(YAML::Nodes::Sequence) + node.raise "Expected sequence, not #{node.class}" + end + + result = [] of String + node.nodes.each do |item| + unless item.is_a?(YAML::Nodes::Scalar) + node.raise "Expected scalar, not #{item.class}" + end + + result << HTML.escape(item.value[0, 100]) + end + rescue ex + if node.is_a?(YAML::Nodes::Scalar) + result = [HTML.escape(node.value[0, 100]), ""] + else + result = ["", ""] + end + end + + result + end + end + + module StringToCookies + def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder) + (value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml) + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies + unless node.is_a?(YAML::Nodes::Scalar) + node.raise "Expected scalar, not #{node.class}" + end + + cookies = HTTP::Cookies.new + node.value.split(";").each do |cookie| + next if cookie.strip.empty? + name, value = cookie.split("=", 2) + cookies << HTTP::Cookie.new(name.strip, value.strip) + end + + cookies + end + end +end diff --git a/src/invidious/users.cr b/src/invidious/users.cr index aff76b53..49074994 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -29,301 +29,31 @@ struct User end end -struct Preferences - include JSON::Serializable - include YAML::Serializable - - property annotations : Bool = CONFIG.default_user_preferences.annotations - property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed - property autoplay : Bool = CONFIG.default_user_preferences.autoplay - property automatic_instance_redirect : Bool = CONFIG.default_user_preferences.automatic_instance_redirect - - @[JSON::Field(converter: Preferences::StringToArray)] - @[YAML::Field(converter: Preferences::StringToArray)] - property captions : Array(String) = CONFIG.default_user_preferences.captions - - @[JSON::Field(converter: Preferences::StringToArray)] - @[YAML::Field(converter: Preferences::StringToArray)] - property comments : Array(String) = CONFIG.default_user_preferences.comments - property continue : Bool = CONFIG.default_user_preferences.continue - property continue_autoplay : Bool = CONFIG.default_user_preferences.continue_autoplay - - @[JSON::Field(converter: Preferences::BoolToString)] - @[YAML::Field(converter: Preferences::BoolToString)] - property dark_mode : String = CONFIG.default_user_preferences.dark_mode - property latest_only : Bool = CONFIG.default_user_preferences.latest_only - property listen : Bool = CONFIG.default_user_preferences.listen - property local : Bool = CONFIG.default_user_preferences.local - property vr_mode : Bool = CONFIG.default_user_preferences.vr_mode - property show_nick : Bool = CONFIG.default_user_preferences.show_nick - - @[JSON::Field(converter: Preferences::ProcessString)] - property locale : String = CONFIG.default_user_preferences.locale - - @[JSON::Field(converter: Preferences::ClampInt)] - property max_results : Int32 = CONFIG.default_user_preferences.max_results - property notifications_only : Bool = CONFIG.default_user_preferences.notifications_only - - @[JSON::Field(converter: Preferences::ProcessString)] - property player_style : String = CONFIG.default_user_preferences.player_style - - @[JSON::Field(converter: Preferences::ProcessString)] - property quality : String = CONFIG.default_user_preferences.quality - @[JSON::Field(converter: Preferences::ProcessString)] - property quality_dash : String = CONFIG.default_user_preferences.quality_dash - property default_home : String? = CONFIG.default_user_preferences.default_home - property feed_menu : Array(String) = CONFIG.default_user_preferences.feed_menu - property related_videos : Bool = CONFIG.default_user_preferences.related_videos - - @[JSON::Field(converter: Preferences::ProcessString)] - property sort : String = CONFIG.default_user_preferences.sort - property speed : Float32 = CONFIG.default_user_preferences.speed - property thin_mode : Bool = CONFIG.default_user_preferences.thin_mode - property unseen_only : Bool = CONFIG.default_user_preferences.unseen_only - property video_loop : Bool = CONFIG.default_user_preferences.video_loop - property extend_desc : Bool = CONFIG.default_user_preferences.extend_desc - property volume : Int32 = CONFIG.default_user_preferences.volume - - module BoolToString - def self.to_json(value : String, json : JSON::Builder) - json.string value - end - - def self.from_json(value : JSON::PullParser) : String - begin - result = value.read_string - - if result.empty? - CONFIG.default_user_preferences.dark_mode - else - result - end - rescue ex - if value.read_bool - "dark" - else - "light" - end - end - end - - def self.to_yaml(value : String, yaml : YAML::Nodes::Builder) - yaml.scalar value - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String - unless node.is_a?(YAML::Nodes::Scalar) - node.raise "Expected scalar, not #{node.class}" - end - - case node.value - when "true" - "dark" - when "false" - "light" - when "" - CONFIG.default_user_preferences.dark_mode - else - node.value - end - end - end - - module ClampInt - def self.to_json(value : Int32, json : JSON::Builder) - json.number value - end - - def self.from_json(value : JSON::PullParser) : Int32 - value.read_int.clamp(0, MAX_ITEMS_PER_PAGE).to_i32 - end - - def self.to_yaml(value : Int32, yaml : YAML::Nodes::Builder) - yaml.scalar value - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Int32 - node.value.clamp(0, MAX_ITEMS_PER_PAGE) - end - end - - module FamilyConverter - def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder) - case value - when Socket::Family::UNSPEC - yaml.scalar nil - when Socket::Family::INET - yaml.scalar "ipv4" - when Socket::Family::INET6 - yaml.scalar "ipv6" - when Socket::Family::UNIX - raise "Invalid socket family #{value}" - end - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family - if node.is_a?(YAML::Nodes::Scalar) - case node.value.downcase - when "ipv4" - Socket::Family::INET - when "ipv6" - Socket::Family::INET6 - else - Socket::Family::UNSPEC - end - else - node.raise "Expected scalar, not #{node.class}" - end - end - end - - module URIConverter - def self.to_yaml(value : URI, yaml : YAML::Nodes::Builder) - yaml.scalar value.normalize! - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : URI - if node.is_a?(YAML::Nodes::Scalar) - URI.parse node.value - else - node.raise "Expected scalar, not #{node.class}" - end - end - end - - module ProcessString - def self.to_json(value : String, json : JSON::Builder) - json.string value - end - - def self.from_json(value : JSON::PullParser) : String - HTML.escape(value.read_string[0, 100]) - end - - def self.to_yaml(value : String, yaml : YAML::Nodes::Builder) - yaml.scalar value - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String - HTML.escape(node.value[0, 100]) - end - end - - module StringToArray - def self.to_json(value : Array(String), json : JSON::Builder) - json.array do - value.each do |element| - json.string element - end - end - end - - def self.from_json(value : JSON::PullParser) : Array(String) - begin - result = [] of String - value.read_array do - result << HTML.escape(value.read_string[0, 100]) - end - rescue ex - result = [HTML.escape(value.read_string[0, 100]), ""] - end - - result - end - - def self.to_yaml(value : Array(String), yaml : YAML::Nodes::Builder) - yaml.sequence do - value.each do |element| - yaml.scalar element - end - end - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(String) - begin - unless node.is_a?(YAML::Nodes::Sequence) - node.raise "Expected sequence, not #{node.class}" - end - - result = [] of String - node.nodes.each do |item| - unless item.is_a?(YAML::Nodes::Scalar) - node.raise "Expected scalar, not #{item.class}" - end - - result << HTML.escape(item.value[0, 100]) - end - rescue ex - if node.is_a?(YAML::Nodes::Scalar) - result = [HTML.escape(node.value[0, 100]), ""] - else - result = ["", ""] - end - end - - result - end - end - - module StringToCookies - def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder) - (value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml) - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies - unless node.is_a?(YAML::Nodes::Scalar) - node.raise "Expected scalar, not #{node.class}" - end - - cookies = HTTP::Cookies.new - node.value.split(";").each do |cookie| - next if cookie.strip.empty? - name, value = cookie.split("=", 2) - cookies << HTTP::Cookie.new(name.strip, value.strip) - end - - cookies - end - end -end - -def get_user(sid, headers, db, refresh = true) - if email = db.query_one?("SELECT email FROM session_ids WHERE id = $1", sid, as: String) - user = db.query_one("SELECT * FROM users WHERE email = $1", email, as: User) +def get_user(sid, headers, refresh = true) + if email = Invidious::Database::SessionIDs.select_email(sid) + user = Invidious::Database::Users.select!(email: email) if refresh && Time.utc - user.updated > 1.minute - user, sid = fetch_user(sid, headers, db) - user_array = user.to_a - user_array[4] = user_array[4].to_json # User preferences - args = arg_array(user_array) - - db.exec("INSERT INTO users VALUES (#{args}) \ - ON CONFLICT (email) DO UPDATE SET updated = $1, subscriptions = $3", args: user_array) + user, sid = fetch_user(sid, headers) - db.exec("INSERT INTO session_ids VALUES ($1,$2,$3) \ - ON CONFLICT (id) DO NOTHING", sid, user.email, Time.utc) + Invidious::Database::Users.insert(user, update_on_conflict: true) + Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true) begin view_name = "subscriptions_#{sha256(user.email)}" - db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") + PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") rescue ex end end else - user, sid = fetch_user(sid, headers, db) - user_array = user.to_a - user_array[4] = user_array[4].to_json # User preferences - args = arg_array(user.to_a) - - db.exec("INSERT INTO users VALUES (#{args}) \ - ON CONFLICT (email) DO UPDATE SET updated = $1, subscriptions = $3", args: user_array) + user, sid = fetch_user(sid, headers) - db.exec("INSERT INTO session_ids VALUES ($1,$2,$3) \ - ON CONFLICT (id) DO NOTHING", sid, user.email, Time.utc) + Invidious::Database::Users.insert(user, update_on_conflict: true) + Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true) begin view_name = "subscriptions_#{sha256(user.email)}" - db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") + PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") rescue ex end end @@ -331,7 +61,7 @@ def get_user(sid, headers, db, refresh = true) return user, sid end -def fetch_user(sid, headers, db) +def fetch_user(sid, headers) feed = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers) feed = XML.parse_html(feed.body) @@ -344,7 +74,7 @@ def fetch_user(sid, headers, db) end end - channels = get_batch_channels(channels, db, false, false) + channels = get_batch_channels(channels, false, false) email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"])) if email @@ -388,7 +118,7 @@ def create_user(sid, email, password) return user, sid end -def generate_captcha(key, db) +def generate_captcha(key) second = Random::Secure.rand(12) second_angle = second * 30 second = second * 5 @@ -440,16 +170,16 @@ def generate_captcha(key, db) return { question: image, - tokens: {generate_response(answer, {":login"}, key, db, use_nonce: true)}, + tokens: {generate_response(answer, {":login"}, key, use_nonce: true)}, } end -def generate_text_captcha(key, db) +def generate_text_captcha(key) response = make_client(TEXTCAPTCHA_URL, &.get("/github.com/iv.org/invidious.json").body) response = JSON.parse(response) tokens = response["a"].as_a.map do |answer| - generate_response(answer.as_s, {":login"}, key, db, use_nonce: true) + generate_response(answer.as_s, {":login"}, key, use_nonce: true) end return { @@ -490,33 +220,29 @@ def subscribe_ajax(channel_id, action, env_headers) end end -def get_subscription_feed(db, user, max_results = 40, page = 1) +def get_subscription_feed(user, max_results = 40, page = 1) limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE) offset = (page - 1) * limit - notifications = db.query_one("SELECT notifications FROM users WHERE email = $1", user.email, - as: Array(String)) + notifications = Invidious::Database::Users.select_notifications(user) view_name = "subscriptions_#{sha256(user.email)}" if user.preferences.notifications_only && !notifications.empty? # Only show notifications - - args = arg_array(notifications) - - notifications = db.query_all("SELECT * FROM channel_videos WHERE id IN (#{args}) ORDER BY published DESC", args: notifications, as: ChannelVideo) + notifications = Invidious::Database::ChannelVideos.select(notifications) videos = [] of ChannelVideo - notifications.sort_by! { |video| video.published }.reverse! + notifications.sort_by!(&.published).reverse! case user.preferences.sort when "alphabetically" - notifications.sort_by! { |video| video.title } + notifications.sort_by!(&.title) when "alphabetically - reverse" - notifications.sort_by! { |video| video.title }.reverse! + notifications.sort_by!(&.title).reverse! when "channel name" - notifications.sort_by! { |video| video.author } + notifications.sort_by!(&.author) when "channel name - reverse" - notifications.sort_by! { |video| video.author }.reverse! + notifications.sort_by!(&.author).reverse! else nil # Ignore end else @@ -537,7 +263,7 @@ def get_subscription_feed(db, user, max_results = 40, page = 1) videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo) end - videos.sort_by! { |video| video.published }.reverse! + videos.sort_by!(&.published).reverse! else if user.preferences.unseen_only # Only show unwatched @@ -557,20 +283,19 @@ def get_subscription_feed(db, user, max_results = 40, page = 1) case user.preferences.sort when "published - reverse" - videos.sort_by! { |video| video.published } + videos.sort_by!(&.published) when "alphabetically" - videos.sort_by! { |video| video.title } + videos.sort_by!(&.title) when "alphabetically - reverse" - videos.sort_by! { |video| video.title }.reverse! + videos.sort_by!(&.title).reverse! when "channel name" - videos.sort_by! { |video| video.author } + videos.sort_by!(&.author) when "channel name - reverse" - videos.sort_by! { |video| video.author }.reverse! + videos.sort_by!(&.author).reverse! else nil # Ignore end - notifications = PG_DB.query_one("SELECT notifications FROM users WHERE email = $1", user.email, as: Array(String)) - + notifications = Invidious::Database::Users.select_notifications(user) notifications = videos.select { |v| notifications.includes? v.id } videos = videos - notifications end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index d9c07142..499ed94d 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -246,6 +246,7 @@ struct VideoPreferences property video_start : Float64 | Int32 property volume : Int32 property vr_mode : Bool + property save_player_pos : Bool end struct Video @@ -275,7 +276,7 @@ struct Video end end - def to_json(locale, json : JSON::Builder) + def to_json(locale : String?, json : JSON::Builder) json.object do json.field "type", "video" @@ -426,7 +427,7 @@ struct Video self.captions.each do |caption| json.object do json.field "label", caption.name - json.field "languageCode", caption.languageCode + json.field "language_code", caption.language_code json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}" end end @@ -474,14 +475,13 @@ struct Video end end - def to_json(locale, json : JSON::Builder | Nil = nil) - if json - to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end - end + # TODO: remove the locale and follow the crystal convention + def to_json(locale : String?, _json : Nil) + JSON.build { |json| to_json(locale, json) } + end + + def to_json(json : JSON::Builder | Nil = nil) + to_json(nil, json) end def title @@ -703,10 +703,10 @@ struct Video return @captions.as(Array(Caption)) if @captions captions = info["captions"]?.try &.["playerCaptionsTracklistRenderer"]?.try &.["captionTracks"]?.try &.as_a.map do |caption| name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"] - languageCode = caption["languageCode"].to_s - baseUrl = caption["baseUrl"].to_s + language_code = caption["languageCode"].to_s + base_url = caption["baseUrl"].to_s - caption = Caption.new(name.to_s, languageCode, baseUrl) + caption = Caption.new(name.to_s, language_code, base_url) caption.name = caption.name.split(" - ")[0] caption end @@ -785,16 +785,16 @@ end struct Caption property name - property languageCode - property baseUrl + property language_code + property base_url getter name : String - getter languageCode : String - getter baseUrl : String + getter language_code : String + getter base_url : String setter name - def initialize(@name, @languageCode, @baseUrl) + def initialize(@name, @language_code, @base_url) end end @@ -858,8 +858,16 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ else client_config.client_type = YoutubeAPI::ClientType::Android end - stream_data = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) - params["streamingData"] = stream_data["streamingData"]? || JSON::Any.new("") + android_player = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) + + # Sometime, the video is available from the web client, but not on Android, so check + # that here, and fallback to the streaming data from the web client if needed. + # See: https://github.com/iv-org/invidious/issues/2549 + if android_player["playabilityStatus"]["status"] == "OK" + params["streamingData"] = android_player["streamingData"]? || JSON::Any.new("") + else + params["streamingData"] = player_response["streamingData"]? || JSON::Any.new("") + end end {"captions", "microformat", "playabilityStatus", "storyboards", "videoDetails"}.each do |f| @@ -878,42 +886,84 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ } ).try { |a| JSON::Any.new(a) } || JSON::Any.new([] of JSON::Any) - primary_results = player_response.try &.["contents"]?.try &.["twoColumnWatchNextResults"]?.try &.["results"]? - .try &.["results"]?.try &.["contents"]? - sentiment_bar = primary_results.try &.as_a.select { |object| object["videoPrimaryInfoRenderer"]? }[0]? - .try &.["videoPrimaryInfoRenderer"]? - .try &.["sentimentBar"]? - .try &.["sentimentBarRenderer"]? - .try &.["tooltip"]? - .try &.as_s - - likes, dislikes = sentiment_bar.try &.split(" / ", 2).map &.gsub(/\D/, "").to_i64 || {0_i64, 0_i64} - params["likes"] = JSON::Any.new(likes) - params["dislikes"] = JSON::Any.new(dislikes) - - params["descriptionHtml"] = JSON::Any.new(primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]? - .try &.["videoSecondaryInfoRenderer"]?.try &.["description"]?.try &.["runs"]? - .try &.as_a.try { |t| content_to_comment_html(t).gsub("\n", "<br/>") } || "<p></p>") - - metadata = primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]? - .try &.["videoSecondaryInfoRenderer"]? - .try &.["metadataRowContainer"]? - .try &.["metadataRowContainerRenderer"]? - .try &.["rows"]? - .try &.as_a + # Top level elements + + primary_results = player_response + .dig?("contents", "twoColumnWatchNextResults", "results", "results", "contents") + + video_primary_renderer = primary_results + .try &.as_a.find(&.["videoPrimaryInfoRenderer"]?) + .try &.["videoPrimaryInfoRenderer"] + + video_secondary_renderer = primary_results + .try &.as_a.find(&.["videoSecondaryInfoRenderer"]?) + .try &.["videoSecondaryInfoRenderer"] + + # Likes/dislikes + + toplevel_buttons = video_primary_renderer + .try &.dig?("videoActions", "menuRenderer", "topLevelButtons") + + if toplevel_buttons + likes_button = toplevel_buttons.as_a + .find(&.dig("toggleButtonRenderer", "defaultIcon", "iconType").as_s.== "LIKE") + .try &.["toggleButtonRenderer"] + + if likes_button + likes_txt = (likes_button["defaultText"]? || likes_button["toggledText"]?) + .try &.dig?("accessibility", "accessibilityData", "label") + likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt + + LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"") + LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes + end + + dislikes_button = toplevel_buttons.as_a + .find(&.dig("toggleButtonRenderer", "defaultIcon", "iconType").as_s.== "DISLIKE") + .try &.["toggleButtonRenderer"] + + if dislikes_button + dislikes_txt = (dislikes_button["defaultText"]? || dislikes_button["toggledText"]?) + .try &.dig?("accessibility", "accessibilityData", "label") + dislikes = dislikes_txt.as_s.gsub(/\D/, "").to_i64? if dislikes_txt + + LOGGER.trace("extract_video_info: Found \"dislikes\" button. Button text is \"#{dislikes_txt}\"") + LOGGER.debug("extract_video_info: Dislikes count is #{dislikes}") if dislikes + end + end + + if likes && likes != 0_i64 && (!dislikes || dislikes == 0_i64) + if rating = player_response.dig?("videoDetails", "averageRating").try { |x| x.as_i64? || x.as_f? } + dislikes = (likes * ((5 - rating)/(rating - 1))).round.to_i64 + LOGGER.debug("extract_video_info: Dislikes count (using fallback method) is #{dislikes}") + end + end + + params["likes"] = JSON::Any.new(likes || 0_i64) + params["dislikes"] = JSON::Any.new(dislikes || 0_i64) + + # Description + + description_html = video_secondary_renderer.try &.dig?("description", "runs") + .try &.as_a.try { |t| content_to_comment_html(t) } + + params["descriptionHtml"] = JSON::Any.new(description_html || "<p></p>") + + # Video metadata + + metadata = video_secondary_renderer + .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows") + .try &.as_a params["genre"] = params["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["category"]? || JSON::Any.new("") params["genreUrl"] = JSON::Any.new(nil) metadata.try &.each do |row| title = row["metadataRowRenderer"]?.try &.["title"]?.try &.["simpleText"]?.try &.as_s - contents = row["metadataRowRenderer"]? - .try &.["contents"]? - .try &.as_a[0]? + contents = row.dig?("metadataRowRenderer", "contents", 0) if title.try &.== "Category" - contents = contents.try &.["runs"]? - .try &.as_a[0]? + contents = contents.try &.dig?("runs", 0) params["genre"] = JSON::Any.new(contents.try &.["text"]?.try &.as_s || "") params["genreUcid"] = JSON::Any.new(contents.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]? @@ -928,21 +978,23 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ end end - author_info = primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]? - .try &.["videoSecondaryInfoRenderer"]?.try &.["owner"]?.try &.["videoOwnerRenderer"]? + # Author infos - params["authorThumbnail"] = JSON::Any.new(author_info.try &.["thumbnail"]? - .try &.["thumbnails"]?.try &.as_a[0]?.try &.["url"]? - .try &.as_s || "") + author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer") + author_thumbnail = author_info.try &.dig?("thumbnail", "thumbnails", 0, "url") + + params["authorThumbnail"] = JSON::Any.new(author_thumbnail.try &.as_s || "") params["subCountText"] = JSON::Any.new(author_info.try &.["subscriberCountText"]? - .try { |t| t["simpleText"]? || t["runs"]?.try &.[0]?.try &.["text"]? }.try &.as_s.split(" ", 2)[0] || "-") + .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") }.try &.as_s.split(" ", 2)[0] || "-") + + # Return data - params + return params end -def get_video(id, db, refresh = true, region = nil, force_refresh = false) - if (video = db.query_one?("SELECT * FROM videos WHERE id = $1", id, as: Video)) && !region +def get_video(id, refresh = true, region = nil, force_refresh = false) + if (video = Invidious::Database::Videos.select(id)) && !region # If record was last updated over 10 minutes ago, or video has since premiered, # refresh (expire param in response lasts for 6 hours) if (refresh && @@ -951,17 +1003,15 @@ def get_video(id, db, refresh = true, region = nil, force_refresh = false) force_refresh begin video = fetch_video(id, region) - db.exec("UPDATE videos SET (id, info, updated) = ($1, $2, $3) WHERE id = $1", video.id, video.info.to_json, video.updated) + Invidious::Database::Videos.update(video) rescue ex - db.exec("DELETE FROM videos * WHERE id = $1", id) + Invidious::Database::Videos.delete(id) raise ex end end else video = fetch_video(id, region) - if !region - db.exec("INSERT INTO videos VALUES ($1, $2, $3) ON CONFLICT (id) DO NOTHING", video.id, video.info.to_json, video.updated) - end + Invidious::Database::Videos.insert(video) if !region end return video @@ -1006,7 +1056,7 @@ def itag_to_metadata?(itag : JSON::Any) return VIDEO_FORMATS[itag.to_s]? end -def process_continuation(db, query, plid, id) +def process_continuation(query, plid, id) continuation = nil if plid if index = query["index"]?.try &.to_i? @@ -1023,13 +1073,13 @@ end def process_video_params(query, preferences) annotations = query["iv_load_policy"]?.try &.to_i? autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } - comments = query["comments"]?.try &.split(",").map { |a| a.downcase } + comments = query["comments"]?.try &.split(",").map(&.downcase) continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe } continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe } local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe } player_style = query["player_style"]? - preferred_captions = query["subtitles"]?.try &.split(",").map { |a| a.downcase } + preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase) quality = query["quality"]? quality_dash = query["quality_dash"]? region = query["region"]? @@ -1039,6 +1089,7 @@ def process_video_params(query, preferences) extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe } volume = query["volume"]?.try &.to_i? vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe } + save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe } if preferences # region ||= preferences.region @@ -1059,6 +1110,7 @@ def process_video_params(query, preferences) extend_desc ||= preferences.extend_desc.to_unsafe volume ||= preferences.volume vr_mode ||= preferences.vr_mode.to_unsafe + save_player_pos ||= preferences.save_player_pos.to_unsafe end annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe @@ -1078,6 +1130,7 @@ def process_video_params(query, preferences) extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe volume ||= CONFIG.default_user_preferences.volume vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe + save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe annotations = annotations == 1 autoplay = autoplay == 1 @@ -1089,6 +1142,7 @@ def process_video_params(query, preferences) video_loop = video_loop == 1 extend_desc = extend_desc == 1 vr_mode = vr_mode == 1 + save_player_pos = save_player_pos == 1 if CONFIG.disabled?("dash") && quality == "dash" quality = "high" @@ -1139,6 +1193,7 @@ def process_video_params(query, preferences) video_start: video_start, volume: volume, vr_mode: vr_mode, + save_player_pos: save_player_pos, }) return params diff --git a/src/invidious/views/add_playlist_items.ecr b/src/invidious/views/add_playlist_items.ecr index 09eacbc8..c62861b0 100644 --- a/src/invidious/views/add_playlist_items.ecr +++ b/src/invidious/views/add_playlist_items.ecr @@ -41,7 +41,7 @@ <div class="pure-g h-box"> <div class="pure-u-1 pure-u-lg-1-5"> <% if page > 1 %> - <a href="/add_playlist_items?list=<%= plid %>&q=<%= HTML.escape(query.not_nil!) %>&page=<%= page - 1 %>"> + <a href="/add_playlist_items?list=<%= plid %>&q=<%= URI.encode_www_form(query.not_nil!) %>&page=<%= page - 1 %>"> <%= translate(locale, "Previous page") %> </a> <% end %> @@ -49,7 +49,7 @@ <div class="pure-u-1 pure-u-lg-3-5"></div> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> <% if count >= 20 %> - <a href="/add_playlist_items?list=<%= plid %>&q=<%= HTML.escape(query.not_nil!) %>&page=<%= page + 1 %>"> + <a href="/add_playlist_items?list=<%= plid %>&q=<%= URI.encode_www_form(query.not_nil!) %>&page=<%= page + 1 %>"> <%= translate(locale, "Next page") %> </a> <% end %> diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr index 09cfb76e..40b553a9 100644 --- a/src/invidious/views/channel.ecr +++ b/src/invidious/views/channel.ecr @@ -45,7 +45,11 @@ <div class="pure-u-1-3"> <a href="https://www.youtube.com/channel/<%= ucid %>"><%= translate(locale, "View channel on YouTube") %></a> <div class="pure-u-1 pure-md-1-3"> - <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a> + <% if env.get("preferences").as(Preferences).automatic_instance_redirect%> + <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a> + <% else %> + <a href="https://redirect.invidious.io<%= env.request.path %>"><%= translate(locale, "Switch Invidious Instance") %></a> + <% end %> </div> <% if !channel.auto_generated %> <div class="pure-u-1 pure-md-1-3"> @@ -96,7 +100,7 @@ <div class="pure-g h-box"> <div class="pure-u-1 pure-u-lg-1-5"> <% if page > 1 %> - <a href="/channel/<%= ucid %>?page=<%= page - 1 %><% if sort_by != "newest" %>&sort_by=<%= HTML.escape(sort_by) %><% end %>"> + <a href="/channel/<%= ucid %>?page=<%= page - 1 %><% if sort_by != "newest" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>"> <%= translate(locale, "Previous page") %> </a> <% end %> @@ -104,7 +108,7 @@ <div class="pure-u-1 pure-u-lg-3-5"></div> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> <% if count == 60 %> - <a href="/channel/<%= ucid %>?page=<%= page + 1 %><% if sort_by != "newest" %>&sort_by=<%= HTML.escape(sort_by) %><% end %>"> + <a href="/channel/<%= ucid %>?page=<%= page + 1 %><% if sort_by != "newest" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>"> <%= translate(locale, "Next page") %> </a> <% end %> diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr index 15d8ed1e..f0add06b 100644 --- a/src/invidious/views/community.ecr +++ b/src/invidious/views/community.ecr @@ -44,7 +44,11 @@ <div class="pure-u-1-3"> <a href="https://www.youtube.com/channel/<%= channel.ucid %>/community"><%= translate(locale, "View channel on YouTube") %></a> <div class="pure-u-1 pure-md-1-3"> - <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a> + <% if env.get("preferences").as(Preferences).automatic_instance_redirect%> + <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a> + <% else %> + <a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a> + <% end %> </div> <% if !channel.auto_generated %> <div class="pure-u-1 pure-md-1-3"> diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index 68aa1812..5a93d802 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -5,13 +5,13 @@ <a href="/channel/<%= item.ucid %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <center> - <img style="width:56.25%" src="/ggpht<%= URI.parse(item.author_thumbnail).request_target.gsub(/=s\d+/, "=s176") %>"/> + <img loading="lazy" style="width:56.25%" src="/ggpht<%= URI.parse(item.author_thumbnail).request_target.gsub(/=s\d+/, "=s176") %>"/> </center> <% end %> <p dir="auto"><%= HTML.escape(item.author) %></p> </a> - <p><%= translate(locale, "`x` subscribers", number_with_separator(item.subscriber_count)) %></p> - <% if !item.auto_generated %><p><%= translate(locale, "`x` videos", number_with_separator(item.video_count)) %></p><% end %> + <p><%= translate_count(locale, "generic_subscribers_count", item.subscriber_count, NumberFormatting::Separator) %></p> + <% if !item.auto_generated %><p><%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %></p><% end %> <h5><%= item.description_html %></h5> <% when SearchPlaylist, InvidiousPlaylist %> <% if item.id.starts_with? "RD" %> @@ -23,8 +23,8 @@ <a style="width:100%" href="<%= url %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> - <img class="thumbnail" src="<%= URI.parse(item.thumbnail || "/").request_target %>"/> - <p class="length"><%= number_with_separator(item.video_count) %> videos</p> + <img loading="lazy" class="thumbnail" src="<%= URI.parse(item.thumbnail || "/").request_target %>"/> + <p class="length"><%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %></p> </div> <% end %> <p dir="auto"><%= HTML.escape(item.title) %></p> @@ -36,7 +36,7 @@ <a href="/watch?v=<%= item.id %>&list=<%= item.rdid %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> - <img class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/> + <img loading="lazy" class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/> <% if item.length_seconds != 0 %> <p class="length"><%= recode_length_seconds(item.length_seconds) %></p> <% end %> @@ -48,10 +48,10 @@ <p dir="auto"><b><%= HTML.escape(item.author) %></b></p> </a> <% when PlaylistVideo %> - <a style="width:100%" href="/watch?v=<%= item.id %>&list=<%= item.plid %>"> + <a style="width:100%" href="/watch?v=<%= item.id %>&list=<%= item.plid %>&index=<%= item.index %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> - <img class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/> + <img loading="lazy" class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/> <% if plid = env.get?("remove_playlist_items") %> <form data-onsubmit="return_false" action="/playlist_ajax?action_remove_video=1&set_video_id=<%= item.index %>&playlist_id=<%= plid %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> @@ -79,6 +79,8 @@ <div class="flex-left"><a href="/channel/<%= item.ucid %>"> <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %></p> </a></div> + <% endpoint_params = "?v=#{item.id}&list=#{item.plid}" %> + <%= rendered "components/video-context-buttons" %> </div> <div class="video-card-row flexible"> @@ -92,15 +94,16 @@ <% if item.responds_to?(:views) && item.views %> <div class="flex-right"> - <p dir="auto"><%= translate(locale, "`x` views", number_to_short_text(item.views || 0)) %></p> + <p dir="auto"><%= translate_count(locale, "generic_views_count", item.views || 0, NumberFormatting::Short) %></p> </div> <% end %> </div> + <% when Category %> <% else %> <a style="width:100%" href="/watch?v=<%= item.id %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> - <img class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/> + <img loading="lazy" class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/> <% if env.get? "show_watched" %> <form data-onsubmit="return_false" action="/watch_ajax?action_mark_watched=1&id=<%= item.id %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> @@ -141,19 +144,9 @@ <div class="flex-left"><a href="/channel/<%= item.ucid %>"> <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %></p> </a></div> - <div class="flex-right"> - <div class="icon-buttons"> - <a title="<%=translate(locale, "Watch on YouTube")%>" href="https://www.youtube.com/watch?v=<%= item.id %>"> - <i class="icon ion-logo-youtube"></i> - </a> - <a title="<%=translate(locale, "Audio mode")%>" href="/watch?v=<%= item.id %>&listen=1"> - <i class="icon ion-md-headset"></i> - </a> - <a title="<%=translate(locale, "Switch Invidious Instance")%>" href="/redirect?referer=<%=HTML.escape("watch?v=#{item.id}")%>"> - <i class="icon ion-md-jet"></i> - </a> - </div> - </div> + + <% endpoint_params = "?v=#{item.id}" %> + <%= rendered "components/video-context-buttons" %> </div> <div class="video-card-row flexible"> @@ -167,7 +160,7 @@ <% if item.responds_to?(:views) && item.views %> <div class="flex-right"> - <p class="video-data" dir="auto"><%= translate(locale, "`x` views", number_to_short_text(item.views || 0)) %></p> + <p class="video-data" dir="auto"><%= translate_count(locale, "generic_views_count", item.views || 0, NumberFormatting::Short) %></p> </div> <% end %> </div> diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index 6418f66b..206ba380 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -32,13 +32,11 @@ <% end %> <% preferred_captions.each do |caption| %> - <track kind="captions" src="/api/v1/captions/<%= video.id %>?label=<%= caption.name %>&hl=<%= env.get("preferences").as(Preferences).locale %>" - label="<%= caption.name %>"> + <track kind="captions" src="/api/v1/captions/<%= video.id %>?label=<%= caption.name %>" label="<%= caption.name %>"> <% end %> <% captions.each do |caption| %> - <track kind="captions" src="/api/v1/captions/<%= video.id %>?label=<%= caption.name %>&hl=<%= env.get("preferences").as(Preferences).locale %>" - label="<%= caption.name %>"> + <track kind="captions" src="/api/v1/captions/<%= video.id %>?label=<%= caption.name %>" label="<%= caption.name %>"> <% end %> <% end %> </video> diff --git a/src/invidious/views/components/search_box.ecr b/src/invidious/views/components/search_box.ecr new file mode 100644 index 00000000..1240e5bd --- /dev/null +++ b/src/invidious/views/components/search_box.ecr @@ -0,0 +1,9 @@ +<form class="pure-form" action="/search" method="get"> + <fieldset> + <input type="search" id="searchbox" autocomplete="off" autocorrect="off" + autocapitalize="none" spellcheck="false" <% if autofocus %>autofocus<% end %> + name="q" placeholder="<%= translate(locale, "search") %>" + title="<%= translate(locale, "search") %>" + value="<%= env.get?("search").try {|x| HTML.escape(x.as(String)) } %>"> + </fieldset> +</form> diff --git a/src/invidious/views/components/video-context-buttons.ecr b/src/invidious/views/components/video-context-buttons.ecr new file mode 100644 index 00000000..ddb6c983 --- /dev/null +++ b/src/invidious/views/components/video-context-buttons.ecr @@ -0,0 +1,21 @@ +<div class="flex-right"> + <div class="icon-buttons"> + <a title="<%=translate(locale, "videoinfo_watch_on_youTube")%>" href="https://www.youtube.com/watch<%=endpoint_params%>"> + <i class="icon ion-logo-youtube"></i> + </a> + <a title="<%=translate(locale, "Audio mode")%>" href="/watch<%=endpoint_params%>&listen=1"> + <i class="icon ion-md-headset"></i> + </a> + + <% if env.get("preferences").as(Preferences).automatic_instance_redirect%> + <a title="<%=translate(locale, "Switch Invidious Instance")%>" href="/redirect?referer=%2Fwatch<%=URI.encode_www_form(endpoint_params)%>"> + <i class="icon ion-md-jet"></i> + </a> + <% else %> + <a title="<%=translate(locale, "Switch Invidious Instance")%>" href="https://redirect.invidious.io/watch<%=endpoint_params%>"> + <i class="icon ion-md-jet"></i> + </a> + <% end %> + + </div> +</div>
\ No newline at end of file diff --git a/src/invidious/views/edit_playlist.ecr b/src/invidious/views/edit_playlist.ecr index 5046abc1..308bd677 100644 --- a/src/invidious/views/edit_playlist.ecr +++ b/src/invidious/views/edit_playlist.ecr @@ -11,7 +11,7 @@ <h3><input class="pure-input-1" maxlength="150" name="title" type="text" value="<%= title %>"></h3> <b> <%= HTML.escape(playlist.author) %> | - <%= translate(locale, "`x` videos", "#{playlist.video_count}") %> | + <%= translate_count(locale, "generic_videos_count", playlist.video_count) %> | <%= translate(locale, "Updated `x` ago", recode_date(playlist.updated, locale)) %> | <i class="icon <%= {"ion-md-globe", "ion-ios-unlock", "ion-ios-lock"}[playlist.privacy.value] %>"></i> <select name="privacy"> diff --git a/src/invidious/views/feeds/history.ecr b/src/invidious/views/feeds/history.ecr index 40584979..6c1243c5 100644 --- a/src/invidious/views/feeds/history.ecr +++ b/src/invidious/views/feeds/history.ecr @@ -4,11 +4,11 @@ <div class="pure-g h-box"> <div class="pure-u-1-3"> - <h3><%= translate(locale, "`x` videos", %(<span id="count">#{user.watched.size}</span>)) %></h3> + <h3><%= translate_count(locale, "generic_videos_count", user.watched.size, NumberFormatting::HtmlSpan) %></h3> </div> <div class="pure-u-1-3"> <h3 style="text-align:center"> - <a href="/feed/subscriptions"><%= translate(locale, "`x` subscriptions", %(<span id="count">#{user.subscriptions.size}</span>)) %></a> + <a href="/feed/subscriptions"><%= translate_count(locale, "generic_subscriptions_count", user.subscriptions.size, NumberFormatting::HtmlSpan) %></a> </h3> </div> <div class="pure-u-1-3"> diff --git a/src/invidious/views/feeds/playlists.ecr b/src/invidious/views/feeds/playlists.ecr index 868cfeda..a59344c4 100644 --- a/src/invidious/views/feeds/playlists.ecr +++ b/src/invidious/views/feeds/playlists.ecr @@ -6,7 +6,7 @@ <div class="pure-g h-box"> <div class="pure-u-2-3"> - <h3><%= translate(locale, "`x` created playlists", %(<span id="count">#{items_created.size}</span>)) %></h3> + <h3><%= translate(locale, "user_created_playlists", %(<span id="count">#{items_created.size}</span>)) %></h3> </div> <div class="pure-u-1-3" style="text-align:right"> <h3> @@ -23,7 +23,7 @@ <div class="pure-g h-box"> <div class="pure-u-1"> - <h3><%= translate(locale, "`x` saved playlists", %(<span id="count">#{items_saved.size}</span>)) %></h3> + <h3><%= translate(locale, "user_saved_playlists", %(<span id="count">#{items_saved.size}</span>)) %></h3> </div> </div> diff --git a/src/invidious/views/feeds/subscriptions.ecr b/src/invidious/views/feeds/subscriptions.ecr index 97184e2b..8d56ad14 100644 --- a/src/invidious/views/feeds/subscriptions.ecr +++ b/src/invidious/views/feeds/subscriptions.ecr @@ -24,7 +24,7 @@ </div> <center> - <%= translate(locale, "`x` unseen notifications", "#{notifications.size}") %> + <%= translate_count(locale, "subscriptions_unseen_notifs_count", notifications.size) %> </center> <% if !notifications.empty? %> diff --git a/src/invidious/views/login.ecr b/src/invidious/views/login.ecr index 1f6618e8..e2963e9f 100644 --- a/src/invidious/views/login.ecr +++ b/src/invidious/views/login.ecr @@ -6,21 +6,6 @@ <div class="pure-u-1 pure-u-lg-1-5"></div> <div class="pure-u-1 pure-u-lg-3-5"> <div class="h-box"> - <div class="pure-g"> - <div class="pure-u-1-2"> - <a class="pure-button <% if account_type == "invidious" %>pure-button-disabled<% end %>" href="/login?type=invidious"> - <%= translate(locale, "Log in/register") %> - </a> - </div> - <div class="pure-u-1-2"> - <a class="pure-button <% if account_type == "google" %>pure-button-disabled<% end %>" href="/login?type=google"> - <%= translate(locale, "Log in with Google") %> - </a> - </div> - </div> - - <hr> - <% case account_type when %> <% when "google" %> <form class="pure-form pure-form-stacked" action="/login?referer=<%= URI.encode_www_form(referer) %>&type=google" method="post"> diff --git a/src/invidious/views/playlist.ecr b/src/invidious/views/playlist.ecr index 12f93a72..df3112db 100644 --- a/src/invidious/views/playlist.ecr +++ b/src/invidious/views/playlist.ecr @@ -16,7 +16,7 @@ <% else %> <%= author %> | <% end %> - <%= translate(locale, "`x` videos", "#{playlist.video_count}") %> | + <%= translate_count(locale, "generic_videos_count", playlist.video_count) %> | <%= translate(locale, "Updated `x` ago", recode_date(playlist.updated, locale)) %> | <% case playlist.as(InvidiousPlaylist).privacy when %> <% when PlaylistPrivacy::Public %> @@ -30,7 +30,7 @@ <% else %> <b> <a href="/channel/<%= playlist.ucid %>"><%= author %></a> | - <%= translate(locale, "`x` videos", "#{playlist.video_count}") %> | + <%= translate_count(locale, "generic_videos_count", playlist.video_count) %> | <%= translate(locale, "Updated `x` ago", recode_date(playlist.updated, locale)) %> </b> <% end %> @@ -41,9 +41,16 @@ <%= translate(locale, "View playlist on YouTube") %> </a> <span> | </span> - <a href="/redirect?referer=<%= env.get?("current_page") %>"> - <%= translate(locale, "Switch Invidious Instance") %> - </a> + + <% if env.get("preferences").as(Preferences).automatic_instance_redirect%> + <a href="/redirect?referer=<%= env.get?("current_page") %>"> + <%= translate(locale, "Switch Invidious Instance") %> + </a> + <% else %> + <a href="https://redirect.invidious.io/playlist?list=<%= playlist.id %>"> + <%= translate(locale, "Switch Invidious Instance") %> + </a> + <% end %> </div> <% end %> </div> @@ -54,7 +61,7 @@ <div class="pure-u-1-3"><a href="/edit_playlist?list=<%= plid %>"><i class="icon ion-md-create"></i></a></div> <div class="pure-u-1-3"><a href="/delete_playlist?list=<%= plid %>"><i class="icon ion-md-trash"></i></a></div> <% else %> - <% if PG_DB.query_one?("SELECT id FROM playlists WHERE id = $1", playlist.id, as: String).nil? %> + <% if Invidious::Database::Playlists.exists?(playlist.id) %> <div class="pure-u-1-3"><a href="/subscribe_playlist?list=<%= plid %>"><i class="icon ion-md-add"></i></a></div> <% else %> <div class="pure-u-1-3"><a href="/delete_playlist?list=<%= plid %>"><i class="icon ion-md-trash"></i></a></div> @@ -67,9 +74,7 @@ </div> <div class="h-box"> - <div id="descriptionWrapper"> - <p><%= playlist.description_html %></p> - </div> + <div id="descriptionWrapper"><%= playlist.description_html %></div> </div> <% if playlist.is_a?(InvidiousPlaylist) && playlist.author == user.try &.email %> diff --git a/src/invidious/views/playlists.ecr b/src/invidious/views/playlists.ecr index d9a17a9b..12dba088 100644 --- a/src/invidious/views/playlists.ecr +++ b/src/invidious/views/playlists.ecr @@ -47,7 +47,11 @@ </div> <div class="pure-u-1 pure-md-1-3"> - <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a> + <% if env.get("preferences").as(Preferences).automatic_instance_redirect%> + <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a> + <% else %> + <a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a> + <% end %> </div> <div class="pure-u-1 pure-md-1-3"> @@ -96,7 +100,7 @@ <div class="pure-u-1 pure-u-md-4-5"></div> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> <% if continuation %> - <a href="/channel/<%= ucid %>/playlists?continuation=<%= continuation %><% if sort_by != "last" %>&sort_by=<%= HTML.escape(sort_by) %><% end %>"> + <a href="/channel/<%= ucid %>/playlists?continuation=<%= continuation %><% if sort_by != "last" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>"> <%= translate(locale, "Next page") %> </a> <% end %> diff --git a/src/invidious/views/preferences.ecr b/src/invidious/views/preferences.ecr index be021c59..96904259 100644 --- a/src/invidious/views/preferences.ecr +++ b/src/invidious/views/preferences.ecr @@ -5,40 +5,40 @@ <div class="h-box"> <form class="pure-form pure-form-aligned" action="/preferences?referer=<%= URI.encode_www_form(referer) %>" method="post"> <fieldset> - <legend><%= translate(locale, "Player preferences") %></legend> + <legend><%= translate(locale, "preferences_category_player") %></legend> <div class="pure-control-group"> - <label for="video_loop"><%= translate(locale, "Always loop: ") %></label> + <label for="video_loop"><%= translate(locale, "preferences_video_loop_label") %></label> <input name="video_loop" id="video_loop" type="checkbox" <% if preferences.video_loop %>checked<% end %>> </div> <div class="pure-control-group"> - <label for="autoplay"><%= translate(locale, "Autoplay: ") %></label> + <label for="autoplay"><%= translate(locale, "preferences_autoplay_label") %></label> <input name="autoplay" id="autoplay" type="checkbox" <% if preferences.autoplay %>checked<% end %>> </div> <div class="pure-control-group"> - <label for="continue"><%= translate(locale, "Play next by default: ") %></label> + <label for="continue"><%= translate(locale, "preferences_continue_label") %></label> <input name="continue" id="continue" type="checkbox" <% if preferences.continue %>checked<% end %>> </div> <div class="pure-control-group"> - <label for="continue_autoplay"><%= translate(locale, "Autoplay next video: ") %></label> + <label for="continue_autoplay"><%= translate(locale, "preferences_continue_autoplay_label") %></label> <input name="continue_autoplay" id="continue_autoplay" type="checkbox" <% if preferences.continue_autoplay %>checked<% end %>> </div> <div class="pure-control-group"> - <label for="local"><%= translate(locale, "Proxy videos: ") %></label> + <label for="local"><%= translate(locale, "preferences_local_label") %></label> <input name="local" id="local" type="checkbox" <% if preferences.local && !CONFIG.disabled?("local") %>checked<% end %> <% if CONFIG.disabled?("local") %>disabled<% end %>> </div> <div class="pure-control-group"> - <label for="listen"><%= translate(locale, "Listen by default: ") %></label> + <label for="listen"><%= translate(locale, "preferences_listen_label") %></label> <input name="listen" id="listen" type="checkbox" <% if preferences.listen %>checked<% end %>> </div> <div class="pure-control-group"> - <label for="speed"><%= translate(locale, "Default speed: ") %></label> + <label for="speed"><%= translate(locale, "preferences_speed_label") %></label> <select name="speed" id="speed"> <% {2.0, 1.75, 1.5, 1.25, 1.0, 0.75, 0.5, 0.25}.each do |option| %> <option <% if preferences.speed == option %> selected <% end %>><%= option %></option> @@ -47,11 +47,11 @@ </div> <div class="pure-control-group"> - <label for="quality"><%= translate(locale, "Preferred video quality: ") %></label> + <label for="quality"><%= translate(locale, "preferences_quality_label") %></label> <select name="quality" id="quality"> <% {"dash", "hd720", "medium", "small"}.each do |option| %> <% if !(option == "dash" && CONFIG.disabled?("dash")) %> - <option value="<%= option %>" <% if preferences.quality == option %> selected <% end %>><%= translate(locale, option) %></option> + <option value="<%= option %>" <% if preferences.quality == option %> selected <% end %>><%= translate(locale, "preferences_quality_option_" + option) %></option> <% end %> <% end %> </select> @@ -59,23 +59,23 @@ <% if !CONFIG.disabled?("dash") %> <div class="pure-control-group"> - <label for="quality_dash"><%= translate(locale, "Preferred dash video quality: ") %></label> + <label for="quality_dash"><%= translate(locale, "preferences_quality_dash_label") %></label> <select name="quality_dash" id="quality_dash"> <% {"auto", "best", "4320p", "2160p", "1440p", "1080p", "720p", "480p", "360p", "240p", "144p", "worst"}.each do |option| %> - <option value="<%= option %>" <% if preferences.quality_dash == option %> selected <% end %>><%= translate(locale, option) %></option> + <option value="<%= option %>" <% if preferences.quality_dash == option %> selected <% end %>><%= translate(locale, "preferences_quality_dash_option_" + option) %></option> <% end %> </select> </div> <% end %> <div class="pure-control-group"> - <label for="volume"><%= translate(locale, "Player volume: ") %></label> + <label for="volume"><%= translate(locale, "preferences_volume_label") %></label> <input name="volume" id="volume" data-onrange="update_volume_value" type="range" min="0" max="100" step="5" value="<%= preferences.volume %>"> <span class="pure-form-message-inline" id="volume-value"><%= preferences.volume %></span> </div> <div class="pure-control-group"> - <label for="comments[0]"><%= translate(locale, "Default comments: ") %></label> + <label for="comments[0]"><%= translate(locale, "preferences_comments_label") %></label> <% preferences.comments.each_with_index do |comments, index| %> <select name="comments[<%= index %>]" id="comments[<%= index %>]"> <% {"", "youtube", "reddit"}.each do |option| %> @@ -86,7 +86,7 @@ </div> <div class="pure-control-group"> - <label for="captions[0]"><%= translate(locale, "Default captions: ") %></label> + <label for="captions[0]"><%= translate(locale, "preferences_captions_label") %></label> <% preferences.captions.each_with_index do |caption, index| %> <select class="pure-u-1-6" name="captions[<%= index %>]" id="captions[<%= index %>]"> <% CAPTION_LANGUAGES.each do |option| %> @@ -97,38 +97,52 @@ </div> <div class="pure-control-group"> - <label for="related_videos"><%= translate(locale, "Show related videos: ") %></label> + <label for="related_videos"><%= translate(locale, "preferences_related_videos_label") %></label> <input name="related_videos" id="related_videos" type="checkbox" <% if preferences.related_videos %>checked<% end %>> </div> <div class="pure-control-group"> - <label for="annotations"><%= translate(locale, "Show annotations by default: ") %></label> + <label for="annotations"><%= translate(locale, "preferences_annotations_label") %></label> <input name="annotations" id="annotations" type="checkbox" <% if preferences.annotations %>checked<% end %>> </div> <div class="pure-control-group"> - <label for="extend_desc"><%= translate(locale, "Automatically extend video description: ") %></label> + <label for="extend_desc"><%= translate(locale, "preferences_extend_desc_label") %></label> <input name="extend_desc" id="extend_desc" type="checkbox" <% if preferences.extend_desc %>checked<% end %>> </div> <div class="pure-control-group"> - <label for="vr_mode"><%= translate(locale, "Interactive 360 degree videos") %></label> + <label for="vr_mode"><%= translate(locale, "preferences_vr_mode_label") %></label> <input name="vr_mode" id="vr_mode" type="checkbox" <% if preferences.vr_mode %>checked<% end %>> </div> - <legend><%= translate(locale, "Visual preferences") %></legend> + <div class="pure-control-group"> + <label for="save_player_pos"><%= translate(locale, "preferences_save_player_pos_label") %></label> + <input name="save_player_pos" id="save_player_pos" type="checkbox" <% if preferences.save_player_pos %>checked<% end %>> + </div> + + <legend><%= translate(locale, "preferences_category_visual") %></legend> <div class="pure-control-group"> - <label for="locale"><%= translate(locale, "Language: ") %></label> + <label for="locale"><%= translate(locale, "preferences_locale_label") %></label> <select name="locale" id="locale"> - <% LOCALES.each_key do |option| %> - <option value="<%= option %>" <% if preferences.locale == option %> selected <% end %>><%= option %></option> + <% LOCALES_LIST.each do |iso_name, full_name| %> + <option value="<%= iso_name %>" <% if preferences.locale == iso_name %> selected <% end %>><%= HTML.escape(full_name) %></option> <% end %> </select> </div> <div class="pure-control-group"> - <label for="player_style"><%= translate(locale, "Player style: ") %></label> + <label for="region"><%= translate(locale, "preferences_region_label") %></label> + <select name="region" id="region"> + <% CONTENT_REGIONS.each do |option| %> + <option value="<%= option %>" <% if preferences.region == option %> selected <% end %>><%= option %></option> + <% end %> + </select> + </div> + + <div class="pure-control-group"> + <label for="player_style"><%= translate(locale, "preferences_player_style_label") %></label> <select name="player_style" id="player_style"> <% {"invidious", "youtube"}.each do |option| %> <option value="<%= option %>" <% if preferences.player_style == option %> selected <% end %>><%= translate(locale, option) %></option> @@ -137,7 +151,7 @@ </div> <div class="pure-control-group"> - <label for="dark_mode"><%= translate(locale, "Theme: ") %></label> + <label for="dark_mode"><%= translate(locale, "preferences_dark_mode_label") %></label> <select name="dark_mode" id="dark_mode"> <% {"", "light", "dark"}.each do |option| %> <option value="<%= option %>" <% if preferences.dark_mode == option %> selected <% end %>><%= translate(locale, option.blank? ? "auto" : option) %></option> @@ -146,7 +160,7 @@ </div> <div class="pure-control-group"> - <label for="thin_mode"><%= translate(locale, "Thin mode: ") %></label> + <label for="thin_mode"><%= translate(locale, "preferences_thin_mode_label") %></label> <input name="thin_mode" id="thin_mode" type="checkbox" <% if preferences.thin_mode %>checked<% end %>> </div> @@ -157,7 +171,7 @@ <% end %> <div class="pure-control-group"> - <label for="default_home"><%= translate(locale, "Default homepage: ") %></label> + <label for="default_home"><%= translate(locale, "preferences_default_home_label") %></label> <select name="default_home" id="default_home"> <% feed_options.each do |option| %> <option value="<%= option %>" <% if preferences.default_home == option %> selected <% end %>><%= translate(locale, option.blank? ? "Search" : option) %></option> @@ -166,7 +180,7 @@ </div> <div class="pure-control-group"> - <label for="feed_menu"><%= translate(locale, "Feed menu: ") %></label> + <label for="feed_menu"><%= translate(locale, "preferences_feed_menu_label") %></label> <% (feed_options.size - 1).times do |index| %> <select name="feed_menu[<%= index %>]" id="feed_menu[<%= index %>]"> <% feed_options.each do |option| %> @@ -177,33 +191,33 @@ </div> <% if env.get? "user" %> <div class="pure-control-group"> - <label for="show_nick"><%= translate(locale, "Show nickname on top: ") %></label> + <label for="show_nick"><%= translate(locale, "preferences_show_nick_label") %></label> <input name="show_nick" id="show_nick" type="checkbox" <% if preferences.show_nick %>checked<% end %>> </div> <% end %> - <legend><%= translate(locale, "Miscellaneous preferences") %></legend> + <legend><%= translate(locale, "preferences_category_misc") %></legend> <div class="pure-control-group"> - <label for="automatic_instance_redirect"><%= translate(locale, "Automaticatic instance redirection (fallback to redirect.invidious.io): ") %></label> + <label for="automatic_instance_redirect"><%= translate(locale, "preferences_automatic_instance_redirect_label") %></label> <input name="automatic_instance_redirect" id="automatic_instance_redirect" type="checkbox" <% if preferences.automatic_instance_redirect %>checked<% end %>> </div> <% if env.get? "user" %> - <legend><%= translate(locale, "Subscription preferences") %></legend> + <legend><%= translate(locale, "preferences_category_subscription") %></legend> <div class="pure-control-group"> - <label for="annotations_subscribed"><%= translate(locale, "Show annotations by default for subscribed channels: ") %></label> + <label for="annotations_subscribed"><%= translate(locale, "preferences_annotations_subscribed_label") %></label> <input name="annotations_subscribed" id="annotations_subscribed" type="checkbox" <% if preferences.annotations_subscribed %>checked<% end %>> </div> <div class="pure-control-group"> - <label for="max_results"><%= translate(locale, "Number of videos shown in feed: ") %></label> + <label for="max_results"><%= translate(locale, "preferences_max_results_label") %></label> <input name="max_results" id="max_results" type="number" value="<%= preferences.max_results %>"> </div> <div class="pure-control-group"> - <label for="sort"><%= translate(locale, "Sort videos by: ") %></label> + <label for="sort"><%= translate(locale, "preferences_sort_label") %></label> <select name="sort" id="sort"> <% {"published", "published - reverse", "alphabetically", "alphabetically - reverse", "channel name", "channel name - reverse"}.each do |option| %> <option value="<%= option %>" <% if preferences.sort == option %> selected <% end %>><%= translate(locale, option) %></option> @@ -221,12 +235,12 @@ </div> <div class="pure-control-group"> - <label for="unseen_only"><%= translate(locale, "Only show unwatched: ") %></label> + <label for="unseen_only"><%= translate(locale, "preferences_unseen_only_label") %></label> <input name="unseen_only" id="unseen_only" type="checkbox" <% if preferences.unseen_only %>checked<% end %>> </div> <div class="pure-control-group"> - <label for="notifications_only"><%= translate(locale, "Only show notifications (if there are any): ") %></label> + <label for="notifications_only"><%= translate(locale, "preferences_notifications_only_label") %></label> <input name="notifications_only" id="notifications_only" type="checkbox" <% if preferences.notifications_only %>checked<% end %>> </div> @@ -239,10 +253,10 @@ <% end %> <% if env.get?("user") && CONFIG.admins.includes? env.get?("user").as(User).email %> - <legend><%= translate(locale, "Administrator preferences") %></legend> + <legend><%= translate(locale, "preferences_category_admin") %></legend> <div class="pure-control-group"> - <label for="admin_default_home"><%= translate(locale, "Default homepage: ") %></label> + <label for="admin_default_home"><%= translate(locale, "preferences_default_home_label") %></label> <select name="admin_default_home" id="admin_default_home"> <% feed_options.each do |option| %> <option value="<%= option %>" <% if CONFIG.default_user_preferences.default_home == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option> @@ -251,7 +265,7 @@ </div> <div class="pure-control-group"> - <label for="admin_feed_menu"><%= translate(locale, "Feed menu: ") %></label> + <label for="admin_feed_menu"><%= translate(locale, "preferences_feed_menu_label") %></label> <% (feed_options.size - 1).times do |index| %> <select name="admin_feed_menu[<%= index %>]" id="admin_feed_menu[<%= index %>]"> <% feed_options.each do |option| %> @@ -286,10 +300,15 @@ <label for="statistics_enabled"><%= translate(locale, "Report statistics: ") %></label> <input name="statistics_enabled" id="statistics_enabled" type="checkbox" <% if CONFIG.statistics_enabled %>checked<% end %>> </div> + + <div class="pure-control-group"> + <label for="modified_source_code_url"><%= translate(locale, "adminprefs_modified_source_code_url_label") %></label> + <input name="modified_source_code_url" id="modified_source_code_url" type="input" <% if CONFIG.modified_source_code_url %>checked<% end %>> + </div> <% end %> <% if env.get? "user" %> - <legend><%= translate(locale, "Data preferences") %></legend> + <legend><%= translate(locale, "preferences_category_data") %></legend> <div class="pure-control-group"> <a href="/clear_watch_history?referer=<%= URI.encode_www_form(referer) %>"><%= translate(locale, "Clear watch history") %></a> diff --git a/src/invidious/views/search.ecr b/src/invidious/views/search.ecr index fd176e41..db374548 100644 --- a/src/invidious/views/search.ecr +++ b/src/invidious/views/search.ecr @@ -2,7 +2,7 @@ <title><%= search_query.not_nil!.size > 30 ? HTML.escape(query.not_nil![0,30].rstrip(".") + "...") : HTML.escape(query.not_nil!) %> - Invidious</title> <% end %> -<% search_query_encoded = env.get?("search").try { |x| URI.encode(x.as(String), space_to_plus: true) } %> +<% search_query_encoded = env.get?("search").try { |x| URI.encode_www_form(x.as(String), space_to_plus: true) } %> <!-- Search redirection and filtering UI --> <% if count == 0 %> @@ -23,7 +23,7 @@ <% if operator_hash.fetch("date", "all") == date %> <b><%= translate(locale, date) %></b> <% else %> - <a href="/search?q=<%= HTML.escape(query.not_nil!.gsub(/ ?date:[a-z]+/, "") + " date:" + date) %>&page=<%= page %>"> + <a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?date:[a-z]+/, "") + " date:" + date) %>&page=<%= page %>"> <%= translate(locale, date) %> </a> <% end %> @@ -38,7 +38,7 @@ <% if operator_hash.fetch("content_type", "all") == content_type %> <b><%= translate(locale, content_type) %></b> <% else %> - <a href="/search?q=<%= HTML.escape(query.not_nil!.gsub(/ ?content_type:[a-z]+/, "") + " content_type:" + content_type) %>&page=<%= page %>"> + <a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?content_type:[a-z]+/, "") + " content_type:" + content_type) %>&page=<%= page %>"> <%= translate(locale, content_type) %> </a> <% end %> @@ -53,7 +53,7 @@ <% if operator_hash.fetch("duration", "all") == duration %> <b><%= translate(locale, duration) %></b> <% else %> - <a href="/search?q=<%= HTML.escape(query.not_nil!.gsub(/ ?duration:[a-z]+/, "") + " duration:" + duration) %>&page=<%= page %>"> + <a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?duration:[a-z]+/, "") + " duration:" + duration) %>&page=<%= page %>"> <%= translate(locale, duration) %> </a> <% end %> @@ -68,11 +68,11 @@ <% if operator_hash.fetch("features", "all").includes?(feature) %> <b><%= translate(locale, feature) %></b> <% elsif operator_hash.has_key?("features") %> - <a href="/search?q=<%= HTML.escape(query.not_nil!.gsub(/features:/, "features:" + feature + ",")) %>&page=<%= page %>"> + <a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/features:/, "features:" + feature + ",")) %>&page=<%= page %>"> <%= translate(locale, feature) %> </a> <% else %> - <a href="/search?q=<%= HTML.escape(query.not_nil! + " features:" + feature) %>&page=<%= page %>"> + <a href="/search?q=<%= URI.encode_www_form(query.not_nil! + " features:" + feature) %>&page=<%= page %>"> <%= translate(locale, feature) %> </a> <% end %> @@ -87,7 +87,7 @@ <% if operator_hash.fetch("sort", "relevance") == sort %> <b><%= translate(locale, sort) %></b> <% else %> - <a href="/search?q=<%= HTML.escape(query.not_nil!.gsub(/ ?sort:[a-z]+/, "") + " sort:" + sort) %>&page=<%= page %>"> + <a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?sort:[a-z]+/, "") + " sort:" + sort) %>&page=<%= page %>"> <%= translate(locale, sort) %> </a> <% end %> diff --git a/src/invidious/views/search_homepage.ecr b/src/invidious/views/search_homepage.ecr index 7d2dab83..2424a1cf 100644 --- a/src/invidious/views/search_homepage.ecr +++ b/src/invidious/views/search_homepage.ecr @@ -14,11 +14,7 @@ </div> <div class="pure-u-1-4"></div> <div class="pure-u-1 pure-u-md-12-24 searchbar"> - <form class="pure-form" action="/search" method="get"> - <fieldset> - <input autofocus type="search" style="width:100%" name="q" placeholder="<%= translate(locale, "search") %>" value="<%= env.get?("search").try {|x| HTML.escape(x.as(String)) } %>"> - </fieldset> - </form> + <% autofocus = true %><%= rendered "components/search_box" %> </div> <div class="pure-u-1-4"></div> </div> diff --git a/src/invidious/views/subscription_manager.ecr b/src/invidious/views/subscription_manager.ecr index acf015f5..5fa7d203 100644 --- a/src/invidious/views/subscription_manager.ecr +++ b/src/invidious/views/subscription_manager.ecr @@ -6,7 +6,7 @@ <div class="pure-u-1-3"> <h3> <a href="/feed/subscriptions"> - <%= translate(locale, "`x` subscriptions", %(<span id="count">#{subscriptions.size}</span>)) %> + <%= translate_count(locale, "generic_subscriptions_count", subscriptions.size, NumberFormatting::HtmlSpan) %> </a> </h3> </div> diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index 7be95959..240b523a 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -19,8 +19,10 @@ <link rel="stylesheet" href="/css/default.css?v=<%= ASSET_COMMIT %>"> </head> -<% locale = LOCALES[env.get("preferences").as(Preferences).locale]? %> -<% dark_mode = env.get("preferences").as(Preferences).dark_mode %> +<% + locale = env.get("preferences").as(Preferences).locale + dark_mode = env.get("preferences").as(Preferences).dark_mode +%> <body class="<%= dark_mode.blank? ? "no" : dark_mode %>-theme"> <span style="display:none" id="dark_mode_pref"><%= env.get("preferences").as(Preferences).dark_mode %></span> @@ -33,11 +35,7 @@ <a href="/" class="index-link pure-menu-heading">Invidious</a> </div> <div class="pure-u-1 pure-u-md-12-24 searchbar"> - <form class="pure-form" action="/search" method="get"> - <fieldset> - <input type="search" style="width:100%" name="q" placeholder="<%= translate(locale, "search") %>" value="<%= env.get?("search").try {|x| HTML.escape(x.as(String)) } %>"> - </fieldset> - </form> + <% autofocus = false %><%= rendered "components/search_box" %> </div> <% end %> @@ -117,38 +115,45 @@ <footer> <div class="pure-g"> <div class="pure-u-1 pure-u-md-1-3"> - <a href="https://github.com/iv-org/invidious"> - <%= translate(locale, "Released under the AGPLv3 on Github.") %> - </a> - </div> - <div class="pure-u-1 pure-u-md-1-3"> - <i class="icon ion-ios-wallet"></i> - BTC: <a href="bitcoin:bc1qfhe7rq3lqzuayzjxzyt9waz9ytrs09kla3tsgr">bc1qfhe7rq3lqzuayzjxzyt9waz9ytrs09kla3tsgr</a> - </div> - <div class="pure-u-1 pure-u-md-1-3"> - <i class="icon ion-ios-wallet"></i> - XMR: <a href="monero:41nMCtek197boJtiUvGnTFYMatrLEpnpkQDmUECqx5Es2uX3sTKKWVhSL76suXsG3LXqkEJBrCZBgPTwJrDp1FrZJfycGPR">Click here</a> - </div> - <div class="pure-u-1 pure-u-md-1-3"> - <a href="https://github.com/iv-org/documentation">Documentation</a> + <span> + <i class="icon ion-logo-github"></i> + <% if CONFIG.modified_source_code_url %> + <a href="https://github.com/iv-org/invidious"><%= translate(locale, "footer_original_source_code") %></a> / + <a href="<%= CONFIG.modified_source_code_url %>"><%= translate(locale, "footer_modfied_source_code") %></a> + <% else %> + <a href="https://github.com/iv-org/invidious"><%= translate(locale, "footer_source_code") %></a> + <% end %> + </span> + <span> + <i class="icon ion-ios-paper"></i> + <a href="https://github.com/iv-org/documentation"><%= translate(locale, "footer_documentation") %></a> + </span> </div> + <div class="pure-u-1 pure-u-md-1-3"> - <i class="icon ion-logo-javascript"></i> - <a rel="jslicense" href="/licenses"> - <%= translate(locale, "View JavaScript license information.") %> - </a> - / - <i class="icon ion-ios-paper"></i> - <a href="/privacy"> - <%= translate(locale, "View privacy policy.") %> - </a> + <span> + <a href="https://github.com/iv-org/invidious/blob/master/LICENSE"><%= translate(locale, "Released under the AGPLv3 on Github.") %></a> + </span> + <span> + <i class="icon ion-logo-javascript"></i> + <a rel="jslicense" href="/licenses"><%= translate(locale, "View JavaScript license information.") %></a> + </span> + <span> + <i class="icon ion-ios-paper"></i> + <a href="/privacy"><%= translate(locale, "View privacy policy.") %></a> + </span> </div> + <div class="pure-u-1 pure-u-md-1-3"> - <i class="icon ion-logo-github"></i> - <%= translate(locale, "Current version: ") %> <%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %> @ <%= CURRENT_BRANCH %> + <span> + <i class="icon ion-ios-wallet"></i> + <a href="https://invidious.io/donate/"><%= translate(locale, "footer_donate_page") %></a> + </span> + <span><%= translate(locale, "Current version: ") %> <%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %> @ <%= CURRENT_BRANCH %></span> </div> </div> </footer> + </div> <div class="pure-u-1 pure-u-md-2-24"></div> </div> diff --git a/src/invidious/views/token_manager.ecr b/src/invidious/views/token_manager.ecr index e48aec2f..12e0e8c9 100644 --- a/src/invidious/views/token_manager.ecr +++ b/src/invidious/views/token_manager.ecr @@ -5,7 +5,7 @@ <div class="pure-g h-box"> <div class="pure-u-1-3"> <h3> - <%= translate(locale, "`x` tokens", %(<span id="count">#{tokens.size}</span>)) %> + <%= translate_count(locale, "tokens_count", tokens.size, NumberFormatting::HtmlSpan) %> </h3> </div> <div class="pure-u-1-3"></div> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 68e7eb80..00f5f8b7 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -11,7 +11,7 @@ <meta property="og:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>"> <meta property="og:title" content="<%= title %>"> <meta property="og:image" content="/vi/<%= video.id %>/maxres.jpg"> -<meta property="og:description" content="<%= video.short_description %>"> +<meta property="og:description" content="<%= HTML.escape(video.short_description) %>"> <meta property="og:type" content="video.other"> <meta property="og:video:url" content="<%= HOST_URL %>/embed/<%= video.id %>"> <meta property="og:video:secure_url" content="<%= HOST_URL %>/embed/<%= video.id %>"> @@ -22,7 +22,7 @@ <meta name="twitter:site" content="@omarroth1"> <meta name="twitter:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>"> <meta name="twitter:title" content="<%= title %>"> -<meta name="twitter:description" content="<%= video.short_description %>"> +<meta name="twitter:description" content="<%= HTML.escape(video.short_description) %>"> <meta name="twitter:image" content="<%= HOST_URL %>/vi/<%= video.id %>/maxres.jpg"> <meta name="twitter:player" content="<%= HOST_URL %>/embed/<%= video.id %>"> <meta name="twitter:player:width" content="1280"> @@ -103,7 +103,7 @@ we're going to need to do it here in order to allow for translations. </h3> <% elsif video.live_now %> <h3> - <%= video.premiere_timestamp.try { |t| translate(locale, "Started streaming `x` ago", recode_date((Time.utc - t).ago, locale)) } %> + <%= video.premiere_timestamp.try { |t| translate(locale, "videoinfo_started_streaming_x_ago", recode_date((Time.utc - t).ago, locale)) } %> </h3> <% end %> </div> @@ -112,14 +112,18 @@ we're going to need to do it here in order to allow for translations. <div class="pure-u-1 pure-u-lg-1-5"> <div class="h-box"> <span id="watch-on-youtube"> - <a href="https://www.youtube.com/watch?v=<%= video.id %>"><%= translate(locale, "Watch on YouTube") %></a> - (<a href="https://www.youtube.com/embed/<%= video.id %>"><%= translate(locale, "Embed") %></a>) + <a href="https://www.youtube.com/watch?v=<%= video.id %>"><%= translate(locale, "videoinfo_watch_on_youTube") %></a> + (<a href="https://www.youtube.com/embed/<%= video.id %>"><%= translate(locale, "videoinfo_youTube_embed_link") %></a>) </span> <p id="watch-on-another-invidious-instance"> + <% if env.get("preferences").as(Preferences).automatic_instance_redirect%> <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a> + <% else %> + <a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a> + <% end %> </p> <p id="embed-link"> - <a href="<%= embed_link %>"><%= translate(locale, "Embed Link") %></a> + <a href="<%= embed_link %>"><%= translate(locale, "videoinfo_invidious_embed_link") %></a> </p> <p id="annotations"> <% if params.annotations %> @@ -134,9 +138,9 @@ we're going to need to do it here in order to allow for translations. </p> <% if user %> - <% playlists = PG_DB.query_all("SELECT id,title FROM playlists WHERE author = $1 AND id LIKE 'IV%'", user.email, as: {String, String}) %> + <% playlists = Invidious::Database::Playlists.select_user_created_playlists(user.email) %> <% if !playlists.empty? %> - <form data-onsubmit="return_false" class="pure-form pure-form-stacked" action="/playlist_ajax" method="post"> + <form data-onsubmit="return_false" class="pure-form pure-form-stacked" action="/playlist_ajax" method="post" target="_blank"> <div class="pure-control-group"> <label for="playlist_id"><%= translate(locale, "Add to playlist: ") %></label> <select style="width:100%" name="playlist_id" id="playlist_id"> @@ -146,6 +150,9 @@ we're going to need to do it here in order to allow for translations. </select> </div> + <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> + <input type="hidden" name="action_add_video" value="1"> + <input type="hidden" name="video_id" value="<%= video.id %>"> <button data-onclick="add_playlist_video" data-id="<%= video.id %>" type="submit" class="pure-button pure-button-primary"> <b><%= translate(locale, "Add to playlist") %></b> </button> @@ -184,8 +191,8 @@ we're going to need to do it here in order to allow for translations. </option> <% end %> <% captions.each do |caption| %> - <option value='{"id":"<%= video.id %>","label":"<%= caption.name %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= caption.languageCode %>.vtt"}'> - <%= translate(locale, "Subtitles - `x` (.vtt)", caption.name) %> + <option value='{"id":"<%= video.id %>","label":"<%= caption.name %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= caption.language_code %>.vtt"}'> + <%= translate(locale, "download_subtitles", translate(locale, caption.name)) %> </option> <% end %> </select> @@ -250,14 +257,10 @@ we're going to need to do it here in order to allow for translations. <div id="description-box"> <!-- Description --> <% if video.description.size < 200 || params.extend_desc %> - <div id="descriptionWrapper"> - <%= video.description_html %> - </div> + <div id="descriptionWrapper"><%= video.description_html %></div> <% else %> <input id="descexpansionbutton" type="checkbox"/> - <div id="descriptionWrapper"> - <%= video.description_html %> - </div> + <div id="descriptionWrapper"><%= video.description_html %></div> <label for="descexpansionbutton"> <a></a> </label> @@ -291,7 +294,7 @@ we're going to need to do it here in order to allow for translations. <% if !video.related_videos.empty? %> <div <% if plid %>style="display:none"<% end %>> <div class="pure-control-group"> - <label for="continue"><%= translate(locale, "Play next by default: ") %></label> + <label for="continue"><%= translate(locale, "preferences_continue_label") %></label> <input name="continue" id="continue" type="checkbox" <% if params.continue %>checked<% end %>> </div> <hr> @@ -303,7 +306,7 @@ we're going to need to do it here in order to allow for translations. <a href="/watch?v=<%= rv["id"] %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> - <img class="thumbnail" src="/vi/<%= rv["id"] %>/mqdefault.jpg"> + <img loading="lazy" class="thumbnail" src="/vi/<%= rv["id"] %>/mqdefault.jpg"> <p class="length"><%= recode_length_seconds(rv["length_seconds"]?.try &.to_i? || 0) %></p> </div> <% end %> @@ -320,7 +323,7 @@ we're going to need to do it here in order to allow for translations. <div class="pure-u-10-24" style="text-align:right"> <% if views = rv["short_view_count_text"]?.try &.delete(", views watching") %> <% if !views.empty? %> - <b class="width:100%"><%= translate(locale, "`x` views", views) %></b> + <b class="width:100%"><%= translate_count(locale, "generic_views_count", views.to_i? || 0) %></b> <% end %> <% end %> </div> diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr new file mode 100644 index 00000000..3feb9233 --- /dev/null +++ b/src/invidious/yt_backend/connection_pool.cr @@ -0,0 +1,113 @@ +{% unless flag?(:disable_quic) %} + require "lsquic" + + alias HTTPClientType = QUIC::Client | HTTP::Client +{% else %} + alias HTTPClientType = HTTP::Client +{% end %} + +def add_yt_headers(request) + request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" + request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" + request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + request.headers["accept-language"] ||= "en-us,en;q=0.5" + return if request.resource.starts_with? "/sorry/index" + request.headers["x-youtube-client-name"] ||= "1" + request.headers["x-youtube-client-version"] ||= "2.20200609" + # Preserve original cookies and add new YT consent cookie for EU servers + request.headers["cookie"] = "#{request.headers["cookie"]?}; CONSENT=YES+" + if !CONFIG.cookies.empty? + request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" + end +end + +struct YoutubeConnectionPool + property! url : URI + property! capacity : Int32 + property! timeout : Float64 + property pool : DB::Pool(HTTPClientType) + + def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true) + @url = url + @pool = build_pool(use_quic) + end + + def client(region = nil, &block) + if region + conn = make_client(url, region) + response = yield conn + else + conn = pool.checkout + begin + response = yield conn + rescue ex + conn.close + {% unless flag?(:disable_quic) %} + conn = CONFIG.use_quic ? QUIC::Client.new(url) : HTTP::Client.new(url) + {% else %} + conn = HTTP::Client.new(url) + {% end %} + + conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET + conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC + conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" + response = yield conn + ensure + pool.release(conn) + end + end + + response + end + + private def build_pool(use_quic) + DB::Pool(HTTPClientType).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do + conn = nil # Declare + {% unless flag?(:disable_quic) %} + if use_quic + conn = QUIC::Client.new(url) + else + conn = HTTP::Client.new(url) + end + {% else %} + conn = HTTP::Client.new(url) + {% end %} + + conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET + conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC + conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" + conn + end + end +end + +def make_client(url : URI, region = nil) + # TODO: Migrate any applicable endpoints to QUIC + client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure) + client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC + client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" + client.read_timeout = 10.seconds + client.connect_timeout = 10.seconds + + if region + PROXY_LIST[region]?.try &.sample(40).each do |proxy| + begin + proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) + client.set_proxy(proxy) + break + rescue ex + end + end + end + + return client +end + +def make_client(url : URI, region = nil, &block) + client = make_client(url, region) + begin + yield client + ensure + client.close + end +end diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr new file mode 100644 index 00000000..66b3cdef --- /dev/null +++ b/src/invidious/yt_backend/extractors.cr @@ -0,0 +1,604 @@ +# This file contains helper methods to parse the Youtube API json data into +# neat little packages we can use + +# Tuple of Parsers/Extractors so we can easily cycle through them. +private ITEM_CONTAINER_EXTRACTOR = { + Extractors::YouTubeTabs, + Extractors::SearchResults, + Extractors::Continuation, +} + +private ITEM_PARSERS = { + Parsers::VideoRendererParser, + Parsers::ChannelRendererParser, + Parsers::GridPlaylistRendererParser, + Parsers::PlaylistRendererParser, + Parsers::CategoryRendererParser, +} + +record AuthorFallback, name : String, id : String + +# Namespace for logic relating to parsing InnerTube data into various datastructs. +# +# Each of the parsers in this namespace are accessed through the #process() method +# which validates the given data as applicable to itself. If it is applicable the given +# data is passed to the private `#parse()` method which returns a datastruct of the given +# type. Otherwise, nil is returned. +private module Parsers + # Parses a InnerTube videoRenderer into a SearchVideo. Returns nil when the given object isn't a videoRenderer + # + # A videoRenderer renders a video to click on within the YouTube and Invidious UI. It is **not** + # the watchable video itself. + # + # See specs for example. + # + # `videoRenderer`s can be found almost everywhere on YouTube. In categories, search results, channels, etc. + # + module VideoRendererParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = (item["videoRenderer"]? || item["gridVideoRenderer"]?) + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + video_id = item_contents["videoId"].as_s + title = extract_text(item_contents["title"]?) || "" + + # Extract author information + if author_info = item_contents.dig?("ownerText", "runs", 0) + author = author_info["text"].as_s + author_id = HelperExtractors.get_browse_id(author_info) + elsif author_info = item_contents.dig?("shortBylineText", "runs", 0) + author = author_info["text"].as_s + author_id = HelperExtractors.get_browse_id(author_info) + else + author = author_fallback.name + author_id = author_fallback.id + end + + # For live videos (and possibly recently premiered videos) there is no published information. + # Instead, in its place is the amount of people currently watching. This behavior should be replicated + # on Invidious once all features of livestreams are supported. On an unrelated note, defaulting to the current + # time for publishing isn't a good idea. + published = item_contents.dig?("publishedTimeText", "simpleText").try { |t| decode_date(t.as_s) } || Time.local + + # Typically views are stored under a "simpleText" in the "viewCountText". However, for + # livestreams and premiered it is stored under a "runs" array: [{"text":123}, {"text": "watching"}] + # When view count is disabled the "viewCountText" is not present on InnerTube data. + # TODO change default value to nil and typical encoding type to tuple storing type (watchers, views, etc) + # and count + view_count = item_contents.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64 + description_html = item_contents["descriptionSnippet"]?.try { |t| parse_content(t) } || "" + + # The length information generally exist in "lengthText". However, the info can sometimes + # be retrieved from "thumbnailOverlays" (e.g when the video is a "shorts" one). + if length_container = item_contents["lengthText"]? + length_seconds = decode_length_seconds(length_container["simpleText"].as_s) + elsif length_container = item_contents["thumbnailOverlays"]?.try &.as_a.find(&.["thumbnailOverlayTimeStatusRenderer"]?) + # This needs to only go down the `simpleText` path (if possible). If more situations came up that requires + # a specific pathway then we should add an argument to extract_text that'll make this possible + length_text = length_container.dig?("thumbnailOverlayTimeStatusRenderer", "text", "simpleText") + + if length_text + length_text = length_text.as_s + + if length_text == "SHORTS" + # Approximate length to one minute, as "shorts" generally don't exceed that length. + # TODO: Add some sort of metadata for the type of video (normal, live, premiere, shorts) + length_seconds = 60_i32 + else + length_seconds = decode_length_seconds(length_text) + end + else + length_seconds = 0 + end + else + length_seconds = 0 + end + + live_now = false + paid = false + premium = false + + premiere_timestamp = item_contents.dig?("upcomingEventData", "startTime").try { |t| Time.unix(t.as_s.to_i64) } + + item_contents["badges"]?.try &.as_a.each do |badge| + b = badge["metadataBadgeRenderer"] + case b["label"].as_s + when "LIVE NOW" + live_now = true + when "New", "4K", "CC" + # TODO + when "Premium" + # TODO: Potentially available as item_contents["topStandaloneBadge"]["metadataBadgeRenderer"] + premium = true + else nil # Ignore + end + end + + SearchVideo.new({ + title: title, + id: video_id, + author: author, + ucid: author_id, + published: published, + views: view_count, + description_html: description_html, + length_seconds: length_seconds, + live_now: live_now, + premium: premium, + premiere_timestamp: premiere_timestamp, + }) + end + + def self.parser_name + return {{@type.name}} + end + end + + # Parses a InnerTube channelRenderer into a SearchChannel. Returns nil when the given object isn't a channelRenderer + # + # A channelRenderer renders a channel to click on within the YouTube and Invidious UI. It is **not** + # the channel page itself. + # + # See specs for example. + # + # `channelRenderer`s can be found almost everywhere on YouTube. In categories, search results, channels, etc. + # + module ChannelRendererParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = (item["channelRenderer"]? || item["gridChannelRenderer"]?) + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + author = extract_text(item_contents["title"]) || author_fallback.name + author_id = item_contents["channelId"]?.try &.as_s || author_fallback.id + + author_thumbnail = HelperExtractors.get_thumbnails(item_contents) + # When public subscriber count is disabled, the subscriberCountText isn't sent by InnerTube. + # Always simpleText + # TODO change default value to nil + subscriber_count = item_contents.dig?("subscriberCountText", "simpleText") + .try { |s| short_text_to_number(s.as_s.split(" ")[0]) } || 0 + + # Auto-generated channels doesn't have videoCountText + # Taken from: https://github.com/iv-org/invidious/pull/2228#discussion_r717620922 + auto_generated = item_contents["videoCountText"]?.nil? + + video_count = HelperExtractors.get_video_count(item_contents) + description_html = item_contents["descriptionSnippet"]?.try { |t| parse_content(t) } || "" + + SearchChannel.new({ + author: author, + ucid: author_id, + author_thumbnail: author_thumbnail, + subscriber_count: subscriber_count, + video_count: video_count, + description_html: description_html, + auto_generated: auto_generated, + }) + end + + def self.parser_name + return {{@type.name}} + end + end + + # Parses a InnerTube gridPlaylistRenderer into a SearchPlaylist. Returns nil when the given object isn't a gridPlaylistRenderer + # + # A gridPlaylistRenderer renders a playlist, that is located in a grid, to click on within the YouTube and Invidious UI. + # It is **not** the playlist itself. + # + # See specs for example. + # + # `gridPlaylistRenderer`s can be found on the playlist-tabs of channels and expanded categories. + # + module GridPlaylistRendererParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["gridPlaylistRenderer"]? + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + title = extract_text(item_contents["title"]) || "" + plid = item_contents["playlistId"]?.try &.as_s || "" + + video_count = HelperExtractors.get_video_count(item_contents) + playlist_thumbnail = HelperExtractors.get_thumbnails(item_contents) + + SearchPlaylist.new({ + title: title, + id: plid, + author: author_fallback.name, + ucid: author_fallback.id, + video_count: video_count, + videos: [] of SearchPlaylistVideo, + thumbnail: playlist_thumbnail, + }) + end + + def self.parser_name + return {{@type.name}} + end + end + + # Parses a InnerTube playlistRenderer into a SearchPlaylist. Returns nil when the given object isn't a playlistRenderer + # + # A playlistRenderer renders a playlist to click on within the YouTube and Invidious UI. It is **not** the playlist itself. + # + # See specs for example. + # + # `playlistRenderer`s can be found almost everywhere on YouTube. In categories, search results, recommended, etc. + # + module PlaylistRendererParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["playlistRenderer"]? + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + title = item_contents["title"]["simpleText"]?.try &.as_s || "" + plid = item_contents["playlistId"]?.try &.as_s || "" + + video_count = HelperExtractors.get_video_count(item_contents) + playlist_thumbnail = HelperExtractors.get_thumbnails_plural(item_contents) + + author_info = item_contents.dig?("shortBylineText", "runs", 0) + author = author_info.try &.["text"].as_s || author_fallback.name + author_id = author_info.try { |x| HelperExtractors.get_browse_id(x) } || author_fallback.id + + videos = item_contents["videos"]?.try &.as_a.map do |v| + v = v["childVideoRenderer"] + v_title = v.dig?("title", "simpleText").try &.as_s || "" + v_id = v["videoId"]?.try &.as_s || "" + v_length_seconds = v.dig?("lengthText", "simpleText").try { |t| decode_length_seconds(t.as_s) } || 0 + SearchPlaylistVideo.new({ + title: v_title, + id: v_id, + length_seconds: v_length_seconds, + }) + end || [] of SearchPlaylistVideo + + # TODO: item_contents["publishedTimeText"]? + + SearchPlaylist.new({ + title: title, + id: plid, + author: author, + ucid: author_id, + video_count: video_count, + videos: videos, + thumbnail: playlist_thumbnail, + }) + end + + def self.parser_name + return {{@type.name}} + end + end + + # Parses a InnerTube shelfRenderer into a Category. Returns nil when the given object isn't a shelfRenderer + # + # A shelfRenderer renders divided sections on YouTube. IE "People also watched" in search results and + # the various organizational sections in the channel home page. A separate one (richShelfRenderer) is used + # for YouTube home. A shelfRenderer can also sometimes be expanded to show more content within it. + # + # See specs for example. + # + # `shelfRenderer`s can be found almost everywhere on YouTube. In categories, search results, channels, etc. + # + module CategoryRendererParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["shelfRenderer"]? + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + title = extract_text(item_contents["title"]?) || "" + url = item_contents.dig?("endpoint", "commandMetadata", "webCommandMetadata", "url") + .try &.as_s + + # Sometimes a category can have badges. + badges = [] of Tuple(String, String) # (Badge style, label) + item_contents["badges"]?.try &.as_a.each do |badge| + badge = badge["metadataBadgeRenderer"] + badges << {badge["style"].as_s, badge["label"].as_s} + end + + # Category description + description_html = item_contents["subtitle"]?.try { |desc| parse_content(desc) } || "" + + # Content parsing + contents = [] of SearchItem + + # InnerTube recognizes some "special" categories, which are organized differently. + if special_category_container = item_contents["content"]? + if content_container = special_category_container["horizontalListRenderer"]? + elsif content_container = special_category_container["expandedShelfContentsRenderer"]? + elsif content_container = special_category_container["verticalListRenderer"]? + else + # Anything else, such as `horizontalMovieListRenderer` is currently unsupported. + return + end + else + # "Normal" category. + content_container = item_contents["contents"] + end + + raw_contents = content_container["items"]?.try &.as_a + if !raw_contents.nil? + raw_contents.each do |item| + result = extract_item(item) + if !result.nil? + contents << result + end + end + end + + Category.new({ + title: title, + contents: contents, + description_html: description_html, + url: url, + badges: badges, + }) + end + + def self.parser_name + return {{@type.name}} + end + end +end + +# The following are the extractors for extracting an array of items from +# the internal Youtube API's JSON response. The result is then packaged into +# a structure we can more easily use via the parsers above. Their internals are +# identical to the item parsers. + +# Namespace for logic relating to extracting InnerTube's initial response to items we can parse. +# +# Each of the extractors in this namespace are accessed through the #process() method +# which validates the given data as applicable to itself. If it is applicable the given +# data is passed to the private `#extract()` method which returns an array of +# parsable items. Otherwise, nil is returned. +# +# NOTE perhaps the result from here should be abstracted into a struct in order to +# get additional metadata regarding the container of the item(s). +private module Extractors + # Extracts items from the selected YouTube tab. + # + # YouTube tabs are typically stored under "twoColumnBrowseResultsRenderer" + # and is structured like this: + # + # "twoColumnBrowseResultsRenderer": { + # {"tabs": [ + # {"tabRenderer": { + # "endpoint": {...} + # "title": "Playlists", + # "selected": true, + # "content": {...}, + # ... + # }} + # ]} + # }] + # + module YouTubeTabs + def self.process(initial_data : Hash(String, JSON::Any)) + if target = initial_data["twoColumnBrowseResultsRenderer"]? + self.extract(target) + end + end + + private def self.extract(target) + raw_items = [] of JSON::Any + content = extract_selected_tab(target["tabs"])["content"] + + content["sectionListRenderer"]["contents"].as_a.each do |renderer_container| + renderer_container_contents = renderer_container["itemSectionRenderer"]["contents"][0] + + # Category extraction + if items_container = renderer_container_contents["shelfRenderer"]? + raw_items << renderer_container_contents + next + elsif items_container = renderer_container_contents["gridRenderer"]? + else + items_container = renderer_container_contents + end + + items_container["items"]?.try &.as_a.each do |item| + raw_items << item + end + end + + return raw_items + end + + def self.extractor_name + return {{@type.name}} + end + end + + # Extracts items from the InnerTube response for search results + # + # Search results are typically stored under "twoColumnSearchResultsRenderer" + # and is structured like this: + # + # "twoColumnSearchResultsRenderer": { + # {"primaryContents": { + # {"sectionListRenderer": { + # "contents": [...], + # ..., + # "subMenu": {...}, + # "hideBottomSeparator": true, + # "targetId": "search-feed" + # }} + # }} + # } + # + module SearchResults + def self.process(initial_data : Hash(String, JSON::Any)) + if target = initial_data["twoColumnSearchResultsRenderer"]? + self.extract(target) + end + end + + private def self.extract(target) + raw_items = [] of Array(JSON::Any) + + target.dig("primaryContents", "sectionListRenderer", "contents").as_a.each do |node| + if node = node["itemSectionRenderer"]? + raw_items << node["contents"].as_a + end + end + + return raw_items.flatten + end + + def self.extractor_name + return {{@type.name}} + end + end + + # Extracts continuation items from a InnerTube response + # + # Continuation items (on YouTube) are items which are appended to the + # end of the page for continuous scrolling. As such, in many cases, + # the items are lacking information such as author or category title, + # since the original results has already rendered them on the top of the page. + # + # The way they are structured is too varied to be accurately written down here. + # However, they all eventually lead to an array of parsable items after traversing + # through the JSON structure. + module Continuation + def self.process(initial_data : Hash(String, JSON::Any)) + if target = initial_data["continuationContents"]? + self.extract(target) + elsif target = initial_data["appendContinuationItemsAction"]? + self.extract(target) + end + end + + private def self.extract(target) + raw_items = [] of JSON::Any + if content = target["gridContinuation"]? + raw_items = content["items"].as_a + elsif content = target["continuationItems"]? + raw_items = content.as_a + end + + return raw_items + end + + def self.extractor_name + return {{@type.name}} + end + end +end + +# Helper methods to aid in the parsing of InnerTube to data structs. +# +# Mostly used to extract out repeated structures to deal with code +# repetition. +private module HelperExtractors + # Retrieves the amount of videos present within the given InnerTube data. + # + # Returns a 0 when it's unable to do so + def self.get_video_count(container : JSON::Any) : Int32 + if box = container["videoCountText"]? + return extract_text(box).try &.gsub(/\D/, "").to_i || 0 + elsif box = container["videoCount"]? + return box.as_s.to_i + else + return 0 + end + end + + # Retrieve lowest quality thumbnail from InnerTube data + # + # TODO allow configuration of image quality (-1 is highest) + # + # Raises when it's unable to parse from the given JSON data. + def self.get_thumbnails(container : JSON::Any) : String + return container.dig("thumbnail", "thumbnails", 0, "url").as_s + end + + # ditto + # + # YouTube sometimes sends the thumbnail as: + # {"thumbnails": [{"thumbnails": [{"url": "example.com"}, ...]}]} + def self.get_thumbnails_plural(container : JSON::Any) : String + return container.dig("thumbnails", 0, "thumbnails", 0, "url").as_s + end + + # Retrieves the ID required for querying the InnerTube browse endpoint. + # Raises when it's unable to do so + def self.get_browse_id(container) + return container.dig("navigationEndpoint", "browseEndpoint", "browseId").as_s + end +end + +# Parses an item from Youtube's JSON response into a more usable structure. +# The end result can either be a SearchVideo, SearchPlaylist or SearchChannel. +def extract_item(item : JSON::Any, author_fallback : String? = "", + author_id_fallback : String? = "") + # We "allow" nil values but secretly use empty strings instead. This is to save us the + # hassle of modifying every author_fallback and author_id_fallback arg usage + # which is more often than not nil. + author_fallback = AuthorFallback.new(author_fallback || "", author_id_fallback || "") + + # Cycles through all of the item parsers and attempt to parse the raw YT JSON data. + # Each parser automatically validates the data given to see if the data is + # applicable to itself. If not nil is returned and the next parser is attemped. + ITEM_PARSERS.each do |parser| + LOGGER.trace("extract_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)") + + if result = parser.process(item, author_fallback) + LOGGER.debug("extract_item: Successfully parsed via #{parser.parser_name}") + + return result + else + LOGGER.trace("extract_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...") + end + end +end + +# Parses multiple items from YouTube's initial JSON response into a more usable structure. +# The end result is an array of SearchItem. +def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, + author_id_fallback : String? = nil) : Array(SearchItem) + items = [] of SearchItem + + if unpackaged_data = initial_data["contents"]?.try &.as_h + elsif unpackaged_data = initial_data["response"]?.try &.as_h + elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 0).try &.as_h + else + unpackaged_data = initial_data + end + + # This is identical to the parser cycling of extract_item(). + ITEM_CONTAINER_EXTRACTOR.each do |extractor| + LOGGER.trace("extract_items: Attempting to extract item container using \"#{extractor.extractor_name}\" (cycling...)") + + if container = extractor.process(unpackaged_data) + LOGGER.debug("extract_items: Successfully unpacked container with \"#{extractor.extractor_name}\"") + # Extract items in container + container.each do |item| + if parsed_result = extract_item(item, author_fallback, author_id_fallback) + items << parsed_result + end + end + + break + else + LOGGER.trace("extract_items: Extractor \"#{extractor.extractor_name}\" does not apply. Cycling to the next one...") + end + end + + return items +end diff --git a/src/invidious/yt_backend/extractors_utils.cr b/src/invidious/yt_backend/extractors_utils.cr new file mode 100644 index 00000000..add5f488 --- /dev/null +++ b/src/invidious/yt_backend/extractors_utils.cr @@ -0,0 +1,67 @@ +# Extracts text from InnerTube response +# +# InnerTube can package text in three different formats +# "runs": [ +# {"text": "something"}, +# {"text": "cont"}, +# ... +# ] +# +# "SimpleText": "something" +# +# Or sometimes just none at all as with the data returned from +# category continuations. +# +# In order to facilitate calling this function with `#[]?`: +# A nil will be accepted. Of course, since nil cannot be parsed, +# another nil will be returned. +def extract_text(item : JSON::Any?) : String? + if item.nil? + return nil + end + + if text_container = item["simpleText"]? + return text_container.as_s + elsif text_container = item["runs"]? + return text_container.as_a.map(&.["text"].as_s).join("") + else + nil + end +end + +def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) + extracted = extract_items(initial_data, author_fallback, author_id_fallback) + + target = [] of SearchItem + extracted.each do |i| + if i.is_a?(Category) + i.contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video } + else + target << i + end + end + return target.select(SearchVideo).map(&.as(SearchVideo)) +end + +def extract_selected_tab(tabs) + # Extract the selected tab from the array of tabs Youtube returns + return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"].as_bool)[0]["tabRenderer"] +end + +def fetch_continuation_token(items : Array(JSON::Any)) + # Fetches the continuation token from an array of items + return items.last["continuationItemRenderer"]? + .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s +end + +def fetch_continuation_token(initial_data : Hash(String, JSON::Any)) + # Fetches the continuation token from initial data + if initial_data["onResponseReceivedActions"]? + continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"] + else + tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]) + continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"] + end + + return fetch_continuation_token(continuation_items.as_a) +end diff --git a/src/invidious/helpers/proxy.cr b/src/invidious/yt_backend/proxy.cr index 3418d887..2d0fd4ba 100644 --- a/src/invidious/helpers/proxy.cr +++ b/src/invidious/yt_backend/proxy.cr @@ -236,7 +236,7 @@ def get_spys_proxies(country_code = "US") proxies << {ip: ip, port: port, score: score} end - proxies = proxies.sort_by { |proxy| proxy[:score] }.reverse + proxies = proxies.sort_by!(&.[:score]).reverse! return proxies end @@ -256,7 +256,7 @@ def decrypt_port(p, x) p = p.gsub(/\b\w+\b/, x) p = p.split(";") - p = p.map { |item| item.split("=") } + p = p.map(&.split("=")) mapping = {} of String => Int32 p.each do |item| diff --git a/src/invidious/helpers/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index b3815f6a..85239e72 100644 --- a/src/invidious/helpers/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -405,28 +405,32 @@ module YoutubeAPI headers = HTTP::Headers{ "Content-Type" => "application/json; charset=UTF-8", - "Accept-Encoding" => "gzip", + "Accept-Encoding" => "gzip, deflate", } # Logging LOGGER.debug("YoutubeAPI: Using endpoint: \"#{endpoint}\"") - LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config.to_s}") - LOGGER.trace("YoutubeAPI: POST data: #{data.to_s}") + LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}") + LOGGER.trace("YoutubeAPI: POST data: #{data}") # Send the POST request - if client_config.proxy_region - response = YT_POOL.client( - client_config.proxy_region, + if {{ !flag?(:disable_quic) }} && CONFIG.use_quic + # Using QUIC client + response = YT_POOL.client(client_config.proxy_region, &.post(url, headers: headers, body: data.to_json) ) + body = response.body else - response = YT_POOL.client &.post( - url, headers: headers, body: data.to_json - ) + # Using HTTP client + body = YT_POOL.client(client_config.proxy_region) do |client| + client.post(url, headers: headers, body: data.to_json) do |response| + self._decompress(response.body_io, response.headers["Content-Encoding"]?) + end + end end # Convert result to Hash - initial_data = JSON.parse(response.body).as_h + initial_data = JSON.parse(body).as_h # Error handling if initial_data.has_key?("error") @@ -436,7 +440,7 @@ module YoutubeAPI # Logging LOGGER.error("YoutubeAPI: Got error #{code} when requesting #{endpoint}") LOGGER.error("YoutubeAPI: #{message}") - LOGGER.info("YoutubeAPI: POST data was: #{data.to_s}") + LOGGER.info("YoutubeAPI: POST data was: #{data}") raise InfoException.new("Could not extract JSON. Youtube API returned \ error #{code} with message:<br>\"#{message}\"") @@ -444,4 +448,35 @@ module YoutubeAPI return initial_data end + + #################################################################### + # _decompress(body_io, headers) + # + # Internal function that reads the Content-Encoding headers and + # decompresses the content accordingly. + # + # We decompress the body ourselves (when using HTTP::Client) because + # the auto-decompress feature is broken in the Crystal stdlib. + # + # Read more: + # - https://github.com/iv-org/invidious/issues/2612 + # - https://github.com/crystal-lang/crystal/issues/11354 + # + def _decompress(body_io : IO, encodings : String?) : String + if encodings + # Multiple encodings can be combined, and are listed in the order + # in which they were applied. E.g: "deflate, gzip" means that the + # content must be first "gunzipped", then "defated". + encodings.split(',').reverse.each do |enc| + case enc.strip(' ') + when "gzip" + body_io = Compress::Gzip::Reader.new(body_io, sync_close: true) + when "deflate" + body_io = Compress::Deflate::Reader.new(body_io, sync_close: true) + end + end + end + + return body_io.gets_to_end + end end # End of module |