diff options
Diffstat (limited to 'src')
78 files changed, 6538 insertions, 4939 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index deb24ac3..9e67e216 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -17,6 +17,7 @@ require "digest/md5" require "file_utils" require "kemal" +require "athena-negotiation" require "openssl/hmac" require "option_parser" require "pg" @@ -27,32 +28,21 @@ require "compress/zip" require "protodec/utils" require "./invidious/helpers/*" require "./invidious/*" +require "./invidious/channels/*" require "./invidious/routes/**" require "./invidious/jobs/**" -ENV_CONFIG_NAME = "INVIDIOUS_CONFIG" +CONFIG = Config.load +HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32) -CONFIG_STR = ENV.has_key?(ENV_CONFIG_NAME) ? ENV.fetch(ENV_CONFIG_NAME) : File.read("config/config.yml") -CONFIG = Config.from_yaml(CONFIG_STR) -HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32) - -PG_URL = URI.new( - scheme: "postgres", - user: CONFIG.db.user, - password: CONFIG.db.password, - host: CONFIG.db.host, - port: CONFIG.db.port, - path: CONFIG.db.dbname, -) - -PG_DB = DB.open PG_URL +PG_DB = DB.open CONFIG.database_url ARCHIVE_URL = URI.parse("https://archive.org") LOGIN_URL = URI.parse("https://accounts.google.com") PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com") REDDIT_URL = URI.parse("https://www.reddit.com") TEXTCAPTCHA_URL = URI.parse("https://textcaptcha.com") YT_URL = URI.parse("https://www.youtube.com") -HOST_URL = make_host_url(CONFIG, Kemal.config) +HOST_URL = make_host_url(Kemal.config) CHARS_SAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"} @@ -77,34 +67,7 @@ SOFTWARE = { "branch" => "#{CURRENT_BRANCH}", } -LOCALES = { - "ar" => load_locale("ar"), - "de" => load_locale("de"), - "el" => load_locale("el"), - "en-US" => load_locale("en-US"), - "eo" => load_locale("eo"), - "es" => load_locale("es"), - "eu" => load_locale("eu"), - "fr" => load_locale("fr"), - "hu" => load_locale("hu-HU"), - "is" => load_locale("is"), - "it" => load_locale("it"), - "ja" => load_locale("ja"), - "nb-NO" => load_locale("nb-NO"), - "nl" => load_locale("nl"), - "pl" => load_locale("pl"), - "pt-BR" => load_locale("pt-BR"), - "pt-PT" => load_locale("pt-PT"), - "ro" => load_locale("ro"), - "ru" => load_locale("ru"), - "sv" => load_locale("sv-SE"), - "tr" => load_locale("tr"), - "uk" => load_locale("uk"), - "zh-CN" => load_locale("zh-CN"), - "zh-TW" => load_locale("zh-TW"), -} - -YT_POOL = QUICPool.new(YT_URL, capacity: CONFIG.pool_size, timeout: 2.0) +YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size, use_quic: CONFIG.use_quic) # CLI Kemal.config.extra_options do |parser| @@ -145,8 +108,6 @@ end OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mode: "a") LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) -config = CONFIG - # Check table integrity if CONFIG.check_tables check_enum(PG_DB, "privacy", PlaylistPrivacy) @@ -167,32 +128,33 @@ end # Start jobs -Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB, config) -Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB, config) +if CONFIG.channel_threads > 0 + Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB) +end + +if CONFIG.feed_threads > 0 + Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB) +end DECRYPT_FUNCTION = DecryptFunction.new(CONFIG.decrypt_polling) -if config.decrypt_polling +if CONFIG.decrypt_polling Invidious::Jobs.register Invidious::Jobs::UpdateDecryptFunctionJob.new end -if config.statistics_enabled - Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, config, SOFTWARE) +if CONFIG.statistics_enabled + Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, SOFTWARE) end -if (config.use_pubsub_feeds.is_a?(Bool) && config.use_pubsub_feeds.as(Bool)) || (config.use_pubsub_feeds.is_a?(Int32) && config.use_pubsub_feeds.as(Int32) > 0) - Invidious::Jobs.register Invidious::Jobs::SubscribeToFeedsJob.new(PG_DB, config, HMAC_KEY) +if (CONFIG.use_pubsub_feeds.is_a?(Bool) && CONFIG.use_pubsub_feeds.as(Bool)) || (CONFIG.use_pubsub_feeds.is_a?(Int32) && CONFIG.use_pubsub_feeds.as(Int32) > 0) + Invidious::Jobs.register Invidious::Jobs::SubscribeToFeedsJob.new(PG_DB, HMAC_KEY) end -if config.popular_enabled +if CONFIG.popular_enabled Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB) end -if config.captcha_key - Invidious::Jobs.register Invidious::Jobs::BypassCaptchaJob.new(config) -end - connection_channel = Channel({Bool, Channel(PQ::Notification)}).new(32) -Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(connection_channel, PG_URL) +Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(connection_channel, CONFIG.database_url) Invidious::Jobs.start_all @@ -201,25 +163,64 @@ def popular_videos end before_all do |env| - preferences = begin - Preferences.from_json(env.request.cookies["PREFS"]?.try &.value || "{}") + preferences = Preferences.from_json("{}") + + begin + if prefs_cookie = env.request.cookies["PREFS"]? + preferences = Preferences.from_json(URI.decode_www_form(prefs_cookie.value)) + else + if language_header = env.request.headers["Accept-Language"]? + if language = ANG.language_negotiator.best(language_header, LOCALES.keys) + preferences.locale = language.header + end + end + end rescue - Preferences.from_json("{}") + preferences = Preferences.from_json("{}") end env.set "preferences", preferences env.response.headers["X-XSS-Protection"] = "1; mode=block" env.response.headers["X-Content-Type-Options"] = "nosniff" - extra_media_csp = "" + + # Allow media resources to be loaded from google servers + # TODO: check if *.youtube.com can be removed if CONFIG.disabled?("local") || !preferences.local - extra_media_csp += " https://*.googlevideo.com:443" - extra_media_csp += " https://*.youtube.com:443" + extra_media_csp = " https://*.googlevideo.com:443 https://*.youtube.com:443" + else + extra_media_csp = "" end - # TODO: Remove style-src's 'unsafe-inline', requires to remove all inline styles (<style> [..] </style>, style=" [..] ") - env.response.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' data:; connect-src 'self'; manifest-src 'self'; media-src 'self' blob:#{extra_media_csp}" + + # Only allow the pages at /embed/* to be embedded + if env.request.resource.starts_with?("/embed") + frame_ancestors = "'self' http: https:" + else + frame_ancestors = "'none'" + end + + # TODO: Remove style-src's 'unsafe-inline', requires to remove all + # inline styles (<style> [..] </style>, style=" [..] ") + env.response.headers["Content-Security-Policy"] = { + "default-src 'none'", + "script-src 'self'", + "style-src 'self' 'unsafe-inline'", + "img-src 'self' data:", + "font-src 'self' data:", + "connect-src 'self'", + "manifest-src 'self'", + "media-src 'self' blob:" + extra_media_csp, + "child-src 'self' blob:", + "frame-src 'self'", + "frame-ancestors " + frame_ancestors, + }.join("; ") + env.response.headers["Referrer-Policy"] = "same-origin" - if (Kemal.config.ssl || config.https_only) && config.hsts + # Ask the chrom*-based browsers to disable FLoC + # See: https://blog.runcloud.io/google-floc/ + env.response.headers["Permissions-Policy"] = "interest-cohort=()" + + if (Kemal.config.ssl || CONFIG.https_only) && CONFIG.hsts env.response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains; preload" end @@ -295,6 +296,7 @@ before_all do |env| preferences.dark_mode = dark_mode preferences.thin_mode = thin_mode preferences.locale = locale + env.set "preferences", preferences current_page = env.request.path if env.request.query @@ -310,33 +312,89 @@ before_all do |env| env.set "current_page", URI.encode_www_form(current_page) end -Invidious::Routing.get "/", Invidious::Routes::Home -Invidious::Routing.get "/privacy", Invidious::Routes::Privacy -Invidious::Routing.get "/licenses", Invidious::Routes::Licenses -Invidious::Routing.get "/watch", Invidious::Routes::Watch -Invidious::Routing.get "/embed/", Invidious::Routes::Embed::Index -Invidious::Routing.get "/embed/:id", Invidious::Routes::Embed::Show -Invidious::Routing.get "/view_all_playlists", Invidious::Routes::Playlists, :index -Invidious::Routing.get "/create_playlist", Invidious::Routes::Playlists, :new -Invidious::Routing.post "/create_playlist", Invidious::Routes::Playlists, :create -Invidious::Routing.get "/subscribe_playlist", Invidious::Routes::Playlists, :subscribe -Invidious::Routing.get "/delete_playlist", Invidious::Routes::Playlists, :delete_page -Invidious::Routing.post "/delete_playlist", Invidious::Routes::Playlists, :delete -Invidious::Routing.get "/edit_playlist", Invidious::Routes::Playlists, :edit -Invidious::Routing.post "/edit_playlist", Invidious::Routes::Playlists, :update -Invidious::Routing.get "/add_playlist_items", Invidious::Routes::Playlists, :add_playlist_items_page -Invidious::Routing.post "/playlist_ajax", Invidious::Routes::Playlists, :playlist_ajax -Invidious::Routing.get "/playlist", Invidious::Routes::Playlists, :show -Invidious::Routing.get "/mix", Invidious::Routes::Playlists, :mix -Invidious::Routing.get "/opensearch.xml", Invidious::Routes::Search, :opensearch -Invidious::Routing.get "/results", Invidious::Routes::Search, :results -Invidious::Routing.get "/search", Invidious::Routes::Search, :search -Invidious::Routing.get "/login", Invidious::Routes::Login, :login_page -Invidious::Routing.post "/login", Invidious::Routes::Login, :login -Invidious::Routing.post "/signout", Invidious::Routes::Login, :signout -Invidious::Routing.get "/preferences", Invidious::Routes::UserPreferences, :show -Invidious::Routing.post "/preferences", Invidious::Routes::UserPreferences, :update -Invidious::Routing.get "/toggle_theme", Invidious::Routes::UserPreferences, :toggle_theme +{% unless flag?(:api_only) %} + Invidious::Routing.get "/", Invidious::Routes::Misc, :home + Invidious::Routing.get "/privacy", Invidious::Routes::Misc, :privacy + Invidious::Routing.get "/licenses", Invidious::Routes::Misc, :licenses + + Invidious::Routing.get "/channel/:ucid", Invidious::Routes::Channels, :home + Invidious::Routing.get "/channel/:ucid/home", Invidious::Routes::Channels, :home + Invidious::Routing.get "/channel/:ucid/videos", Invidious::Routes::Channels, :videos + Invidious::Routing.get "/channel/:ucid/playlists", Invidious::Routes::Channels, :playlists + Invidious::Routing.get "/channel/:ucid/community", Invidious::Routes::Channels, :community + Invidious::Routing.get "/channel/:ucid/about", Invidious::Routes::Channels, :about + + ["", "/videos", "/playlists", "/community", "/about"].each do |path| + # /c/LinusTechTips + Invidious::Routing.get "/c/:user#{path}", Invidious::Routes::Channels, :brand_redirect + # /user/linustechtips | Not always the same as /c/ + Invidious::Routing.get "/user/:user#{path}", Invidious::Routes::Channels, :brand_redirect + # /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow + Invidious::Routing.get "/attribution_link#{path}", Invidious::Routes::Channels, :brand_redirect + # /profile?user=linustechtips + Invidious::Routing.get "/profile/#{path}", Invidious::Routes::Channels, :profile + end + + Invidious::Routing.get "/watch", Invidious::Routes::Watch, :handle + Invidious::Routing.get "/watch/:id", Invidious::Routes::Watch, :redirect + Invidious::Routing.get "/shorts/:id", Invidious::Routes::Watch, :redirect + Invidious::Routing.get "/w/:id", Invidious::Routes::Watch, :redirect + Invidious::Routing.get "/v/:id", Invidious::Routes::Watch, :redirect + Invidious::Routing.get "/e/:id", Invidious::Routes::Watch, :redirect + Invidious::Routing.get "/redirect", Invidious::Routes::Misc, :cross_instance_redirect + + Invidious::Routing.get "/embed/", Invidious::Routes::Embed, :redirect + Invidious::Routing.get "/embed/:id", Invidious::Routes::Embed, :show + + Invidious::Routing.get "/create_playlist", Invidious::Routes::Playlists, :new + Invidious::Routing.post "/create_playlist", Invidious::Routes::Playlists, :create + Invidious::Routing.get "/subscribe_playlist", Invidious::Routes::Playlists, :subscribe + Invidious::Routing.get "/delete_playlist", Invidious::Routes::Playlists, :delete_page + Invidious::Routing.post "/delete_playlist", Invidious::Routes::Playlists, :delete + Invidious::Routing.get "/edit_playlist", Invidious::Routes::Playlists, :edit + Invidious::Routing.post "/edit_playlist", Invidious::Routes::Playlists, :update + Invidious::Routing.get "/add_playlist_items", Invidious::Routes::Playlists, :add_playlist_items_page + Invidious::Routing.post "/playlist_ajax", Invidious::Routes::Playlists, :playlist_ajax + Invidious::Routing.get "/playlist", Invidious::Routes::Playlists, :show + Invidious::Routing.get "/mix", Invidious::Routes::Playlists, :mix + + Invidious::Routing.get "/opensearch.xml", Invidious::Routes::Search, :opensearch + Invidious::Routing.get "/results", Invidious::Routes::Search, :results + Invidious::Routing.get "/search", Invidious::Routes::Search, :search + + Invidious::Routing.get "/login", Invidious::Routes::Login, :login_page + Invidious::Routing.post "/login", Invidious::Routes::Login, :login + Invidious::Routing.post "/signout", Invidious::Routes::Login, :signout + + Invidious::Routing.get "/preferences", Invidious::Routes::PreferencesRoute, :show + Invidious::Routing.post "/preferences", Invidious::Routes::PreferencesRoute, :update + Invidious::Routing.get "/toggle_theme", Invidious::Routes::PreferencesRoute, :toggle_theme + + # Feeds + Invidious::Routing.get "/view_all_playlists", Invidious::Routes::Feeds, :view_all_playlists_redirect + Invidious::Routing.get "/feed/playlists", Invidious::Routes::Feeds, :playlists + Invidious::Routing.get "/feed/popular", Invidious::Routes::Feeds, :popular + Invidious::Routing.get "/feed/trending", Invidious::Routes::Feeds, :trending + Invidious::Routing.get "/feed/subscriptions", Invidious::Routes::Feeds, :subscriptions + Invidious::Routing.get "/feed/history", Invidious::Routes::Feeds, :history + + # RSS Feeds + Invidious::Routing.get "/feed/channel/:ucid", Invidious::Routes::Feeds, :rss_channel + Invidious::Routing.get "/feed/private", Invidious::Routes::Feeds, :rss_private + Invidious::Routing.get "/feed/playlist/:plid", Invidious::Routes::Feeds, :rss_playlist + Invidious::Routing.get "/feeds/videos.xml", Invidious::Routes::Feeds, :rss_videos + + # Support push notifications via PubSubHubbub + Invidious::Routing.get "/feed/webhook/:token", Invidious::Routes::Feeds, :push_notifications_get + Invidious::Routing.post "/feed/webhook/:token", Invidious::Routes::Feeds, :push_notifications_post +{% end %} + +# API routes (macro) +define_v1_api_routes() + +# Video playback (macros) +define_api_manifest_routes() +define_video_playback_routes() # Users @@ -445,7 +503,7 @@ get "/modify_notifications" do |env| html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers) - cookies = HTTP::Cookies.from_headers(headers) + cookies = HTTP::Cookies.from_client_headers(headers) html.cookies.each do |cookie| if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name if cookies[cookie.name]? @@ -757,10 +815,16 @@ post "/data_control" do |env| end end when "import_youtube" - subscriptions = JSON.parse(body) - - user.subscriptions += subscriptions.as_a.compact_map do |entry| - entry["snippet"]["resourceId"]["channelId"].as_s + if body[0..4] == "<opml" + subscriptions = XML.parse(body) + user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| + channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] + end + else + subscriptions = JSON.parse(body) + user.subscriptions += subscriptions.as_a.compact_map do |entry| + entry["snippet"]["resourceId"]["channelId"].as_s + end end user.subscriptions.uniq! @@ -1148,426 +1212,6 @@ post "/token_ajax" do |env| end end -# Feeds - -get "/feed/playlists" do |env| - env.redirect "/view_all_playlists" -end - -get "/feed/top" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - message = translate(locale, "The Top feed has been removed from Invidious.") - templated "message" -end - -get "/feed/popular" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - if config.popular_enabled - templated "popular" - else - message = translate(locale, "The Popular feed has been disabled by the administrator.") - templated "message" - end -end - -get "/feed/trending" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - trending_type = env.params.query["type"]? - trending_type ||= "Default" - - region = env.params.query["region"]? - region ||= "US" - - begin - trending, plid = fetch_trending(trending_type, region, locale) - rescue ex - next error_template(500, ex) - end - - templated "trending" -end - -get "/feed/subscriptions" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - token = user.token - - if user.preferences.unseen_only - env.set "show_watched", true - end - - # Refresh account - headers = HTTP::Headers.new - headers["Cookie"] = env.request.headers["Cookie"] - - if !user.password - user, sid = get_user(sid, headers, PG_DB) - end - - max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE) - max_results ||= user.preferences.max_results - max_results ||= CONFIG.default_user_preferences.max_results - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) - - # "updated" here is used for delivering new notifications, so if - # we know a user has looked at their feed e.g. in the past 10 minutes, - # they've already seen a video posted 20 minutes ago, and don't need - # to be notified. - PG_DB.exec("UPDATE users SET notifications = $1, updated = $2 WHERE email = $3", [] of String, Time.utc, - user.email) - user.notifications = [] of String - env.set "user", user - - templated "subscriptions" -end - -get "/feed/history" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - referer = get_referer(env) - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - if !user - next env.redirect referer - end - - user = user.as(User) - - max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE) - max_results ||= user.preferences.max_results - max_results ||= CONFIG.default_user_preferences.max_results - - if user.watched[(page - 1) * max_results]? - watched = user.watched.reverse[(page - 1) * max_results, max_results] - end - watched ||= [] of String - - templated "history" -end - -get "/feed/channel/:ucid" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/atom+xml" - - ucid = env.params.url["ucid"] - - params = HTTP::Params.parse(env.params.query["params"]? || "") - - begin - channel = get_about_info(ucid, locale) - rescue ex : ChannelRedirect - next env.redirect env.request.resource.gsub(ucid, ex.channel_id) - rescue ex - next error_atom(500, ex) - end - - response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}") - rss = XML.parse_html(response.body) - - videos = rss.xpath_nodes("//feed/entry").map do |entry| - video_id = entry.xpath_node("videoid").not_nil!.content - title = entry.xpath_node("title").not_nil!.content - - published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) - updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) - - author = entry.xpath_node("author/name").not_nil!.content - ucid = entry.xpath_node("channelid").not_nil!.content - description_html = entry.xpath_node("group/description").not_nil!.to_s - views = entry.xpath_node("group/community/statistics").not_nil!.["views"].to_i64 - - SearchVideo.new({ - title: title, - id: video_id, - author: author, - ucid: ucid, - published: published, - views: views, - description_html: description_html, - length_seconds: 0, - live_now: false, - paid: false, - premium: false, - premiere_timestamp: nil, - }) - end - - XML.build(indent: " ", encoding: "UTF-8") do |xml| - xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", - "xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom", - "xml:lang": "en-US") do - xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}") - xml.element("id") { xml.text "yt:channel:#{channel.ucid}" } - xml.element("yt:channelId") { xml.text channel.ucid } - xml.element("icon") { xml.text channel.author_thumbnail } - xml.element("title") { xml.text channel.author } - xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{channel.ucid}") - - xml.element("author") do - xml.element("name") { xml.text channel.author } - xml.element("uri") { xml.text "#{HOST_URL}/channel/#{channel.ucid}" } - end - - videos.each do |video| - video.to_xml(channel.auto_generated, params, xml) - end - end - end -end - -get "/feed/private" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/atom+xml" - - token = env.params.query["token"]? - - if !token - env.response.status_code = 403 - next - end - - user = PG_DB.query_one?("SELECT * FROM users WHERE token = $1", token.strip, as: User) - if !user - env.response.status_code = 403 - next - end - - max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE) - max_results ||= user.preferences.max_results - max_results ||= CONFIG.default_user_preferences.max_results - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - params = HTTP::Params.parse(env.params.query["params"]? || "") - - videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) - - XML.build(indent: " ", encoding: "UTF-8") do |xml| - xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", - "xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom", - "xml:lang": "en-US") do - xml.element("link", "type": "text/html", rel: "alternate", href: "#{HOST_URL}/feed/subscriptions") - xml.element("link", "type": "application/atom+xml", rel: "self", - href: "#{HOST_URL}#{env.request.resource}") - xml.element("title") { xml.text translate(locale, "Invidious Private Feed for `x`", user.email) } - - (notifications + videos).each do |video| - video.to_xml(locale, params, xml) - end - end - end -end - -get "/feed/playlist/:plid" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/atom+xml" - - plid = env.params.url["plid"] - - params = HTTP::Params.parse(env.params.query["params"]? || "") - path = env.request.path - - if plid.starts_with? "IV" - if playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) - videos = get_playlist_videos(PG_DB, playlist, offset: 0, locale: locale) - - next XML.build(indent: " ", encoding: "UTF-8") do |xml| - xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", - "xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom", - "xml:lang": "en-US") do - xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}") - xml.element("id") { xml.text "iv:playlist:#{plid}" } - xml.element("iv:playlistId") { xml.text plid } - xml.element("title") { xml.text playlist.title } - xml.element("link", rel: "alternate", href: "#{HOST_URL}/playlist?list=#{plid}") - - xml.element("author") do - xml.element("name") { xml.text playlist.author } - end - - videos.each do |video| - video.to_xml(false, xml) - end - end - end - else - env.response.status_code = 404 - next - end - end - - response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}") - document = XML.parse(response.body) - - document.xpath_nodes(%q(//*[@href]|//*[@url])).each do |node| - node.attributes.each do |attribute| - case attribute.name - when "url", "href" - full_path = URI.parse(node[attribute.name]).full_path - query_string_opt = full_path.starts_with?("/watch?v=") ? "&#{params}" : "" - node[attribute.name] = "#{HOST_URL}#{full_path}#{query_string_opt}" - else nil # Skip - end - end - end - - document = document.to_xml(options: XML::SaveOptions::NO_DECL) - - document.scan(/<uri>(?<url>[^<]+)<\/uri>/).each do |match| - content = "#{HOST_URL}#{URI.parse(match["url"]).full_path}" - document = document.gsub(match[0], "<uri>#{content}</uri>") - end - - document -end - -get "/feeds/videos.xml" do |env| - if ucid = env.params.query["channel_id"]? - env.redirect "/feed/channel/#{ucid}" - elsif user = env.params.query["user"]? - env.redirect "/feed/channel/#{user}" - elsif plid = env.params.query["playlist_id"]? - env.redirect "/feed/playlist/#{plid}" - end -end - -# Support push notifications via PubSubHubbub - -get "/feed/webhook/:token" do |env| - verify_token = env.params.url["token"] - - mode = env.params.query["hub.mode"]? - topic = env.params.query["hub.topic"]? - challenge = env.params.query["hub.challenge"]? - - if !mode || !topic || !challenge - env.response.status_code = 400 - next - else - mode = mode.not_nil! - topic = topic.not_nil! - challenge = challenge.not_nil! - end - - case verify_token - when .starts_with? "v1" - _, time, nonce, signature = verify_token.split(":") - data = "#{time}:#{nonce}" - when .starts_with? "v2" - time, signature = verify_token.split(":") - data = "#{time}" - else - env.response.status_code = 400 - next - end - - # The hub will sometimes check if we're still subscribed after delivery errors, - # so we reply with a 200 as long as the request hasn't expired - if Time.utc.to_unix - time.to_i > 432000 - env.response.status_code = 400 - next - end - - if OpenSSL::HMAC.hexdigest(:sha1, HMAC_KEY, data) != signature - env.response.status_code = 400 - next - end - - if ucid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["channel_id"]? - PG_DB.exec("UPDATE channels SET subscribed = $1 WHERE id = $2", Time.utc, ucid) - elsif plid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["playlist_id"]? - PG_DB.exec("UPDATE playlists SET subscribed = $1 WHERE id = $2", Time.utc, ucid) - else - env.response.status_code = 400 - next - end - - env.response.status_code = 200 - challenge -end - -post "/feed/webhook/:token" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - token = env.params.url["token"] - body = env.request.body.not_nil!.gets_to_end - signature = env.request.headers["X-Hub-Signature"].lchop("sha1=") - - if signature != OpenSSL::HMAC.hexdigest(:sha1, HMAC_KEY, body) - LOGGER.error("/feed/webhook/#{token} : Invalid signature") - env.response.status_code = 200 - next - end - - spawn do - rss = XML.parse_html(body) - rss.xpath_nodes("//feed/entry").each do |entry| - id = entry.xpath_node("videoid").not_nil!.content - author = entry.xpath_node("author/name").not_nil!.content - published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) - updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) - - video = get_video(id, PG_DB, force_refresh: true) - - # Deliver notifications to `/api/v1/auth/notifications` - payload = { - "topic" => video.ucid, - "videoId" => video.id, - "published" => published.to_unix, - }.to_json - PG_DB.exec("NOTIFY notifications, E'#{payload}'") - - video = ChannelVideo.new({ - id: id, - title: video.title, - published: published, - updated: updated, - ucid: video.ucid, - author: author, - length_seconds: video.length_seconds, - live_now: video.live_now, - premiere_timestamp: video.premiere_timestamp, - views: video.views, - }) - - was_insert = PG_DB.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ - ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ - updated = $4, ucid = $5, author = $6, length_seconds = $7, \ - live_now = $8, premiere_timestamp = $9, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) - - PG_DB.exec("UPDATE users SET notifications = array_append(notifications, $1), \ - feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert - end - end - - env.response.status_code = 200 - next -end - # Channels {"/channel/:ucid/live", "/user/:user/live", "/c/:user/live"}.each do |route| @@ -1605,1131 +1249,12 @@ end end end -# YouTube appears to let users set a "brand" URL that -# is different from their username, so we convert that here -get "/c/:user" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.params.url["user"] - - response = YT_POOL.client &.get("/c/#{user}") - html = XML.parse_html(response.body) - - ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1] - next env.redirect "/" if !ucid - - env.redirect "/channel/#{ucid}" -end - -# Legacy endpoint for /user/:username -get "/profile" do |env| - user = env.params.query["user"]? - if !user - env.redirect "/" - else - env.redirect "/user/#{user}" - end -end - -get "/attribution_link" do |env| - if query = env.params.query["u"]? - url = URI.parse(query).full_path - else - url = "/" - end - - env.redirect url -end - -# Page used by YouTube to provide captioning widget, since we -# don't support it we redirect to '/' -get "/timedtext_video" do |env| - env.redirect "/" -end - -get "/user/:user" do |env| - user = env.params.url["user"] - env.redirect "/channel/#{user}" -end - -get "/user/:user/videos" do |env| - user = env.params.url["user"] - env.redirect "/channel/#{user}/videos" -end - -get "/user/:user/about" do |env| - user = env.params.url["user"] - env.redirect "/channel/#{user}" -end - -get "/channel/:ucid/about" do |env| - ucid = env.params.url["ucid"] - env.redirect "/channel/#{ucid}" -end - -get "/channel/:ucid" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - if user - user = user.as(User) - subscriptions = user.subscriptions - end - subscriptions ||= [] of String - - ucid = env.params.url["ucid"] - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - continuation = env.params.query["continuation"]? - - sort_by = env.params.query["sort_by"]?.try &.downcase - - begin - channel = get_about_info(ucid, locale) - rescue ex : ChannelRedirect - next env.redirect env.request.resource.gsub(ucid, ex.channel_id) - rescue ex - next error_template(500, ex) - end - - if channel.auto_generated - sort_options = {"last", "oldest", "newest"} - sort_by ||= "last" - - items, continuation = fetch_channel_playlists(channel.ucid, channel.author, channel.auto_generated, continuation, sort_by) - items.uniq! do |item| - if item.responds_to?(:title) - item.title - elsif item.responds_to?(:author) - item.author - end - end - items = items.select(&.is_a?(SearchPlaylist)).map(&.as(SearchPlaylist)) - items.each { |item| item.author = "" } - else - sort_options = {"newest", "oldest", "popular"} - sort_by ||= "newest" - - count, items = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) - items.reject! &.paid - - env.set "search", "channel:#{channel.ucid} " - end - - templated "channel" -end - -get "/channel/:ucid/videos" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - ucid = env.params.url["ucid"] - params = env.request.query - - if !params || params.empty? - params = "" - else - params = "?#{params}" - end - - env.redirect "/channel/#{ucid}#{params}" -end - -get "/channel/:ucid/playlists" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - if user - user = user.as(User) - subscriptions = user.subscriptions - end - subscriptions ||= [] of String - - ucid = env.params.url["ucid"] - - continuation = env.params.query["continuation"]? - - sort_by = env.params.query["sort_by"]?.try &.downcase - sort_by ||= "last" - - begin - channel = get_about_info(ucid, locale) - rescue ex : ChannelRedirect - next env.redirect env.request.resource.gsub(ucid, ex.channel_id) - rescue ex - next error_template(500, ex) - end - - if channel.auto_generated - next env.redirect "/channel/#{channel.ucid}" - end - - items, continuation = fetch_channel_playlists(channel.ucid, channel.author, channel.auto_generated, continuation, sort_by) - items = items.select { |item| item.is_a?(SearchPlaylist) }.map { |item| item.as(SearchPlaylist) } - items.each { |item| item.author = "" } - - env.set "search", "channel:#{channel.ucid} " - templated "playlists" -end - -get "/channel/:ucid/community" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - if user - user = user.as(User) - subscriptions = user.subscriptions - end - subscriptions ||= [] of String - - ucid = env.params.url["ucid"] - - thin_mode = env.params.query["thin_mode"]? || env.get("preferences").as(Preferences).thin_mode - thin_mode = thin_mode == "true" - - continuation = env.params.query["continuation"]? - # sort_by = env.params.query["sort_by"]?.try &.downcase - - begin - channel = get_about_info(ucid, locale) - rescue ex : ChannelRedirect - next env.redirect env.request.resource.gsub(ucid, ex.channel_id) - rescue ex - next error_template(500, ex) - end - - if !channel.tabs.includes? "community" - next env.redirect "/channel/#{channel.ucid}" - end - - begin - items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode)) - rescue ex : InfoException - env.response.status_code = 500 - error_message = ex.message - rescue ex - next error_template(500, ex) - end - - env.set "search", "channel:#{channel.ucid} " - templated "community" -end - -# API Endpoints - -get "/api/v1/stats" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - env.response.content_type = "application/json" - - if !config.statistics_enabled - next error_json(400, "Statistics are not enabled.") - end - - Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json -end - -# YouTube provides "storyboards", which are sprites containing x * y -# preview thumbnails for individual scenes in a video. -# See https://support.jwplayer.com/articles/how-to-add-preview-thumbnails -get "/api/v1/storyboards/:id" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - id = env.params.url["id"] - region = env.params.query["region"]? - - begin - video = get_video(id, PG_DB, region: region) - rescue ex : VideoRedirect - env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) - next error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) - rescue ex - env.response.status_code = 500 - next - end - - storyboards = video.storyboards - width = env.params.query["width"]? - height = env.params.query["height"]? - - if !width && !height - response = JSON.build do |json| - json.object do - json.field "storyboards" do - generate_storyboards(json, id, storyboards) - end - end - end - - next response - end - - env.response.content_type = "text/vtt" - - storyboard = storyboards.select { |storyboard| width == "#{storyboard[:width]}" || height == "#{storyboard[:height]}" } - - if storyboard.empty? - env.response.status_code = 404 - next - else - storyboard = storyboard[0] - end - - String.build do |str| - str << <<-END_VTT - WEBVTT - - - END_VTT - - start_time = 0.milliseconds - end_time = storyboard[:interval].milliseconds - - storyboard[:storyboard_count].times do |i| - url = storyboard[:url] - authority = /(i\d?).ytimg.com/.match(url).not_nil![1]? - url = url.gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "") - url = "#{HOST_URL}/sb/#{authority}/#{url}" - - storyboard[:storyboard_height].times do |j| - storyboard[:storyboard_width].times do |k| - str << <<-END_CUE - #{start_time}.000 --> #{end_time}.000 - #{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]} - - - END_CUE - - start_time += storyboard[:interval].milliseconds - end_time += storyboard[:interval].milliseconds - end - end - end - end -end - -get "/api/v1/captions/:id" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - id = env.params.url["id"] - region = env.params.query["region"]? - - # See https://github.com/ytdl-org/youtube-dl/blob/6ab30ff50bf6bd0585927cb73c7421bef184f87a/youtube_dl/extractor/youtube.py#L1354 - # It is possible to use `/api/timedtext?type=list&v=#{id}` and - # `/api/timedtext?type=track&v=#{id}&lang=#{lang_code}` directly, - # but this does not provide links for auto-generated captions. - # - # In future this should be investigated as an alternative, since it does not require - # getting video info. - - begin - video = get_video(id, PG_DB, region: region) - rescue ex : VideoRedirect - env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) - next error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) - rescue ex - env.response.status_code = 500 - next - end - - captions = video.captions - - label = env.params.query["label"]? - lang = env.params.query["lang"]? - tlang = env.params.query["tlang"]? - - if !label && !lang - response = JSON.build do |json| - json.object do - json.field "captions" do - json.array do - captions.each do |caption| - json.object do - json.field "label", caption.name.simpleText - json.field "languageCode", caption.languageCode - json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name.simpleText)}" - end - end - end - end - end - end - - next response - end - - env.response.content_type = "text/vtt; charset=UTF-8" - - if lang - caption = captions.select { |caption| caption.languageCode == lang } - else - caption = captions.select { |caption| caption.name.simpleText == label } - end - - if caption.empty? - env.response.status_code = 404 - next - else - caption = caption[0] - end - - url = URI.parse("#{caption.baseUrl}&tlang=#{tlang}").full_path - - # Auto-generated captions often have cues that aren't aligned properly with the video, - # as well as some other markup that makes it cumbersome, so we try to fix that here - if caption.name.simpleText.includes? "auto-generated" - caption_xml = YT_POOL.client &.get(url).body - caption_xml = XML.parse(caption_xml) - - webvtt = String.build do |str| - str << <<-END_VTT - WEBVTT - Kind: captions - Language: #{tlang || caption.languageCode} - - - END_VTT - - caption_nodes = caption_xml.xpath_nodes("//transcript/text") - caption_nodes.each_with_index do |node, i| - start_time = node["start"].to_f.seconds - duration = node["dur"]?.try &.to_f.seconds - duration ||= start_time - - if caption_nodes.size > i + 1 - end_time = caption_nodes[i + 1]["start"].to_f.seconds - else - end_time = start_time + duration - end - - start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}" - end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}" - - text = HTML.unescape(node.content) - text = text.gsub(/<font color="#[a-fA-F0-9]{6}">/, "") - text = text.gsub(/<\/font>/, "") - if md = text.match(/(?<name>.*) : (?<text>.*)/) - text = "<v #{md["name"]}>#{md["text"]}</v>" - end - - str << <<-END_CUE - #{start_time} --> #{end_time} - #{text} - - - END_CUE - end - end - else - webvtt = YT_POOL.client &.get("#{url}&format=vtt").body - end - - if title = env.params.query["title"]? - # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ - env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}" - end - - webvtt -end - -get "/api/v1/comments/:id" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - region = env.params.query["region"]? - - env.response.content_type = "application/json" - - id = env.params.url["id"] - - source = env.params.query["source"]? - source ||= "youtube" - - thin_mode = env.params.query["thin_mode"]? - thin_mode = thin_mode == "true" - - format = env.params.query["format"]? - format ||= "json" - - continuation = env.params.query["continuation"]? - sort_by = env.params.query["sort_by"]?.try &.downcase - - if source == "youtube" - sort_by ||= "top" - - begin - comments = fetch_youtube_comments(id, PG_DB, continuation, format, locale, thin_mode, region, sort_by: sort_by) - rescue ex - next error_json(500, ex) - end - - next comments - elsif source == "reddit" - sort_by ||= "confidence" - - begin - comments, reddit_thread = fetch_reddit_comments(id, sort_by: sort_by) - content_html = template_reddit_comments(comments, locale) - - content_html = fill_links(content_html, "https", "www.reddit.com") - content_html = replace_links(content_html) - rescue ex - comments = nil - reddit_thread = nil - content_html = "" - end - - if !reddit_thread || !comments - env.response.status_code = 404 - next - end - - if format == "json" - reddit_thread = JSON.parse(reddit_thread.to_json).as_h - reddit_thread["comments"] = JSON.parse(comments.to_json) - - next reddit_thread.to_json - else - response = { - "title" => reddit_thread.title, - "permalink" => reddit_thread.permalink, - "contentHtml" => content_html, - } - - next response.to_json - end - end -end - -get "/api/v1/insights/:id" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - next error_json(410, "YouTube has removed publicly available analytics.") -end - -get "/api/v1/annotations/:id" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "text/xml" - - id = env.params.url["id"] - source = env.params.query["source"]? - source ||= "archive" - - if !id.match(/[a-zA-Z0-9_-]{11}/) - env.response.status_code = 400 - next - end - - annotations = "" - - case source - when "archive" - if CONFIG.cache_annotations && (cached_annotation = PG_DB.query_one?("SELECT * FROM annotations WHERE id = $1", id, as: Annotation)) - annotations = cached_annotation.annotations - else - index = CHARS_SAFE.index(id[0]).not_nil!.to_s.rjust(2, '0') - - # IA doesn't handle leading hyphens, - # so we use https://archive.org/details/youtubeannotations_64 - if index == "62" - index = "64" - id = id.sub(/^-/, 'A') - end - - file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml") - - location = make_client(ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}")) - - if !location.headers["Location"]? - env.response.status_code = location.status_code - end - - response = make_client(URI.parse(location.headers["Location"]), &.get(location.headers["Location"])) - - if response.body.empty? - env.response.status_code = 404 - next - end - - if response.status_code != 200 - env.response.status_code = response.status_code - next - end - - annotations = response.body - - cache_annotation(PG_DB, id, annotations) - end - else # "youtube" - response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}") - - if response.status_code != 200 - env.response.status_code = response.status_code - next - end - - annotations = response.body - end - - etag = sha256(annotations)[0, 16] - if env.request.headers["If-None-Match"]?.try &.== etag - env.response.status_code = 304 - else - env.response.headers["ETag"] = etag - annotations - end -end - -get "/api/v1/videos/:id" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - id = env.params.url["id"] - region = env.params.query["region"]? - - begin - video = get_video(id, PG_DB, region: region) - rescue ex : VideoRedirect - env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) - next error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) - rescue ex - next error_json(500, ex) - end - - video.to_json(locale) -end - -get "/api/v1/trending" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - region = env.params.query["region"]? - trending_type = env.params.query["type"]? - - begin - trending, plid = fetch_trending(trending_type, region, locale) - rescue ex - next error_json(500, ex) - end - - videos = JSON.build do |json| - json.array do - trending.each do |video| - video.to_json(locale, json) - end - end - end - - videos -end - -get "/api/v1/popular" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - if !config.popular_enabled - error_message = {"error" => "Administrator has disabled this endpoint."}.to_json - env.response.status_code = 400 - next error_message - end - - JSON.build do |json| - json.array do - popular_videos.each do |video| - video.to_json(locale, json) - end - end - end -end - -get "/api/v1/top" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - env.response.status_code = 400 - {"error" => "The Top feed has been removed from Invidious."}.to_json -end - -get "/api/v1/channels/:ucid" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - ucid = env.params.url["ucid"] - sort_by = env.params.query["sort_by"]?.try &.downcase - sort_by ||= "newest" - - begin - channel = get_about_info(ucid, locale) - rescue ex : ChannelRedirect - env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) - next error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) - rescue ex - next error_json(500, ex) - end - - page = 1 - if channel.auto_generated - videos = [] of SearchVideo - count = 0 - else - begin - count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) - rescue ex - next error_json(500, ex) - end - end - - JSON.build do |json| - # TODO: Refactor into `to_json` for InvidiousChannel - json.object do - json.field "author", channel.author - json.field "authorId", channel.ucid - json.field "authorUrl", channel.author_url - - json.field "authorBanners" do - json.array do - if channel.banner - qualities = { - {width: 2560, height: 424}, - {width: 2120, height: 351}, - {width: 1060, height: 175}, - } - qualities.each do |quality| - json.object do - json.field "url", channel.banner.not_nil!.gsub("=w1060-", "=w#{quality[:width]}-") - json.field "width", quality[:width] - json.field "height", quality[:height] - end - end - - json.object do - json.field "url", channel.banner.not_nil!.split("=w1060-")[0] - json.field "width", 512 - json.field "height", 288 - end - end - end - end - - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - - qualities.each do |quality| - json.object do - json.field "url", channel.author_thumbnail.gsub(/=s\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", quality - end - end - end - end - - json.field "subCount", channel.sub_count - json.field "totalViews", channel.total_views - json.field "joined", channel.joined.to_unix - json.field "paid", channel.paid - - json.field "autoGenerated", channel.auto_generated - json.field "isFamilyFriendly", channel.is_family_friendly - json.field "description", html_to_content(channel.description_html) - json.field "descriptionHtml", channel.description_html - - json.field "allowedRegions", channel.allowed_regions - - json.field "latestVideos" do - json.array do - videos.each do |video| - video.to_json(locale, json) - end - end - end - - json.field "relatedChannels" do - json.array do - channel.related_channels.each do |related_channel| - json.object do - json.field "author", related_channel.author - json.field "authorId", related_channel.ucid - json.field "authorUrl", related_channel.author_url - - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - - qualities.each do |quality| - json.object do - json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", quality - end - end - end - end - end - end - end - end - end - end -end - -{"/api/v1/channels/:ucid/videos", "/api/v1/channels/videos/:ucid"}.each do |route| - get route do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - ucid = env.params.url["ucid"] - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - sort_by = env.params.query["sort"]?.try &.downcase - sort_by ||= env.params.query["sort_by"]?.try &.downcase - sort_by ||= "newest" - - begin - channel = get_about_info(ucid, locale) - rescue ex : ChannelRedirect - env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) - next error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) - rescue ex - next error_json(500, ex) - end - - begin - count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) - rescue ex - next error_json(500, ex) - end - - JSON.build do |json| - json.array do - videos.each do |video| - video.to_json(locale, json) - end - end - end - end -end - -{"/api/v1/channels/:ucid/latest", "/api/v1/channels/latest/:ucid"}.each do |route| - get route do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - ucid = env.params.url["ucid"] - - begin - videos = get_latest_videos(ucid) - rescue ex - next error_json(500, ex) - end - - JSON.build do |json| - json.array do - videos.each do |video| - video.to_json(locale, json) - end - end - end - end -end - -{"/api/v1/channels/:ucid/playlists", "/api/v1/channels/playlists/:ucid"}.each do |route| - get route do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - ucid = env.params.url["ucid"] - continuation = env.params.query["continuation"]? - sort_by = env.params.query["sort"]?.try &.downcase || - env.params.query["sort_by"]?.try &.downcase || - "last" - - begin - channel = get_about_info(ucid, locale) - rescue ex : ChannelRedirect - env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) - next error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) - rescue ex - next error_json(500, ex) - end - - items, continuation = fetch_channel_playlists(channel.ucid, channel.author, channel.auto_generated, continuation, sort_by) - - JSON.build do |json| - json.object do - json.field "playlists" do - json.array do - items.each do |item| - item.to_json(locale, json) if item.is_a?(SearchPlaylist) - end - end - end - - json.field "continuation", continuation - end - end - end -end - -{"/api/v1/channels/:ucid/comments", "/api/v1/channels/comments/:ucid"}.each do |route| - get route do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - ucid = env.params.url["ucid"] - - thin_mode = env.params.query["thin_mode"]? - thin_mode = thin_mode == "true" - - format = env.params.query["format"]? - format ||= "json" - - continuation = env.params.query["continuation"]? - # sort_by = env.params.query["sort_by"]?.try &.downcase - - begin - fetch_channel_community(ucid, continuation, locale, format, thin_mode) - rescue ex - next error_json(500, ex) - end - end -end - -get "/api/v1/channels/search/:ucid" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - ucid = env.params.url["ucid"] - - query = env.params.query["q"]? - query ||= "" - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - count, search_results = channel_search(query, page, ucid) - JSON.build do |json| - json.array do - search_results.each do |item| - item.to_json(locale, json) - end - end - end -end - -get "/api/v1/search" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - region = env.params.query["region"]? - - env.response.content_type = "application/json" - - query = env.params.query["q"]? - query ||= "" - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - sort_by = env.params.query["sort_by"]?.try &.downcase - sort_by ||= "relevance" - - date = env.params.query["date"]?.try &.downcase - date ||= "" - - duration = env.params.query["duration"]?.try &.downcase - duration ||= "" - - features = env.params.query["features"]?.try &.split(",").map { |feature| feature.downcase } - features ||= [] of String - - content_type = env.params.query["type"]?.try &.downcase - content_type ||= "video" - - begin - search_params = produce_search_params(sort_by, date, content_type, duration, features) - rescue ex - next error_json(400, ex) - end - - count, search_results = search(query, page, search_params, region).as(Tuple) - JSON.build do |json| - json.array do - search_results.each do |item| - item.to_json(locale, json) - end - end - end -end - -get "/api/v1/search/suggestions" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - region = env.params.query["region"]? - - env.response.content_type = "application/json" - - query = env.params.query["q"]? - query ||= "" - - begin - headers = HTTP::Headers{":authority" => "suggestqueries.google.com"} - response = YT_POOL.client &.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback", headers).body - - body = response[35..-2] - body = JSON.parse(body).as_a - suggestions = body[1].as_a[0..-2] - - JSON.build do |json| - json.object do - json.field "query", body[0].as_s - json.field "suggestions" do - json.array do - suggestions.each do |suggestion| - json.string suggestion[0].as_s - end - end - end - end - end - rescue ex - next error_json(500, ex) - end -end - -{"/api/v1/playlists/:plid", "/api/v1/auth/playlists/:plid"}.each do |route| - get route do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - plid = env.params.url["plid"] - - offset = env.params.query["index"]?.try &.to_i? - offset ||= env.params.query["page"]?.try &.to_i?.try { |page| (page - 1) * 100 } - offset ||= 0 - - continuation = env.params.query["continuation"]? - - format = env.params.query["format"]? - format ||= "json" - - if plid.starts_with? "RD" - next env.redirect "/api/v1/mixes/#{plid}" - end - - begin - playlist = get_playlist(PG_DB, plid, locale) - rescue ex : InfoException - next error_json(404, ex) - rescue ex - next error_json(404, "Playlist does not exist.") - end - - user = env.get?("user").try &.as(User) - if !playlist || playlist.privacy.private? && playlist.author != user.try &.email - next error_json(404, "Playlist does not exist.") - end - - response = playlist.to_json(offset, locale, continuation: continuation) - - if format == "html" - response = JSON.parse(response) - playlist_html = template_playlist(response) - index, next_video = response["videos"].as_a.skip(1).select { |video| !video["author"].as_s.empty? }[0]?.try { |v| {v["index"], v["videoId"]} } || {nil, nil} - - response = { - "playlistHtml" => playlist_html, - "index" => index, - "nextVideo" => next_video, - }.to_json - end - - response - end -end - -get "/api/v1/mixes/:rdid" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - rdid = env.params.url["rdid"] - - continuation = env.params.query["continuation"]? - continuation ||= rdid.lchop("RD")[0, 11] - - format = env.params.query["format"]? - format ||= "json" - - begin - mix = fetch_mix(rdid, continuation, locale: locale) - - if !rdid.ends_with? continuation - mix = fetch_mix(rdid, mix.videos[1].id) - index = mix.videos.index(mix.videos.select { |video| video.id == continuation }[0]?) - end - - mix.videos = mix.videos[index..-1] - rescue ex - next error_json(500, ex) - end - - response = JSON.build do |json| - json.object do - json.field "title", mix.title - json.field "mixId", mix.id - - json.field "videos" do - json.array do - mix.videos.each do |video| - json.object do - json.field "title", video.title - json.field "videoId", video.id - json.field "author", video.author - - json.field "authorId", video.ucid - json.field "authorUrl", "/channel/#{video.ucid}" - - json.field "videoThumbnails" do - json.array do - generate_thumbnails(json, video.id) - end - end - - json.field "index", video.index - json.field "lengthSeconds", video.length_seconds - end - end - end - end - end - end - - if format == "html" - response = JSON.parse(response) - playlist_html = template_mix(response) - next_video = response["videos"].as_a.select { |video| !video["author"].as_s.empty? }[0]?.try &.["videoId"] - - response = { - "playlistHtml" => playlist_html, - "nextVideo" => next_video, - }.to_json - end - - response -end - # Authenticated endpoints +# The notification APIs can't be extracted yet +# due to the requirement of the `connection_channel` +# used by the `NotificationJob` + get "/api/v1/auth/notifications" do |env| env.response.content_type = "text/event-stream" @@ -2748,917 +1273,6 @@ post "/api/v1/auth/notifications" do |env| create_notification_stream(env, topics, connection_channel) end -get "/api/v1/auth/preferences" do |env| - env.response.content_type = "application/json" - user = env.get("user").as(User) - user.preferences.to_json -end - -post "/api/v1/auth/preferences" do |env| - env.response.content_type = "application/json" - user = env.get("user").as(User) - - begin - preferences = Preferences.from_json(env.request.body || "{}") - rescue - preferences = user.preferences - end - - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) - - env.response.status_code = 204 -end - -get "/api/v1/auth/feed" do |env| - env.response.content_type = "application/json" - - user = env.get("user").as(User) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - max_results = env.params.query["max_results"]?.try &.to_i? - max_results ||= user.preferences.max_results - max_results ||= CONFIG.default_user_preferences.max_results - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) - - JSON.build do |json| - json.object do - json.field "notifications" do - json.array do - notifications.each do |video| - video.to_json(locale, json) - end - end - end - - json.field "videos" do - json.array do - videos.each do |video| - video.to_json(locale, json) - end - end - end - end - end -end - -get "/api/v1/auth/subscriptions" do |env| - env.response.content_type = "application/json" - user = env.get("user").as(User) - - if user.subscriptions.empty? - values = "'{}'" - else - values = "VALUES #{user.subscriptions.map { |id| %(('#{id}')) }.join(",")}" - end - - subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousChannel) - - JSON.build do |json| - json.array do - subscriptions.each do |subscription| - json.object do - json.field "author", subscription.author - json.field "authorId", subscription.id - end - end - end - end -end - -post "/api/v1/auth/subscriptions/:ucid" do |env| - env.response.content_type = "application/json" - user = env.get("user").as(User) - - ucid = env.params.url["ucid"] - - if !user.subscriptions.includes? ucid - get_channel(ucid, PG_DB, false, false) - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email) - end - - # For Google accounts, access tokens don't have enough information to - # make a request on the user's behalf, which is why we don't sync with - # YouTube. - - env.response.status_code = 204 -end - -delete "/api/v1/auth/subscriptions/:ucid" do |env| - env.response.content_type = "application/json" - user = env.get("user").as(User) - - ucid = env.params.url["ucid"] - - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_remove(subscriptions, $1) WHERE email = $2", ucid, user.email) - - env.response.status_code = 204 -end - -get "/api/v1/auth/playlists" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - user = env.get("user").as(User) - - playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1", user.email, as: InvidiousPlaylist) - - JSON.build do |json| - json.array do - playlists.each do |playlist| - playlist.to_json(0, locale, json) - end - end - end -end - -post "/api/v1/auth/playlists" do |env| - env.response.content_type = "application/json" - user = env.get("user").as(User) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - title = env.params.json["title"]?.try &.as(String).delete("<>").byte_slice(0, 150) - if !title - next error_json(400, "Invalid title.") - end - - privacy = env.params.json["privacy"]?.try { |privacy| PlaylistPrivacy.parse(privacy.as(String).downcase) } - if !privacy - next error_json(400, "Invalid privacy setting.") - end - - if PG_DB.query_one("SELECT count(*) FROM playlists WHERE author = $1", user.email, as: Int64) >= 100 - next error_json(400, "User cannot have more than 100 playlists.") - end - - playlist = create_playlist(PG_DB, title, privacy, user) - env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{playlist.id}" - env.response.status_code = 201 - { - "title" => title, - "playlistId" => playlist.id, - }.to_json -end - -patch "/api/v1/auth/playlists/:plid" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - user = env.get("user").as(User) - - plid = env.params.url["plid"] - - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) - if !playlist || playlist.author != user.email && playlist.privacy.private? - next error_json(404, "Playlist does not exist.") - end - - if playlist.author != user.email - next error_json(403, "Invalid user") - end - - title = env.params.json["title"].try &.as(String).delete("<>").byte_slice(0, 150) || playlist.title - privacy = env.params.json["privacy"]?.try { |privacy| PlaylistPrivacy.parse(privacy.as(String).downcase) } || playlist.privacy - description = env.params.json["description"]?.try &.as(String).delete("\r") || playlist.description - - if title != playlist.title || - privacy != playlist.privacy || - description != playlist.description - updated = Time.utc - else - updated = playlist.updated - end - - PG_DB.exec("UPDATE playlists SET title = $1, privacy = $2, description = $3, updated = $4 WHERE id = $5", title, privacy, description, updated, plid) - env.response.status_code = 204 -end - -delete "/api/v1/auth/playlists/:plid" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - user = env.get("user").as(User) - - plid = env.params.url["plid"] - - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) - if !playlist || playlist.author != user.email && playlist.privacy.private? - next error_json(404, "Playlist does not exist.") - end - - if playlist.author != user.email - next error_json(403, "Invalid user") - end - - PG_DB.exec("DELETE FROM playlist_videos * WHERE plid = $1", plid) - PG_DB.exec("DELETE FROM playlists * WHERE id = $1", plid) - - env.response.status_code = 204 -end - -post "/api/v1/auth/playlists/:plid/videos" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - user = env.get("user").as(User) - - plid = env.params.url["plid"] - - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) - if !playlist || playlist.author != user.email && playlist.privacy.private? - next error_json(404, "Playlist does not exist.") - end - - if playlist.author != user.email - next error_json(403, "Invalid user") - end - - if playlist.index.size >= 500 - next error_json(400, "Playlist cannot have more than 500 videos") - end - - video_id = env.params.json["videoId"].try &.as(String) - if !video_id - next error_json(403, "Invalid videoId") - end - - begin - video = get_video(video_id, PG_DB) - rescue ex - next error_json(500, ex) - end - - playlist_video = PlaylistVideo.new({ - title: video.title, - id: video.id, - author: video.author, - ucid: video.ucid, - length_seconds: video.length_seconds, - published: video.published, - plid: plid, - live_now: video.live_now, - index: Random::Secure.rand(0_i64..Int64::MAX), - }) - - video_array = playlist_video.to_a - args = arg_array(video_array) - - PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array) - PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, plid) - - env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{plid}/videos/#{playlist_video.index.to_u64.to_s(16).upcase}" - env.response.status_code = 201 - playlist_video.to_json(locale, index: playlist.index.size) -end - -delete "/api/v1/auth/playlists/:plid/videos/:index" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - user = env.get("user").as(User) - - plid = env.params.url["plid"] - index = env.params.url["index"].to_i64(16) - - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) - if !playlist || playlist.author != user.email && playlist.privacy.private? - next error_json(404, "Playlist does not exist.") - end - - if playlist.author != user.email - next error_json(403, "Invalid user") - end - - if !playlist.index.includes? index - next error_json(404, "Playlist does not contain index") - end - - PG_DB.exec("DELETE FROM playlist_videos * WHERE index = $1", index) - PG_DB.exec("UPDATE playlists SET index = array_remove(index, $1), video_count = cardinality(index) - 1, updated = $2 WHERE id = $3", index, Time.utc, plid) - - env.response.status_code = 204 -end - -# patch "/api/v1/auth/playlists/:plid/videos/:index" do |env| -# TODO: Playlist stub -# end - -get "/api/v1/auth/tokens" do |env| - env.response.content_type = "application/json" - user = env.get("user").as(User) - scopes = env.get("scopes").as(Array(String)) - - tokens = PG_DB.query_all("SELECT id, issued FROM session_ids WHERE email = $1", user.email, as: {session: String, issued: Time}) - - JSON.build do |json| - json.array do - tokens.each do |token| - json.object do - json.field "session", token[:session] - json.field "issued", token[:issued].to_unix - end - end - end - end -end - -post "/api/v1/auth/tokens/register" do |env| - user = env.get("user").as(User) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - case env.request.headers["Content-Type"]? - when "application/x-www-form-urlencoded" - scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v } - callback_url = env.params.body["callbackUrl"]? - expire = env.params.body["expire"]?.try &.to_i? - when "application/json" - scopes = env.params.json["scopes"].as(Array).map { |v| v.as_s } - callback_url = env.params.json["callbackUrl"]?.try &.as(String) - expire = env.params.json["expire"]?.try &.as(Int64) - else - next error_json(400, "Invalid or missing header 'Content-Type'") - end - - if callback_url && callback_url.empty? - callback_url = nil - end - - if callback_url - callback_url = URI.parse(callback_url) - end - - if sid = env.get?("sid").try &.as(String) - env.response.content_type = "text/html" - - csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, PG_DB, use_nonce: true) - next templated "authorize_token" - else - env.response.content_type = "application/json" - - superset_scopes = env.get("scopes").as(Array(String)) - - authorized_scopes = [] of String - scopes.each do |scope| - if scopes_include_scope(superset_scopes, scope) - authorized_scopes << scope - end - end - - access_token = generate_token(user.email, authorized_scopes, expire, HMAC_KEY, PG_DB) - - if callback_url - access_token = URI.encode_www_form(access_token) - - if query = callback_url.query - query = HTTP::Params.parse(query.not_nil!) - else - query = HTTP::Params.new - end - - query["token"] = access_token - callback_url.query = query.to_s - - env.redirect callback_url.to_s - else - access_token - end - end -end - -post "/api/v1/auth/tokens/unregister" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - env.response.content_type = "application/json" - user = env.get("user").as(User) - scopes = env.get("scopes").as(Array(String)) - - session = env.params.json["session"]?.try &.as(String) - session ||= env.get("session").as(String) - - # Allow tokens to revoke other tokens with correct scope - if session == env.get("session").as(String) - PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", session) - elsif scopes_include_scope(scopes, "GET:tokens") - PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", session) - else - next error_json(400, "Cannot revoke session #{session}") - end - - env.response.status_code = 204 -end - -get "/api/manifest/dash/id/videoplayback" do |env| - env.response.headers.delete("Content-Type") - env.response.headers["Access-Control-Allow-Origin"] = "*" - env.redirect "/videoplayback?#{env.params.query}" -end - -get "/api/manifest/dash/id/videoplayback/*" do |env| - env.response.headers.delete("Content-Type") - env.response.headers["Access-Control-Allow-Origin"] = "*" - env.redirect env.request.path.lchop("/api/manifest/dash/id") -end - -get "/api/manifest/dash/id/:id" do |env| - env.response.headers.add("Access-Control-Allow-Origin", "*") - env.response.content_type = "application/dash+xml" - - local = env.params.query["local"]?.try &.== "true" - id = env.params.url["id"] - region = env.params.query["region"]? - - # Since some implementations create playlists based on resolution regardless of different codecs, - # we can opt to only add a source to a representation if it has a unique height within that representation - unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe } - - begin - video = get_video(id, PG_DB, region: region) - rescue ex : VideoRedirect - next env.redirect env.request.resource.gsub(id, ex.video_id) - rescue ex - env.response.status_code = 403 - next - end - - if dashmpd = video.dash_manifest_url - manifest = YT_POOL.client &.get(URI.parse(dashmpd).full_path).body - - manifest = manifest.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl| - url = baseurl.lchop("<BaseURL>") - url = url.rchop("</BaseURL>") - - if local - uri = URI.parse(url) - url = "#{uri.full_path}host/#{uri.host}/" - end - - "<BaseURL>#{url}</BaseURL>" - end - - next manifest - end - - adaptive_fmts = video.adaptive_fmts - - if local - adaptive_fmts.each do |fmt| - fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) - end - end - - audio_streams = video.audio_streams - video_streams = video.video_streams.sort_by { |stream| {stream["width"].as_i, stream["fps"].as_i} }.reverse - - XML.build(indent: " ", encoding: "UTF-8") do |xml| - xml.element("MPD", "xmlns": "urn:mpeg:dash:schema:mpd:2011", - "profiles": "urn:mpeg:dash:profile:full:2011", minBufferTime: "PT1.5S", type: "static", - mediaPresentationDuration: "PT#{video.length_seconds}S") do - xml.element("Period") do - i = 0 - - {"audio/mp4", "audio/webm"}.each do |mime_type| - mime_streams = audio_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type } - next if mime_streams.empty? - - xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true) do - mime_streams.each do |fmt| - codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"') - bandwidth = fmt["bitrate"].as_i - itag = fmt["itag"].as_i - url = fmt["url"].as_s - - xml.element("Representation", id: fmt["itag"], codecs: codecs, bandwidth: bandwidth) do - xml.element("AudioChannelConfiguration", schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011", - value: "2") - xml.element("BaseURL") { xml.text url } - xml.element("SegmentBase", indexRange: "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}") do - xml.element("Initialization", range: "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}") - end - end - end - end - - i += 1 - end - - potential_heights = {4320, 2160, 1440, 1080, 720, 480, 360, 240, 144} - - {"video/mp4", "video/webm"}.each do |mime_type| - mime_streams = video_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type } - next if mime_streams.empty? - - heights = [] of Int32 - xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, scanType: "progressive") do - mime_streams.each do |fmt| - codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"') - bandwidth = fmt["bitrate"].as_i - itag = fmt["itag"].as_i - url = fmt["url"].as_s - width = fmt["width"].as_i - height = fmt["height"].as_i - - # Resolutions reported by YouTube player (may not accurately reflect source) - height = potential_heights.min_by { |i| (height - i).abs } - next if unique_res && heights.includes? height - heights << height - - xml.element("Representation", id: itag, codecs: codecs, width: width, height: height, - startWithSAP: "1", maxPlayoutRate: "1", - bandwidth: bandwidth, frameRate: fmt["fps"]) do - xml.element("BaseURL") { xml.text url } - xml.element("SegmentBase", indexRange: "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}") do - xml.element("Initialization", range: "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}") - end - end - end - end - - i += 1 - end - end - end - end -end - -get "/api/manifest/hls_variant/*" do |env| - response = YT_POOL.client &.get(env.request.path) - - if response.status_code != 200 - env.response.status_code = response.status_code - next - end - - local = env.params.query["local"]?.try &.== "true" - - env.response.content_type = "application/x-mpegURL" - env.response.headers.add("Access-Control-Allow-Origin", "*") - - manifest = response.body - - if local - manifest = manifest.gsub("https://www.youtube.com", HOST_URL) - manifest = manifest.gsub("index.m3u8", "index.m3u8?local=true") - end - - manifest -end - -get "/api/manifest/hls_playlist/*" do |env| - response = YT_POOL.client &.get(env.request.path) - - if response.status_code != 200 - env.response.status_code = response.status_code - next - end - - local = env.params.query["local"]?.try &.== "true" - - env.response.content_type = "application/x-mpegURL" - env.response.headers.add("Access-Control-Allow-Origin", "*") - - manifest = response.body - - if local - manifest = manifest.gsub(/^https:\/\/r\d---.{11}\.c\.youtube\.com[^\n]*/m) do |match| - path = URI.parse(match).path - - path = path.lchop("/videoplayback/") - path = path.rchop("/") - - path = path.gsub(/mime\/\w+\/\w+/) do |mimetype| - mimetype = mimetype.split("/") - mimetype[0] + "/" + mimetype[1] + "%2F" + mimetype[2] - end - - path = path.split("/") - - raw_params = {} of String => Array(String) - path.each_slice(2) do |pair| - key, value = pair - value = URI.decode_www_form(value) - - if raw_params[key]? - raw_params[key] << value - else - raw_params[key] = [value] - end - end - - raw_params = HTTP::Params.new(raw_params) - if fvip = raw_params["hls_chunk_host"].match(/r(?<fvip>\d+)---/) - raw_params["fvip"] = fvip["fvip"] - end - - raw_params["local"] = "true" - - "#{HOST_URL}/videoplayback?#{raw_params}" - end - end - - manifest -end - -# YouTube /videoplayback links expire after 6 hours, -# so we have a mechanism here to redirect to the latest version -get "/latest_version" do |env| - if env.params.query["download_widget"]? - download_widget = JSON.parse(env.params.query["download_widget"]) - - id = download_widget["id"].as_s - title = download_widget["title"].as_s - - if label = download_widget["label"]? - env.redirect "/api/v1/captions/#{id}?label=#{label}&title=#{title}" - next - else - itag = download_widget["itag"].as_s.to_i - local = "true" - end - end - - id ||= env.params.query["id"]? - itag ||= env.params.query["itag"]?.try &.to_i - - region = env.params.query["region"]? - - local ||= env.params.query["local"]? - local ||= "false" - local = local == "true" - - if !id || !itag - env.response.status_code = 400 - next - end - - video = get_video(id, PG_DB, region: region) - - fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag } - url = fmt.try &.["url"]?.try &.as_s - - if !url - env.response.status_code = 404 - next - end - - url = URI.parse(url).full_path.not_nil! if local - url = "#{url}&title=#{title}" if title - - env.redirect url -end - -options "/videoplayback" do |env| - env.response.headers.delete("Content-Type") - env.response.headers["Access-Control-Allow-Origin"] = "*" - env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" - env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range" -end - -options "/videoplayback/*" do |env| - env.response.headers.delete("Content-Type") - env.response.headers["Access-Control-Allow-Origin"] = "*" - env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" - env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range" -end - -options "/api/manifest/dash/id/videoplayback" do |env| - env.response.headers.delete("Content-Type") - env.response.headers["Access-Control-Allow-Origin"] = "*" - env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" - env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range" -end - -options "/api/manifest/dash/id/videoplayback/*" do |env| - env.response.headers.delete("Content-Type") - env.response.headers["Access-Control-Allow-Origin"] = "*" - env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" - env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range" -end - -get "/videoplayback/*" do |env| - path = env.request.path - - path = path.lchop("/videoplayback/") - path = path.rchop("/") - - path = path.gsub(/mime\/\w+\/\w+/) do |mimetype| - mimetype = mimetype.split("/") - mimetype[0] + "/" + mimetype[1] + "%2F" + mimetype[2] - end - - path = path.split("/") - - raw_params = {} of String => Array(String) - path.each_slice(2) do |pair| - key, value = pair - value = URI.decode_www_form(value) - - if raw_params[key]? - raw_params[key] << value - else - raw_params[key] = [value] - end - end - - query_params = HTTP::Params.new(raw_params) - - env.response.headers["Access-Control-Allow-Origin"] = "*" - env.redirect "/videoplayback?#{query_params}" -end - -get "/videoplayback" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - query_params = env.params.query - - fvip = query_params["fvip"]? || "3" - mns = query_params["mn"]?.try &.split(",") - mns ||= [] of String - - if query_params["region"]? - region = query_params["region"] - query_params.delete("region") - end - - if query_params["host"]? && !query_params["host"].empty? - host = "https://#{query_params["host"]}" - query_params.delete("host") - else - host = "https://r#{fvip}---#{mns.pop}.googlevideo.com" - end - - url = "/videoplayback?#{query_params.to_s}" - - headers = HTTP::Headers.new - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end - - client = make_client(URI.parse(host), region) - response = HTTP::Client::Response.new(500) - error = "" - 5.times do - begin - response = client.head(url, headers) - - if response.headers["Location"]? - location = URI.parse(response.headers["Location"]) - env.response.headers["Access-Control-Allow-Origin"] = "*" - - new_host = "#{location.scheme}://#{location.host}" - if new_host != host - host = new_host - client.close - client = make_client(URI.parse(new_host), region) - end - - url = "#{location.full_path}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" - else - break - end - rescue Socket::Addrinfo::Error - if !mns.empty? - mn = mns.pop - end - fvip = "3" - - host = "https://r#{fvip}---#{mn}.googlevideo.com" - client = make_client(URI.parse(host), region) - rescue ex - error = ex.message - end - end - - if response.status_code >= 400 - env.response.status_code = response.status_code - env.response.content_type = "text/plain" - next error - end - - if url.includes? "&file=seg.ts" - if CONFIG.disabled?("livestreams") - next error_template(403, "Administrator has disabled this endpoint.") - end - - begin - client.get(url, headers) do |response| - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if location = response.headers["Location"]? - location = URI.parse(location) - location = "#{location.full_path}&host=#{location.host}" - - if region - location += "®ion=#{region}" - end - - next env.redirect location - end - - IO.copy(response.body_io, env.response) - end - rescue ex - end - else - if query_params["title"]? && CONFIG.disabled?("downloads") || - CONFIG.disabled?("dash") - next error_template(403, "Administrator has disabled this endpoint.") - end - - content_length = nil - first_chunk = true - range_start, range_end = parse_range(env.request.headers["Range"]?) - chunk_start = range_start - chunk_end = range_end - - if !chunk_end || chunk_end - chunk_start > HTTP_CHUNK_SIZE - chunk_end = chunk_start + HTTP_CHUNK_SIZE - 1 - end - - # TODO: Record bytes written so we can restart after a chunk fails - while true - if !range_end && content_length - range_end = content_length - end - - if range_end && chunk_start > range_end - break - end - - if range_end && chunk_end > range_end - chunk_end = range_end - end - - headers["Range"] = "bytes=#{chunk_start}-#{chunk_end}" - - begin - client.get(url, headers) do |response| - if first_chunk - if !env.request.headers["Range"]? && response.status_code == 206 - env.response.status_code = 200 - else - env.response.status_code = response.status_code - end - - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) && key.downcase != "content-range" - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if location = response.headers["Location"]? - location = URI.parse(location) - location = "#{location.full_path}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" - - env.redirect location - break - end - - if title = query_params["title"]? - # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ - env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}" - end - - if !response.headers.includes_word?("Transfer-Encoding", "chunked") - content_length = response.headers["Content-Range"].split("/")[-1].to_i64 - if env.request.headers["Range"]? - env.response.headers["Content-Range"] = "bytes #{range_start}-#{range_end || (content_length - 1)}/#{content_length}" - env.response.content_length = ((range_end.try &.+ 1) || content_length) - range_start - else - env.response.content_length = content_length - end - end - end - - proxy_file(response, env) - end - rescue ex - if ex.message != "Error reading socket: Connection reset by peer" - break - else - client.close - client = make_client(URI.parse(host), region) - end - end - - chunk_start = chunk_end + 1 - chunk_end += HTTP_CHUNK_SIZE - first_chunk = false - end - end - client.close -end - get "/ggpht/*" do |env| url = env.request.path.lchop("/ggpht") @@ -3858,7 +1472,7 @@ end get "/watch_videos" do |env| response = YT_POOL.client &.get(env.request.resource) if url = response.headers["Location"]? - url = URI.parse(url).full_path + url = URI.parse(url).request_target next env.redirect url end @@ -3873,7 +1487,7 @@ error 404 do |env| response = YT_POOL.client &.get("/#{item}") if response.status_code == 301 - response = YT_POOL.client &.get(URI.parse(response.headers["Location"]).full_path) + response = YT_POOL.client &.get(URI.parse(response.headers["Location"]).request_target) end if response.body.empty? @@ -3934,4 +1548,5 @@ add_context_storage_type(User) Kemal.config.logger = LOGGER Kemal.config.host_binding = Kemal.config.host_binding != "0.0.0.0" ? Kemal.config.host_binding : CONFIG.host_binding Kemal.config.port = Kemal.config.port != 3000 ? Kemal.config.port : CONFIG.port +Kemal.config.app_name = "Invidious" Kemal.run diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr deleted file mode 100644 index 9986fe1b..00000000 --- a/src/invidious/channels.cr +++ /dev/null @@ -1,961 +0,0 @@ -struct InvidiousChannel - include DB::Serializable - - property id : String - property author : String - property updated : Time - property deleted : Bool - property subscribed : Time? -end - -struct ChannelVideo - include DB::Serializable - - property id : String - property title : String - property published : Time - property updated : Time - property ucid : String - property author : String - property length_seconds : Int32 = 0 - property live_now : Bool = false - property premiere_timestamp : Time? = nil - property views : Int64? = nil - - def to_json(locale, json : JSON::Builder) - json.object do - json.field "type", "shortVideo" - - json.field "title", self.title - json.field "videoId", self.id - json.field "videoThumbnails" do - generate_thumbnails(json, self.id) - end - - json.field "lengthSeconds", self.length_seconds - - json.field "author", self.author - json.field "authorId", self.ucid - json.field "authorUrl", "/channel/#{self.ucid}" - json.field "published", self.published.to_unix - json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) - - json.field "viewCount", self.views - end - end - - def to_json(locale, json : JSON::Builder | Nil = nil) - if json - to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end - end - end - - def to_xml(locale, query_params, xml : XML::Builder) - query_params["v"] = self.id - - xml.element("entry") do - xml.element("id") { xml.text "yt:video:#{self.id}" } - xml.element("yt:videoId") { xml.text self.id } - xml.element("yt:channelId") { xml.text self.ucid } - xml.element("title") { xml.text self.title } - xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}") - - xml.element("author") do - xml.element("name") { xml.text self.author } - xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } - end - - xml.element("content", type: "xhtml") do - xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do - xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do - xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg") - end - end - end - - xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") } - xml.element("updated") { xml.text self.updated.to_s("%Y-%m-%dT%H:%M:%S%:z") } - - xml.element("media:group") do - xml.element("media:title") { xml.text self.title } - xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg", - width: "320", height: "180") - end - end - end - - def to_xml(locale, xml : XML::Builder | Nil = nil) - if xml - to_xml(locale, xml) - else - XML.build do |xml| - to_xml(locale, xml) - end - end - end - - def to_tuple - {% begin %} - { - {{*@type.instance_vars.map { |var| var.name }}} - } - {% end %} - end -end - -struct AboutRelatedChannel - include DB::Serializable - - property ucid : String - property author : String - property author_url : String - property author_thumbnail : String -end - -# TODO: Refactor into either SearchChannel or InvidiousChannel -struct AboutChannel - include DB::Serializable - - property ucid : String - property author : String - property auto_generated : Bool - property author_url : String - property author_thumbnail : String - property banner : String? - property description_html : String - property paid : Bool - property total_views : Int64 - property sub_count : Int32 - property joined : Time - property is_family_friendly : Bool - property allowed_regions : Array(String) - property related_channels : Array(AboutRelatedChannel) - property tabs : Array(String) -end - -class ChannelRedirect < Exception - property channel_id : String - - def initialize(@channel_id) - end -end - -def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, max_threads = 10) - finished_channel = Channel(String | Nil).new - - spawn do - active_threads = 0 - active_channel = Channel(Nil).new - - channels.each do |ucid| - if active_threads >= max_threads - active_channel.receive - active_threads -= 1 - end - - active_threads += 1 - spawn do - begin - get_channel(ucid, db, refresh, pull_all_videos) - finished_channel.send(ucid) - rescue ex - finished_channel.send(nil) - ensure - active_channel.send(nil) - end - end - end - end - - final = [] of String - channels.size.times do - if ucid = finished_channel.receive - final << ucid - end - end - - return final -end - -def get_channel(id, db, refresh = true, pull_all_videos = true) - if channel = db.query_one?("SELECT * FROM channels WHERE id = $1", id, as: InvidiousChannel) - if refresh && Time.utc - channel.updated > 10.minutes - channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) - channel_array = channel.to_a - args = arg_array(channel_array) - - db.exec("INSERT INTO channels VALUES (#{args}) \ - ON CONFLICT (id) DO UPDATE SET author = $2, updated = $3", args: channel_array) - end - else - channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) - channel_array = channel.to_a - args = arg_array(channel_array) - - db.exec("INSERT INTO channels VALUES (#{args})", args: channel_array) - end - - return channel -end - -def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) - LOGGER.debug("fetch_channel: #{ucid}") - LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}, locale = #{locale}") - - LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed") - rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body - LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed") - rss = XML.parse_html(rss) - - author = rss.xpath_node(%q(//feed/title)) - if !author - raise InfoException.new("Deleted or invalid channel") - end - author = author.content - - # Auto-generated channels - # https://support.google.com/youtube/answer/2579942 - if author.ends_with?(" - Topic") || - {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? author - auto_generated = true - end - - LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}") - - page = 1 - - LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page") - response = get_channel_videos_response(ucid, page, auto_generated: auto_generated) - - videos = [] of SearchVideo - begin - initial_data = JSON.parse(response.body).as_a.find &.["response"]? - raise InfoException.new("Could not extract channel JSON") if !initial_data - - LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel videos page initial_data") - videos = extract_videos(initial_data.as_h, author, ucid) - rescue ex - if response.body.includes?("To continue with your YouTube experience, please fill out the form below.") || - response.body.includes?("https://www.google.com/sorry/index") - raise InfoException.new("Could not extract channel info. Instance is likely blocked.") - end - raise ex - end - - LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed") - rss.xpath_nodes("//feed/entry").each do |entry| - video_id = entry.xpath_node("videoid").not_nil!.content - title = entry.xpath_node("title").not_nil!.content - published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) - updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) - author = entry.xpath_node("author/name").not_nil!.content - ucid = entry.xpath_node("channelid").not_nil!.content - views = entry.xpath_node("group/community/statistics").try &.["views"]?.try &.to_i64? - views ||= 0_i64 - - channel_video = videos.select { |video| video.id == video_id }[0]? - - length_seconds = channel_video.try &.length_seconds - length_seconds ||= 0 - - live_now = channel_video.try &.live_now - live_now ||= false - - premiere_timestamp = channel_video.try &.premiere_timestamp - - video = ChannelVideo.new({ - id: video_id, - title: title, - published: published, - updated: Time.utc, - ucid: ucid, - author: author, - length_seconds: length_seconds, - live_now: live_now, - premiere_timestamp: premiere_timestamp, - views: views, - }) - - LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video") - - # We don't include the 'premiere_timestamp' here because channel pages don't include them, - # meaning the above timestamp is always null - was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ - ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ - updated = $4, ucid = $5, author = $6, length_seconds = $7, \ - live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) - - if was_insert - LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions") - db.exec("UPDATE users SET notifications = array_append(notifications, $1), \ - feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) - else - LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated") - end - end - - if pull_all_videos - page += 1 - - ids = [] of String - - loop do - response = get_channel_videos_response(ucid, page, auto_generated: auto_generated) - initial_data = JSON.parse(response.body).as_a.find &.["response"]? - raise InfoException.new("Could not extract channel JSON") if !initial_data - videos = extract_videos(initial_data.as_h, author, ucid) - - count = videos.size - videos = videos.map { |video| ChannelVideo.new({ - id: video.id, - title: video.title, - published: video.published, - updated: Time.utc, - ucid: video.ucid, - author: video.author, - length_seconds: video.length_seconds, - live_now: video.live_now, - premiere_timestamp: video.premiere_timestamp, - views: video.views, - }) } - - videos.each do |video| - ids << video.id - - # We are notified of Red videos elsewhere (PubSub), which includes a correct published date, - # so since they don't provide a published date here we can safely ignore them. - if Time.utc - video.published > 1.minute - was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ - ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ - updated = $4, ucid = $5, author = $6, length_seconds = $7, \ - live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) - - db.exec("UPDATE users SET notifications = array_append(notifications, $1), \ - feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert - end - end - - break if count < 25 - page += 1 - end - end - - channel = InvidiousChannel.new({ - id: ucid, - author: author, - updated: Time.utc, - deleted: false, - subscribed: nil, - }) - - return channel -end - -def fetch_channel_playlists(ucid, author, auto_generated, continuation, sort_by) - if continuation || auto_generated - url = produce_channel_playlists_url(ucid, continuation, sort_by, auto_generated) - - response = YT_POOL.client &.get(url) - - continuation = response.body.match(/"continuation":"(?<continuation>[^"]+)"/).try &.["continuation"]? - initial_data = JSON.parse(response.body).as_a.find(&.["response"]?).try &.as_h - else - url = "/channel/#{ucid}/playlists?flow=list&view=1" - - case sort_by - when "last", "last_added" - # - when "oldest", "oldest_created" - url += "&sort=da" - when "newest", "newest_created" - url += "&sort=dd" - else nil # Ignore - end - - response = YT_POOL.client &.get(url) - continuation = response.body.match(/"continuation":"(?<continuation>[^"]+)"/).try &.["continuation"]? - initial_data = extract_initial_data(response.body) - end - - return [] of SearchItem, nil if !initial_data - items = extract_items(initial_data) - continuation = extract_channel_playlists_cursor(continuation, auto_generated) if continuation - - return items, continuation -end - -def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:base64" => { - "2:string" => "videos", - "6:varint" => 2_i64, - "7:varint" => 1_i64, - "12:varint" => 1_i64, - "13:string" => "", - "23:varint" => 0_i64, - }, - }, - } - - if !v2 - if auto_generated - seed = Time.unix(1525757349) - until seed >= Time.utc - seed += 1.month - end - timestamp = seed - (page - 1).months - - object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x36_i64 - object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{timestamp.to_unix}" - else - object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0_i64 - object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{page}" - end - else - object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0_i64 - - object["80226972:embedded"]["3:base64"].as(Hash)["61:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json({ - "1:string" => Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json({ - "1:varint" => 30_i64 * (page - 1), - }))), - }))) - end - - case sort_by - when "newest" - when "popular" - object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 0x01_i64 - when "oldest" - object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 0x02_i64 - else nil # Ignore - end - - object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"]))) - object["80226972:embedded"].delete("3:base64") - - continuation = object.try { |i| Protodec::Any.cast_json(object) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" -end - -def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated = false) - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:base64" => { - "2:string" => "playlists", - "6:varint" => 2_i64, - "7:varint" => 1_i64, - "12:varint" => 1_i64, - "13:string" => "", - "23:varint" => 0_i64, - }, - }, - } - - if cursor - cursor = Base64.urlsafe_encode(cursor, false) if !auto_generated - object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = cursor - end - - if auto_generated - object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x32_i64 - else - object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 1_i64 - case sort - when "oldest", "oldest_created" - object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 2_i64 - when "newest", "newest_created" - object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 3_i64 - when "last", "last_added" - object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 4_i64 - else nil # Ignore - end - end - - object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"]))) - object["80226972:embedded"].delete("3:base64") - - continuation = object.try { |i| Protodec::Any.cast_json(object) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" -end - -def extract_channel_playlists_cursor(cursor, auto_generated) - cursor = URI.decode_www_form(cursor) - .try { |i| Base64.decode(i) } - .try { |i| IO::Memory.new(i) } - .try { |i| Protodec::Any.parse(i) } - .try { |i| i["80226972:0:embedded"]["3:1:base64"].as_h.find { |k, v| k.starts_with? "15:" } } - .try &.[1] - - if cursor.try &.as_h? - cursor = cursor.try { |i| Protodec::Any.cast_json(i.as_h) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } || "" - else - cursor = cursor.try &.as_s || "" - end - - if !auto_generated - cursor = URI.decode_www_form(cursor) - .try { |i| Base64.decode_string(i) } - end - - return cursor -end - -# TODO: Add "sort_by" -def fetch_channel_community(ucid, continuation, locale, format, thin_mode) - response = YT_POOL.client &.get("/channel/#{ucid}/community?gl=US&hl=en") - if response.status_code != 200 - response = YT_POOL.client &.get("/user/#{ucid}/community?gl=US&hl=en") - end - - if response.status_code != 200 - raise InfoException.new("This channel does not exist.") - end - - ucid = response.body.match(/https:\/\/www.youtube.com\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/).not_nil!["ucid"] - - if !continuation || continuation.empty? - initial_data = extract_initial_data(response.body) - body = initial_data["contents"]?.try &.["twoColumnBrowseResultsRenderer"]["tabs"].as_a.select { |tab| tab["tabRenderer"]?.try &.["selected"].as_bool.== true }[0]? - - if !body - raise InfoException.new("Could not extract community tab.") - end - - body = body["tabRenderer"]["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"] - else - continuation = produce_channel_community_continuation(ucid, continuation) - - headers = HTTP::Headers.new - headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"] - - session_token = response.body.match(/"XSRF_TOKEN":"(?<session_token>[^"]+)"/).try &.["session_token"]? || "" - post_req = { - session_token: session_token, - } - - response = YT_POOL.client &.post("/comment_service_ajax?action_get_comments=1&ctoken=#{continuation}&continuation=#{continuation}&hl=en&gl=US", headers, form: post_req) - body = JSON.parse(response.body) - - body = body["response"]["continuationContents"]["itemSectionContinuation"]? || - body["response"]["continuationContents"]["backstageCommentsContinuation"]? - - if !body - raise InfoException.new("Could not extract continuation.") - end - end - - continuation = body["continuations"]?.try &.[0]["nextContinuationData"]["continuation"].as_s - posts = body["contents"].as_a - - if message = posts[0]["messageRenderer"]? - error_message = (message["text"]["simpleText"]? || - message["text"]["runs"]?.try &.[0]?.try &.["text"]?) - .try &.as_s || "" - raise InfoException.new(error_message) - end - - response = JSON.build do |json| - json.object do - json.field "authorId", ucid - json.field "comments" do - json.array do - posts.each do |post| - comments = post["backstagePostThreadRenderer"]?.try &.["comments"]? || - post["backstageCommentsContinuation"]? - - post = post["backstagePostThreadRenderer"]?.try &.["post"]["backstagePostRenderer"]? || - post["commentThreadRenderer"]?.try &.["comment"]["commentRenderer"]? - - next if !post - - content_html = post["contentText"]?.try { |t| parse_content(t) } || "" - author = post["authorText"]?.try &.["simpleText"]? || "" - - json.object do - json.field "author", author - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - author_thumbnail = post["authorThumbnail"]["thumbnails"].as_a[0]["url"].as_s - - qualities.each do |quality| - json.object do - json.field "url", author_thumbnail.gsub(/s\d+-/, "s#{quality}-") - json.field "width", quality - json.field "height", quality - end - end - end - end - - if post["authorEndpoint"]? - json.field "authorId", post["authorEndpoint"]["browseEndpoint"]["browseId"] - json.field "authorUrl", post["authorEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s - else - json.field "authorId", "" - json.field "authorUrl", "" - end - - published_text = post["publishedTimeText"]["runs"][0]["text"].as_s - published = decode_date(published_text.rchop(" (edited)")) - - if published_text.includes?(" (edited)") - json.field "isEdited", true - else - json.field "isEdited", false - end - - like_count = post["actionButtons"]["commentActionButtonsRenderer"]["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"] - .try &.as_s.gsub(/\D/, "").to_i? || 0 - - json.field "content", html_to_content(content_html) - json.field "contentHtml", content_html - - json.field "published", published.to_unix - json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale)) - - json.field "likeCount", like_count - json.field "commentId", post["postId"]? || post["commentId"]? || "" - json.field "authorIsChannelOwner", post["authorEndpoint"]["browseEndpoint"]["browseId"] == ucid - - if attachment = post["backstageAttachment"]? - json.field "attachment" do - json.object do - case attachment.as_h - when .has_key?("videoRenderer") - attachment = attachment["videoRenderer"] - json.field "type", "video" - - if !attachment["videoId"]? - error_message = (attachment["title"]["simpleText"]? || - attachment["title"]["runs"]?.try &.[0]?.try &.["text"]?) - - json.field "error", error_message - else - video_id = attachment["videoId"].as_s - - video_title = attachment["title"]["simpleText"]? || attachment["title"]["runs"]?.try &.[0]?.try &.["text"]? - json.field "title", video_title - json.field "videoId", video_id - json.field "videoThumbnails" do - generate_thumbnails(json, video_id) - end - - json.field "lengthSeconds", decode_length_seconds(attachment["lengthText"]["simpleText"].as_s) - - author_info = attachment["ownerText"]["runs"][0].as_h - - json.field "author", author_info["text"].as_s - json.field "authorId", author_info["navigationEndpoint"]["browseEndpoint"]["browseId"] - json.field "authorUrl", author_info["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"] - - # TODO: json.field "authorThumbnails", "channelThumbnailSupportedRenderers" - # TODO: json.field "authorVerified", "ownerBadges" - - published = decode_date(attachment["publishedTimeText"]["simpleText"].as_s) - - json.field "published", published.to_unix - json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale)) - - view_count = attachment["viewCountText"]?.try &.["simpleText"].as_s.gsub(/\D/, "").to_i64? || 0_i64 - - json.field "viewCount", view_count - json.field "viewCountText", translate(locale, "`x` views", number_to_short_text(view_count)) - end - when .has_key?("backstageImageRenderer") - attachment = attachment["backstageImageRenderer"] - json.field "type", "image" - - json.field "imageThumbnails" do - json.array do - thumbnail = attachment["image"]["thumbnails"][0].as_h - width = thumbnail["width"].as_i - height = thumbnail["height"].as_i - aspect_ratio = (width.to_f / height.to_f) - url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640") - - qualities = {320, 560, 640, 1280, 2000} - - qualities.each do |quality| - json.object do - json.field "url", url.gsub(/=s\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", (quality / aspect_ratio).ceil.to_i - end - end - end - end - # TODO - # when .has_key?("pollRenderer") - # attachment = attachment["pollRenderer"] - # json.field "type", "poll" - else - json.field "type", "unknown" - json.field "error", "Unrecognized attachment type." - end - end - end - end - - if comments && (reply_count = (comments["backstageCommentsRenderer"]["moreText"]["simpleText"]? || - comments["backstageCommentsRenderer"]["moreText"]["runs"]?.try &.[0]?.try &.["text"]?) - .try &.as_s.gsub(/\D/, "").to_i?) - continuation = comments["backstageCommentsRenderer"]["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s - continuation ||= "" - - json.field "replies" do - json.object do - json.field "replyCount", reply_count - json.field "continuation", extract_channel_community_cursor(continuation) - end - end - end - end - end - end - end - - if body["continuations"]? - continuation = body["continuations"][0]["nextContinuationData"]["continuation"].as_s - json.field "continuation", extract_channel_community_cursor(continuation) - end - end - end - - if format == "html" - response = JSON.parse(response) - content_html = template_youtube_comments(response, locale, thin_mode) - - response = JSON.build do |json| - json.object do - json.field "contentHtml", content_html - end - end - end - - return response -end - -def produce_channel_community_continuation(ucid, cursor) - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:string" => cursor || "", - }, - } - - continuation = object.try { |i| Protodec::Any.cast_json(object) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return continuation -end - -def extract_channel_community_cursor(continuation) - object = URI.decode_www_form(continuation) - .try { |i| Base64.decode(i) } - .try { |i| IO::Memory.new(i) } - .try { |i| Protodec::Any.parse(i) } - .try { |i| i["80226972:0:embedded"]["3:1:base64"].as_h } - - if object["53:2:embedded"]?.try &.["3:0:embedded"]? - object["53:2:embedded"]["3:0:embedded"]["2:0:string"] = object["53:2:embedded"]["3:0:embedded"] - .try { |i| i["2:0:base64"].as_h } - .try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i, padding: false) } - - object["53:2:embedded"]["3:0:embedded"].as_h.delete("2:0:base64") - end - - cursor = Protodec::Any.cast_json(object) - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - - cursor -end - -def get_about_info(ucid, locale) - result = YT_POOL.client &.get("/channel/#{ucid}/about?gl=US&hl=en") - if result.status_code != 200 - result = YT_POOL.client &.get("/user/#{ucid}/about?gl=US&hl=en") - end - - if md = result.headers["location"]?.try &.match(/\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/) - raise ChannelRedirect.new(channel_id: md["ucid"]) - end - - if result.status_code != 200 - raise InfoException.new("This channel does not exist.") - end - - about = XML.parse_html(result.body) - if about.xpath_node(%q(//div[contains(@class, "channel-empty-message")])) - raise InfoException.new("This channel does not exist.") - end - - initdata = extract_initial_data(result.body) - if initdata.empty? - error_message = about.xpath_node(%q(//div[@class="yt-alert-content"])).try &.content.strip - error_message ||= translate(locale, "Could not get channel info.") - raise InfoException.new(error_message) - end - - if browse_endpoint = initdata["onResponseReceivedActions"]?.try &.[0]?.try &.["navigateAction"]?.try &.["endpoint"]?.try &.["browseEndpoint"]? - raise ChannelRedirect.new(channel_id: browse_endpoint["browseId"].to_s) - end - - author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s - author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s - author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s - - ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s - - # Raises a KeyError on failure. - banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? - banner = banners.try &.[-1]?.try &.["url"].as_s? - - # if banner.includes? "channels/c4/default_banner" - # banner = nil - # end - - description = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]?.try &.as_s? || "" - description_html = HTML.escape(description).gsub("\n", "<br>") - - paid = about.xpath_node(%q(//meta[@itemprop="paid"])).not_nil!["content"] == "True" - is_family_friendly = about.xpath_node(%q(//meta[@itemprop="isFamilyFriendly"])).not_nil!["content"] == "True" - allowed_regions = about.xpath_node(%q(//meta[@itemprop="regionsAllowed"])).not_nil!["content"].split(",") - - related_channels = initdata["contents"]["twoColumnBrowseResultsRenderer"] - .["secondaryContents"]?.try &.["browseSecondaryContentsRenderer"]["contents"][0]? - .try &.["verticalChannelSectionRenderer"]?.try &.["items"]?.try &.as_a.map do |node| - renderer = node["miniChannelRenderer"]? - related_id = renderer.try &.["channelId"]?.try &.as_s? - related_id ||= "" - - related_title = renderer.try &.["title"]?.try &.["simpleText"]?.try &.as_s? - related_title ||= "" - - related_author_url = renderer.try &.["navigationEndpoint"]?.try &.["commandMetadata"]?.try &.["webCommandMetadata"]? - .try &.["url"]?.try &.as_s? - related_author_url ||= "" - - related_author_thumbnails = renderer.try &.["thumbnail"]?.try &.["thumbnails"]?.try &.as_a? - related_author_thumbnails ||= [] of JSON::Any - - related_author_thumbnail = "" - if related_author_thumbnails.size > 0 - related_author_thumbnail = related_author_thumbnails[-1]["url"]?.try &.as_s? - related_author_thumbnail ||= "" - end - - AboutRelatedChannel.new({ - ucid: related_id, - author: related_title, - author_url: related_author_url, - author_thumbnail: related_author_thumbnail, - }) - end - related_channels ||= [] of AboutRelatedChannel - - total_views = 0_i64 - joined = Time.unix(0) - tabs = [] of String - auto_generated = false - - tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?.try &.as_a? - if !tabs_json.nil? - # Retrieve information from the tabs array. The index we are looking for varies between channels. - tabs_json.each do |node| - # Try to find the about section which is located in only one of the tabs. - channel_about_meta = node["tabRenderer"]?.try &.["content"]?.try &.["sectionListRenderer"]? - .try &.["contents"]?.try &.[0]?.try &.["itemSectionRenderer"]?.try &.["contents"]? - .try &.[0]?.try &.["channelAboutFullMetadataRenderer"]? - - if !channel_about_meta.nil? - total_views = channel_about_meta["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 - - # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. - joined = channel_about_meta["joinedDateText"]?.try &.["runs"]?.try &.as_a.reduce("") { |acc, node| acc + node["text"].as_s } - .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) - - # Auto-generated channels - # https://support.google.com/youtube/answer/2579942 - # For auto-generated channels, channel_about_meta only has ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] - if (channel_about_meta["primaryLinks"]?.try &.size || 0) == 1 && (channel_about_meta["primaryLinks"][0]?) && - (channel_about_meta["primaryLinks"][0]["title"]?.try &.["simpleText"]?.try &.as_s? || "") == "Auto-generated by YouTube" - auto_generated = true - end - end - end - tabs = tabs_json.reject { |node| node["tabRenderer"]?.nil? }.map { |node| node["tabRenderer"]["title"].as_s.downcase } - end - - sub_count = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s? - .try { |text| short_text_to_number(text.split(" ")[0]) } || 0 - - AboutChannel.new({ - ucid: ucid, - author: author, - auto_generated: auto_generated, - author_url: author_url, - author_thumbnail: author_thumbnail, - banner: banner, - description_html: description_html, - paid: paid, - total_views: total_views, - sub_count: sub_count, - joined: joined, - is_family_friendly: is_family_friendly, - allowed_regions: allowed_regions, - related_channels: related_channels, - tabs: tabs, - }) -end - -def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest") - url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated, sort_by: sort_by, v2: true) - return YT_POOL.client &.get(url) -end - -def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") - videos = [] of SearchVideo - - 2.times do |i| - response = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by) - initial_data = JSON.parse(response.body).as_a.find &.["response"]? - break if !initial_data - videos.concat extract_videos(initial_data.as_h, author, ucid) - end - - return videos.size, videos -end - -def get_latest_videos(ucid) - response = get_channel_videos_response(ucid, 1) - initial_data = JSON.parse(response.body).as_a.find &.["response"]? - return [] of SearchVideo if !initial_data - author = initial_data["response"]?.try &.["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s - items = extract_videos(initial_data.as_h, author, ucid) - - return items -end diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr new file mode 100644 index 00000000..628d5b6f --- /dev/null +++ b/src/invidious/channels/about.cr @@ -0,0 +1,174 @@ +# TODO: Refactor into either SearchChannel or InvidiousChannel +struct AboutChannel + include DB::Serializable + + property ucid : String + property author : String + property auto_generated : Bool + property author_url : String + property author_thumbnail : String + property banner : String? + property description_html : String + property total_views : Int64 + property sub_count : Int32 + property joined : Time + property is_family_friendly : Bool + property allowed_regions : Array(String) + property related_channels : Array(AboutRelatedChannel) + property tabs : Array(String) +end + +struct AboutRelatedChannel + include DB::Serializable + + property ucid : String + property author : String + property author_url : String + property author_thumbnail : String +end + +def get_about_info(ucid, locale) + begin + # "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"} + initdata = YoutubeAPI.browse(browse_id: ucid, params: "EgVhYm91dA==") + rescue + raise InfoException.new("Could not get channel info.") + end + + if initdata.dig?("alerts", 0, "alertRenderer", "type") == "ERROR" + raise InfoException.new(initdata["alerts"][0]["alertRenderer"]["text"]["simpleText"].as_s) + end + + if browse_endpoint = initdata["onResponseReceivedActions"]?.try &.[0]?.try &.["navigateAction"]?.try &.["endpoint"]?.try &.["browseEndpoint"]? + raise ChannelRedirect.new(channel_id: browse_endpoint["browseId"].to_s) + end + + auto_generated = false + # Check for special auto generated gaming channels + if !initdata.has_key?("metadata") + auto_generated = true + end + + if auto_generated + author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s + author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s + author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s + + # Raises a KeyError on failure. + banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? + banner = banners.try &.[-1]?.try &.["url"].as_s? + + description = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"]["simpleText"].as_s + description_html = HTML.escape(description).gsub("\n", "<br>") + + is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool + allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map { |a| a.as_s } + + related_channels = [] of AboutRelatedChannel + else + author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s + author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s + author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s + + ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s + + # Raises a KeyError on failure. + banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? + banner = banners.try &.[-1]?.try &.["url"].as_s? + + # if banner.includes? "channels/c4/default_banner" + # banner = nil + # end + + description = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]?.try &.as_s? || "" + description_html = HTML.escape(description).gsub("\n", "<br>") + + is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool + allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map { |a| a.as_s } + + related_channels = initdata["contents"]["twoColumnBrowseResultsRenderer"] + .["secondaryContents"]?.try &.["browseSecondaryContentsRenderer"]["contents"][0]? + .try &.["verticalChannelSectionRenderer"]?.try &.["items"]?.try &.as_a.map do |node| + renderer = node["miniChannelRenderer"]? + related_id = renderer.try &.["channelId"]?.try &.as_s? + related_id ||= "" + + related_title = renderer.try &.["title"]?.try &.["simpleText"]?.try &.as_s? + related_title ||= "" + + related_author_url = renderer.try &.["navigationEndpoint"]?.try &.["commandMetadata"]?.try &.["webCommandMetadata"]? + .try &.["url"]?.try &.as_s? + related_author_url ||= "" + + related_author_thumbnails = renderer.try &.["thumbnail"]?.try &.["thumbnails"]?.try &.as_a? + related_author_thumbnails ||= [] of JSON::Any + + related_author_thumbnail = "" + if related_author_thumbnails.size > 0 + related_author_thumbnail = related_author_thumbnails[-1]["url"]?.try &.as_s? + related_author_thumbnail ||= "" + end + + AboutRelatedChannel.new({ + ucid: related_id, + author: related_title, + author_url: related_author_url, + author_thumbnail: related_author_thumbnail, + }) + end + related_channels ||= [] of AboutRelatedChannel + end + + total_views = 0_i64 + joined = Time.unix(0) + + tabs = [] of String + + tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?.try &.as_a? + if !tabs_json.nil? + # Retrieve information from the tabs array. The index we are looking for varies between channels. + tabs_json.each do |node| + # Try to find the about section which is located in only one of the tabs. + channel_about_meta = node["tabRenderer"]?.try &.["content"]?.try &.["sectionListRenderer"]? + .try &.["contents"]?.try &.[0]?.try &.["itemSectionRenderer"]?.try &.["contents"]? + .try &.[0]?.try &.["channelAboutFullMetadataRenderer"]? + + if !channel_about_meta.nil? + total_views = channel_about_meta["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 + + # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. + joined = channel_about_meta["joinedDateText"]?.try &.["runs"]?.try &.as_a.reduce("") { |acc, node| acc + node["text"].as_s } + .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) + + # Normal Auto-generated channels + # https://support.google.com/youtube/answer/2579942 + # For auto-generated channels, channel_about_meta only has ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] + if (channel_about_meta["primaryLinks"]?.try &.size || 0) == 1 && (channel_about_meta["primaryLinks"][0]?) && + (channel_about_meta["primaryLinks"][0]["title"]?.try &.["simpleText"]?.try &.as_s? || "") == "Auto-generated by YouTube" + auto_generated = true + end + end + end + tabs = tabs_json.reject { |node| node["tabRenderer"]?.nil? }.map { |node| node["tabRenderer"]["title"].as_s.downcase } + end + + sub_count = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s? + .try { |text| short_text_to_number(text.split(" ")[0]) } || 0 + + AboutChannel.new({ + ucid: ucid, + author: author, + auto_generated: auto_generated, + author_url: author_url, + author_thumbnail: author_thumbnail, + banner: banner, + description_html: description_html, + total_views: total_views, + sub_count: sub_count, + joined: joined, + is_family_friendly: is_family_friendly, + allowed_regions: allowed_regions, + related_channels: related_channels, + tabs: tabs, + }) +end diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr new file mode 100644 index 00000000..70623cc0 --- /dev/null +++ b/src/invidious/channels/channels.cr @@ -0,0 +1,311 @@ +struct InvidiousChannel + include DB::Serializable + + property id : String + property author : String + property updated : Time + property deleted : Bool + property subscribed : Time? +end + +struct ChannelVideo + include DB::Serializable + + property id : String + property title : String + property published : Time + property updated : Time + property ucid : String + property author : String + property length_seconds : Int32 = 0 + property live_now : Bool = false + property premiere_timestamp : Time? = nil + property views : Int64? = nil + + def to_json(locale, json : JSON::Builder) + json.object do + json.field "type", "shortVideo" + + json.field "title", self.title + json.field "videoId", self.id + json.field "videoThumbnails" do + generate_thumbnails(json, self.id) + end + + json.field "lengthSeconds", self.length_seconds + + json.field "author", self.author + json.field "authorId", self.ucid + json.field "authorUrl", "/channel/#{self.ucid}" + json.field "published", self.published.to_unix + json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) + + json.field "viewCount", self.views + end + end + + def to_json(locale, json : JSON::Builder | Nil = nil) + if json + to_json(locale, json) + else + JSON.build do |json| + to_json(locale, json) + end + end + end + + def to_xml(locale, query_params, xml : XML::Builder) + query_params["v"] = self.id + + xml.element("entry") do + xml.element("id") { xml.text "yt:video:#{self.id}" } + xml.element("yt:videoId") { xml.text self.id } + xml.element("yt:channelId") { xml.text self.ucid } + xml.element("title") { xml.text self.title } + xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}") + + xml.element("author") do + xml.element("name") { xml.text self.author } + xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } + end + + xml.element("content", type: "xhtml") do + xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do + xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do + xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg") + end + end + end + + xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") } + xml.element("updated") { xml.text self.updated.to_s("%Y-%m-%dT%H:%M:%S%:z") } + + xml.element("media:group") do + xml.element("media:title") { xml.text self.title } + xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg", + width: "320", height: "180") + end + end + end + + def to_xml(locale, xml : XML::Builder | Nil = nil) + if xml + to_xml(locale, xml) + else + XML.build do |xml| + to_xml(locale, xml) + end + end + end + + def to_tuple + {% begin %} + { + {{*@type.instance_vars.map { |var| var.name }}} + } + {% end %} + end +end + +class ChannelRedirect < Exception + property channel_id : String + + def initialize(@channel_id) + end +end + +def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, max_threads = 10) + finished_channel = Channel(String | Nil).new + + spawn do + active_threads = 0 + active_channel = Channel(Nil).new + + channels.each do |ucid| + if active_threads >= max_threads + active_channel.receive + active_threads -= 1 + end + + active_threads += 1 + spawn do + begin + get_channel(ucid, db, refresh, pull_all_videos) + finished_channel.send(ucid) + rescue ex + finished_channel.send(nil) + ensure + active_channel.send(nil) + end + end + end + end + + final = [] of String + channels.size.times do + if ucid = finished_channel.receive + final << ucid + end + end + + return final +end + +def get_channel(id, db, refresh = true, pull_all_videos = true) + if channel = db.query_one?("SELECT * FROM channels WHERE id = $1", id, as: InvidiousChannel) + if refresh && Time.utc - channel.updated > 10.minutes + channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) + channel_array = channel.to_a + args = arg_array(channel_array) + + db.exec("INSERT INTO channels VALUES (#{args}) \ + ON CONFLICT (id) DO UPDATE SET author = $2, updated = $3", args: channel_array) + end + else + channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) + channel_array = channel.to_a + args = arg_array(channel_array) + + db.exec("INSERT INTO channels VALUES (#{args})", args: channel_array) + end + + return channel +end + +def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) + LOGGER.debug("fetch_channel: #{ucid}") + LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}, locale = #{locale}") + + LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed") + rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body + LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed") + rss = XML.parse_html(rss) + + author = rss.xpath_node(%q(//feed/title)) + if !author + raise InfoException.new("Deleted or invalid channel") + end + + author = author.content + + # Auto-generated channels + # https://support.google.com/youtube/answer/2579942 + if author.ends_with?(" - Topic") || + {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? author + auto_generated = true + end + + LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}") + + page = 1 + + LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page") + initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated) + videos = extract_videos(initial_data, author, ucid) + + LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed") + rss.xpath_nodes("//feed/entry").each do |entry| + video_id = entry.xpath_node("videoid").not_nil!.content + title = entry.xpath_node("title").not_nil!.content + published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) + updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) + author = entry.xpath_node("author/name").not_nil!.content + ucid = entry.xpath_node("channelid").not_nil!.content + views = entry.xpath_node("group/community/statistics").try &.["views"]?.try &.to_i64? + views ||= 0_i64 + + channel_video = videos.select { |video| video.id == video_id }[0]? + + length_seconds = channel_video.try &.length_seconds + length_seconds ||= 0 + + live_now = channel_video.try &.live_now + live_now ||= false + + premiere_timestamp = channel_video.try &.premiere_timestamp + + video = ChannelVideo.new({ + id: video_id, + title: title, + published: published, + updated: Time.utc, + ucid: ucid, + author: author, + length_seconds: length_seconds, + live_now: live_now, + premiere_timestamp: premiere_timestamp, + views: views, + }) + + LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video") + + # We don't include the 'premiere_timestamp' here because channel pages don't include them, + # meaning the above timestamp is always null + was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ + ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ + updated = $4, ucid = $5, author = $6, length_seconds = $7, \ + live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) + + if was_insert + LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions") + db.exec("UPDATE users SET notifications = array_append(notifications, $1), \ + feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) + else + LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated") + end + end + + if pull_all_videos + page += 1 + + ids = [] of String + + loop do + initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated) + videos = extract_videos(initial_data, author, ucid) + + count = videos.size + videos = videos.map { |video| ChannelVideo.new({ + id: video.id, + title: video.title, + published: video.published, + updated: Time.utc, + ucid: video.ucid, + author: video.author, + length_seconds: video.length_seconds, + live_now: video.live_now, + premiere_timestamp: video.premiere_timestamp, + views: video.views, + }) } + + videos.each do |video| + ids << video.id + + # We are notified of Red videos elsewhere (PubSub), which includes a correct published date, + # so since they don't provide a published date here we can safely ignore them. + if Time.utc - video.published > 1.minute + was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ + ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ + updated = $4, ucid = $5, author = $6, length_seconds = $7, \ + live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) + + db.exec("UPDATE users SET notifications = array_append(notifications, $1), \ + feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert + end + end + + break if count < 25 + page += 1 + end + end + + channel = InvidiousChannel.new({ + id: ucid, + author: author, + updated: Time.utc, + deleted: false, + subscribed: nil, + }) + + return channel +end diff --git a/src/invidious/channels/community.cr b/src/invidious/channels/community.cr new file mode 100644 index 00000000..97ab30ec --- /dev/null +++ b/src/invidious/channels/community.cr @@ -0,0 +1,275 @@ +# TODO: Add "sort_by" +def fetch_channel_community(ucid, continuation, locale, format, thin_mode) + response = YT_POOL.client &.get("/channel/#{ucid}/community?gl=US&hl=en") + if response.status_code != 200 + response = YT_POOL.client &.get("/user/#{ucid}/community?gl=US&hl=en") + end + + if response.status_code != 200 + raise InfoException.new("This channel does not exist.") + end + + ucid = response.body.match(/https:\/\/www.youtube.com\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/).not_nil!["ucid"] + + if !continuation || continuation.empty? + initial_data = extract_initial_data(response.body) + body = initial_data["contents"]?.try &.["twoColumnBrowseResultsRenderer"]["tabs"].as_a.select { |tab| tab["tabRenderer"]?.try &.["selected"].as_bool.== true }[0]? + + if !body + raise InfoException.new("Could not extract community tab.") + end + + body = body["tabRenderer"]["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"] + else + continuation = produce_channel_community_continuation(ucid, continuation) + + headers = HTTP::Headers.new + headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"] + + session_token = response.body.match(/"XSRF_TOKEN":"(?<session_token>[^"]+)"/).try &.["session_token"]? || "" + post_req = { + session_token: session_token, + } + + response = YT_POOL.client &.post("/comment_service_ajax?action_get_comments=1&ctoken=#{continuation}&continuation=#{continuation}&hl=en&gl=US", headers, form: post_req) + body = JSON.parse(response.body) + + body = body["response"]["continuationContents"]["itemSectionContinuation"]? || + body["response"]["continuationContents"]["backstageCommentsContinuation"]? + + if !body + raise InfoException.new("Could not extract continuation.") + end + end + + continuation = body["continuations"]?.try &.[0]["nextContinuationData"]["continuation"].as_s + posts = body["contents"].as_a + + if message = posts[0]["messageRenderer"]? + error_message = (message["text"]["simpleText"]? || + message["text"]["runs"]?.try &.[0]?.try &.["text"]?) + .try &.as_s || "" + raise InfoException.new(error_message) + end + + response = JSON.build do |json| + json.object do + json.field "authorId", ucid + json.field "comments" do + json.array do + posts.each do |post| + comments = post["backstagePostThreadRenderer"]?.try &.["comments"]? || + post["backstageCommentsContinuation"]? + + post = post["backstagePostThreadRenderer"]?.try &.["post"]["backstagePostRenderer"]? || + post["commentThreadRenderer"]?.try &.["comment"]["commentRenderer"]? + + next if !post + + content_html = post["contentText"]?.try { |t| parse_content(t) } || "" + author = post["authorText"]?.try &.["simpleText"]? || "" + + json.object do + json.field "author", author + json.field "authorThumbnails" do + json.array do + qualities = {32, 48, 76, 100, 176, 512} + author_thumbnail = post["authorThumbnail"]["thumbnails"].as_a[0]["url"].as_s + + qualities.each do |quality| + json.object do + json.field "url", author_thumbnail.gsub(/s\d+-/, "s#{quality}-") + json.field "width", quality + json.field "height", quality + end + end + end + end + + if post["authorEndpoint"]? + json.field "authorId", post["authorEndpoint"]["browseEndpoint"]["browseId"] + json.field "authorUrl", post["authorEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s + else + json.field "authorId", "" + json.field "authorUrl", "" + end + + published_text = post["publishedTimeText"]["runs"][0]["text"].as_s + published = decode_date(published_text.rchop(" (edited)")) + + if published_text.includes?(" (edited)") + json.field "isEdited", true + else + json.field "isEdited", false + end + + like_count = post["actionButtons"]["commentActionButtonsRenderer"]["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"] + .try &.as_s.gsub(/\D/, "").to_i? || 0 + + json.field "content", html_to_content(content_html) + json.field "contentHtml", content_html + + json.field "published", published.to_unix + json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale)) + + json.field "likeCount", like_count + json.field "commentId", post["postId"]? || post["commentId"]? || "" + json.field "authorIsChannelOwner", post["authorEndpoint"]["browseEndpoint"]["browseId"] == ucid + + if attachment = post["backstageAttachment"]? + json.field "attachment" do + json.object do + case attachment.as_h + when .has_key?("videoRenderer") + attachment = attachment["videoRenderer"] + json.field "type", "video" + + if !attachment["videoId"]? + error_message = (attachment["title"]["simpleText"]? || + attachment["title"]["runs"]?.try &.[0]?.try &.["text"]?) + + json.field "error", error_message + else + video_id = attachment["videoId"].as_s + + video_title = attachment["title"]["simpleText"]? || attachment["title"]["runs"]?.try &.[0]?.try &.["text"]? + json.field "title", video_title + json.field "videoId", video_id + json.field "videoThumbnails" do + generate_thumbnails(json, video_id) + end + + json.field "lengthSeconds", decode_length_seconds(attachment["lengthText"]["simpleText"].as_s) + + author_info = attachment["ownerText"]["runs"][0].as_h + + json.field "author", author_info["text"].as_s + json.field "authorId", author_info["navigationEndpoint"]["browseEndpoint"]["browseId"] + json.field "authorUrl", author_info["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"] + + # TODO: json.field "authorThumbnails", "channelThumbnailSupportedRenderers" + # TODO: json.field "authorVerified", "ownerBadges" + + published = decode_date(attachment["publishedTimeText"]["simpleText"].as_s) + + json.field "published", published.to_unix + json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale)) + + view_count = attachment["viewCountText"]?.try &.["simpleText"].as_s.gsub(/\D/, "").to_i64? || 0_i64 + + json.field "viewCount", view_count + json.field "viewCountText", translate(locale, "`x` views", number_to_short_text(view_count)) + end + when .has_key?("backstageImageRenderer") + attachment = attachment["backstageImageRenderer"] + json.field "type", "image" + + json.field "imageThumbnails" do + json.array do + thumbnail = attachment["image"]["thumbnails"][0].as_h + width = thumbnail["width"].as_i + height = thumbnail["height"].as_i + aspect_ratio = (width.to_f / height.to_f) + url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640") + + qualities = {320, 560, 640, 1280, 2000} + + qualities.each do |quality| + json.object do + json.field "url", url.gsub(/=s\d+/, "=s#{quality}") + json.field "width", quality + json.field "height", (quality / aspect_ratio).ceil.to_i + end + end + end + end + # TODO + # when .has_key?("pollRenderer") + # attachment = attachment["pollRenderer"] + # json.field "type", "poll" + else + json.field "type", "unknown" + json.field "error", "Unrecognized attachment type." + end + end + end + end + + if comments && (reply_count = (comments["backstageCommentsRenderer"]["moreText"]["simpleText"]? || + comments["backstageCommentsRenderer"]["moreText"]["runs"]?.try &.[0]?.try &.["text"]?) + .try &.as_s.gsub(/\D/, "").to_i?) + continuation = comments["backstageCommentsRenderer"]["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s + continuation ||= "" + + json.field "replies" do + json.object do + json.field "replyCount", reply_count + json.field "continuation", extract_channel_community_cursor(continuation) + end + end + end + end + end + end + end + + if body["continuations"]? + continuation = body["continuations"][0]["nextContinuationData"]["continuation"].as_s + json.field "continuation", extract_channel_community_cursor(continuation) + end + end + end + + if format == "html" + response = JSON.parse(response) + content_html = template_youtube_comments(response, locale, thin_mode) + + response = JSON.build do |json| + json.object do + json.field "contentHtml", content_html + end + end + end + + return response +end + +def produce_channel_community_continuation(ucid, cursor) + object = { + "80226972:embedded" => { + "2:string" => ucid, + "3:string" => cursor || "", + }, + } + + continuation = object.try { |i| Protodec::Any.cast_json(object) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + return continuation +end + +def extract_channel_community_cursor(continuation) + object = URI.decode_www_form(continuation) + .try { |i| Base64.decode(i) } + .try { |i| IO::Memory.new(i) } + .try { |i| Protodec::Any.parse(i) } + .try { |i| i["80226972:0:embedded"]["3:1:base64"].as_h } + + if object["53:2:embedded"]?.try &.["3:0:embedded"]? + object["53:2:embedded"]["3:0:embedded"]["2:0:string"] = object["53:2:embedded"]["3:0:embedded"] + .try { |i| i["2:0:base64"].as_h } + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i, padding: false) } + + object["53:2:embedded"]["3:0:embedded"].as_h.delete("2:0:base64") + end + + cursor = Protodec::Any.cast_json(object) + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + + cursor +end diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr new file mode 100644 index 00000000..393b055e --- /dev/null +++ b/src/invidious/channels/playlists.cr @@ -0,0 +1,93 @@ +def fetch_channel_playlists(ucid, author, continuation, sort_by) + if continuation + response_json = YoutubeAPI.browse(continuation) + continuationItems = response_json["onResponseReceivedActions"]? + .try &.[0]["appendContinuationItemsAction"]["continuationItems"] + + return [] of SearchItem, nil if !continuationItems + + items = [] of SearchItem + continuationItems.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item| + extract_item(item, author, ucid).try { |t| items << t } + } + + continuation = continuationItems.as_a.last["continuationItemRenderer"]? + .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s + else + url = "/channel/#{ucid}/playlists?flow=list&view=1" + + case sort_by + when "last", "last_added" + # + when "oldest", "oldest_created" + url += "&sort=da" + when "newest", "newest_created" + url += "&sort=dd" + else nil # Ignore + end + + response = YT_POOL.client &.get(url) + initial_data = extract_initial_data(response.body) + return [] of SearchItem, nil if !initial_data + + items = extract_items(initial_data, author, ucid) + continuation = response.body.match(/"token":"(?<continuation>[^"]+)"/).try &.["continuation"]? + end + + return items, continuation +end + +# ## NOTE: DEPRECATED +# Reason -> Unstable +# The Protobuf object must be provided with an id of the last playlist from the current "page" +# in order to fetch the next one accurately +# (if the id isn't included, entries shift around erratically between pages, +# leading to repetitions and skip overs) +# +# Since it's impossible to produce the appropriate Protobuf without an id being provided by the user, +# it's better to stick to continuation tokens provided by the first request and onward +def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated = false) + object = { + "80226972:embedded" => { + "2:string" => ucid, + "3:base64" => { + "2:string" => "playlists", + "6:varint" => 2_i64, + "7:varint" => 1_i64, + "12:varint" => 1_i64, + "13:string" => "", + "23:varint" => 0_i64, + }, + }, + } + + if cursor + cursor = Base64.urlsafe_encode(cursor, false) if !auto_generated + object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = cursor + end + + if auto_generated + object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x32_i64 + else + object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 1_i64 + case sort + when "oldest", "oldest_created" + object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 2_i64 + when "newest", "newest_created" + object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 3_i64 + when "last", "last_added" + object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 4_i64 + else nil # Ignore + end + end + + object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"]))) + object["80226972:embedded"].delete("3:base64") + + continuation = object.try { |i| Protodec::Any.cast_json(object) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" +end diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr new file mode 100644 index 00000000..2c43bf0b --- /dev/null +++ b/src/invidious/channels/videos.cr @@ -0,0 +1,89 @@ +def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) + object = { + "80226972:embedded" => { + "2:string" => ucid, + "3:base64" => { + "2:string" => "videos", + "6:varint" => 2_i64, + "7:varint" => 1_i64, + "12:varint" => 1_i64, + "13:string" => "", + "23:varint" => 0_i64, + }, + }, + } + + if !v2 + if auto_generated + seed = Time.unix(1525757349) + until seed >= Time.utc + seed += 1.month + end + timestamp = seed - (page - 1).months + + object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x36_i64 + object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{timestamp.to_unix}" + else + object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0_i64 + object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{page}" + end + else + object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0_i64 + + object["80226972:embedded"]["3:base64"].as(Hash)["61:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json({ + "1:string" => Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json({ + "1:varint" => 30_i64 * (page - 1), + }))), + }))) + end + + case sort_by + when "newest" + when "popular" + object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 0x01_i64 + when "oldest" + object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 0x02_i64 + else nil # Ignore + end + + object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"]))) + object["80226972:embedded"].delete("3:base64") + + continuation = object.try { |i| Protodec::Any.cast_json(object) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + return continuation +end + +def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest") + continuation = produce_channel_videos_continuation(ucid, page, + auto_generated: auto_generated, sort_by: sort_by, v2: true) + + return YoutubeAPI.browse(continuation) +end + +def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") + videos = [] of SearchVideo + + 2.times do |i| + initial_data = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by) + videos.concat extract_videos(initial_data, author, ucid) + end + + return videos.size, videos +end + +def get_latest_videos(ucid) + initial_data = get_channel_videos_response(ucid) + author = initial_data["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s + + return extract_videos(initial_data, author, ucid) +end + +# Used in bypass_captcha_job.cr +def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) + continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2) + return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" +end diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index 0ac99ba5..a5506b03 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -56,10 +56,7 @@ class RedditListing property modhash : String end -def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, sort_by = "top") - video = get_video(id, db, region: region) - session_token = video.session_token - +def fetch_youtube_comments(id, cursor, format, locale, thin_mode, region, sort_by = "top") case cursor when nil, "" ctoken = produce_comment_continuation(id, cursor: "", sort_by: sort_by) @@ -71,38 +68,41 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so ctoken = cursor end - if !session_token - if format == "json" - return {"comments" => [] of String}.to_json + client_config = YoutubeAPI::ClientConfig.new(region: region) + response = YoutubeAPI.next(continuation: ctoken, client_config: client_config) + contents = nil + + if response["onResponseReceivedEndpoints"]? + onResponseReceivedEndpoints = response["onResponseReceivedEndpoints"] + header = nil + onResponseReceivedEndpoints.as_a.each do |item| + if item["reloadContinuationItemsCommand"]? + case item["reloadContinuationItemsCommand"]["slot"] + when "RELOAD_CONTINUATION_SLOT_HEADER" + header = item["reloadContinuationItemsCommand"]["continuationItems"][0] + when "RELOAD_CONTINUATION_SLOT_BODY" + contents = item["reloadContinuationItemsCommand"]["continuationItems"] + end + elsif item["appendContinuationItemsAction"]? + contents = item["appendContinuationItemsAction"]["continuationItems"] + end + end + elsif response["continuationContents"]? + response = response["continuationContents"] + if response["commentRepliesContinuation"]? + body = response["commentRepliesContinuation"] else - return {"contentHtml" => "", "commentCount" => 0}.to_json + body = response["itemSectionContinuation"] + end + contents = body["contents"]? + header = body["header"]? + if body["continuations"]? + moreRepliesContinuation = body["continuations"][0]["nextContinuationData"]["continuation"].as_s end - end - - post_req = { - page_token: ctoken, - session_token: session_token, - } - - headers = HTTP::Headers{ - "cookie" => video.cookie, - } - - response = YT_POOL.client(region, &.post("/comment_service_ajax?action_get_comments=1&hl=en&gl=US&pbj=1", headers, form: post_req)) - response = JSON.parse(response.body) - - if !response["response"]["continuationContents"]? - raise InfoException.new("Could not fetch comments") - end - - response = response["response"]["continuationContents"] - if response["commentRepliesContinuation"]? - body = response["commentRepliesContinuation"] else - body = response["itemSectionContinuation"] + raise InfoException.new("Could not fetch comments") end - contents = body["contents"]? if !contents if format == "json" return {"comments" => [] of String}.to_json @@ -111,13 +111,20 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so end end + continuationItemRenderer = nil + contents.as_a.reject! do |item| + if item["continuationItemRenderer"]? + continuationItemRenderer = item["continuationItemRenderer"] + true + end + end + response = JSON.build do |json| json.object do - if body["header"]? - count_text = body["header"]["commentsHeaderRenderer"]["countText"] + if header + count_text = header["commentsHeaderRenderer"]["countText"] comment_count = (count_text["simpleText"]? || count_text["runs"]?.try &.[0]?.try &.["text"]?) .try &.as_s.gsub(/\D/, "").to_i? || 0 - json.field "commentCount", comment_count end @@ -127,7 +134,7 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so json.array do contents.as_a.each do |node| json.object do - if !response["commentRepliesContinuation"]? + if node["commentThreadRenderer"]? node = node["commentThreadRenderer"] end @@ -135,7 +142,7 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so node_replies = node["replies"]["commentRepliesRenderer"] end - if !response["commentRepliesContinuation"]? + if node["comment"]? node_comment = node["comment"]["commentRenderer"] else node_comment = node["commentRenderer"] @@ -180,12 +187,14 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so json.field "published", published.to_unix json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale)) - json.field "likeCount", node_comment["likeCount"] + comment_action_buttons_renderer = node_comment["actionButtons"]["commentActionButtonsRenderer"] + + json.field "likeCount", comment_action_buttons_renderer["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"].as_s.scan(/\d/).map(&.[0]).join.to_i json.field "commentId", node_comment["commentId"] json.field "authorIsChannelOwner", node_comment["authorIsChannelOwner"] - if node_comment["actionButtons"]["commentActionButtonsRenderer"]["creatorHeart"]? - hearth_data = node_comment["actionButtons"]["commentActionButtonsRenderer"]["creatorHeart"]["creatorHeartRenderer"]["creatorThumbnail"] + if comment_action_buttons_renderer["creatorHeart"]? + hearth_data = comment_action_buttons_renderer["creatorHeart"]["creatorHeartRenderer"]["creatorThumbnail"] json.field "creatorHeart" do json.object do json.field "creatorThumbnail", hearth_data["thumbnails"][-1]["url"] @@ -195,10 +204,20 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so end if node_replies && !response["commentRepliesContinuation"]? - reply_count = (node_replies["moreText"]["simpleText"]? || node_replies["moreText"]["runs"]?.try &.[0]?.try &.["text"]?) - .try &.as_s.gsub(/\D/, "").to_i? || 1 + if node_replies["moreText"]? + reply_count = (node_replies["moreText"]["simpleText"]? || node_replies["moreText"]["runs"]?.try &.[0]?.try &.["text"]?) + .try &.as_s.gsub(/\D/, "").to_i? || 1 + elsif node_replies["viewReplies"]? + reply_count = node_replies["viewReplies"]["buttonRenderer"]["text"]?.try &.["runs"][1]?.try &.["text"]?.try &.as_s.to_i? || 1 + else + reply_count = 1 + end - continuation = node_replies["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s + if node_replies["continuations"]? + continuation = node_replies["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s + elsif node_replies["contents"]? + continuation = node_replies["contents"]?.try &.as_a[0]["continuationItemRenderer"]["continuationEndpoint"]["continuationCommand"]["token"].as_s + end continuation ||= "" json.field "replies" do @@ -213,9 +232,15 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so end end - if body["continuations"]? - continuation = body["continuations"][0]["nextContinuationData"]["continuation"].as_s - json.field "continuation", cursor.try &.starts_with?("E") ? continuation : extract_comment_cursor(continuation) + if continuationItemRenderer + if continuationItemRenderer["continuationEndpoint"]? + continuationEndpoint = continuationItemRenderer["continuationEndpoint"] + elsif continuationItemRenderer["button"]? + continuationEndpoint = continuationItemRenderer["button"]["buttonRenderer"]["command"] + end + if continuationEndpoint + json.field "continuation", continuationEndpoint["continuationCommand"]["token"].as_s + end end end end @@ -275,7 +300,7 @@ def fetch_reddit_comments(id, sort_by = "confidence") return comments, thread end -def template_youtube_comments(comments, locale, thin_mode) +def template_youtube_comments(comments, locale, thin_mode, is_replies = false) String.build do |html| root = comments["comments"].as_a root.each do |child| @@ -286,7 +311,7 @@ def template_youtube_comments(comments, locale, thin_mode) <div class="pure-u-23-24"> <p> <a href="javascript:void(0)" data-continuation="#{child["replies"]["continuation"]}" - data-onclick="get_youtube_replies">#{translate(locale, "View `x` replies", number_with_separator(child["replies"]["replyCount"]))}</a> + data-onclick="get_youtube_replies" data-load-replies>#{translate(locale, "View `x` replies", number_with_separator(child["replies"]["replyCount"]))}</a> </p> </div> </div> @@ -294,20 +319,22 @@ def template_youtube_comments(comments, locale, thin_mode) end if !thin_mode - author_thumbnail = "/ggpht#{URI.parse(child["authorThumbnails"][-1]["url"].as_s).full_path}" + author_thumbnail = "/ggpht#{URI.parse(child["authorThumbnails"][-1]["url"].as_s).request_target}" else author_thumbnail = "" end + author_name = HTML.escape(child["author"].as_s) + html << <<-END_HTML <div class="pure-g" style="width:100%"> <div class="channel-profile pure-u-4-24 pure-u-md-2-24"> - <img style="padding-right:1em;padding-top:1em;width:90%" src="#{author_thumbnail}"> + <img style="margin-right:1em;margin-top:1em;width:90%" src="#{author_thumbnail}"> </div> <div class="pure-u-20-24 pure-u-md-22-24"> <p> <b> - <a class="#{child["authorIsChannelOwner"] == true ? "channel-owner" : ""}" href="#{child["authorUrl"]}">#{child["author"]}</a> + <a class="#{child["authorIsChannelOwner"] == true ? "channel-owner" : ""}" href="#{child["authorUrl"]}">#{author_name}</a> </b> <p style="white-space:pre-wrap">#{child["contentHtml"]}</p> END_HTML @@ -322,7 +349,7 @@ def template_youtube_comments(comments, locale, thin_mode) html << <<-END_HTML <div class="pure-g"> <div class="pure-u-1 pure-u-md-1-2"> - <img style="width:100%" src="/ggpht#{URI.parse(attachment["url"].as_s).full_path}"> + <img style="width:100%" src="/ggpht#{URI.parse(attachment["url"].as_s).request_target}"> </div> </div> END_HTML @@ -375,7 +402,7 @@ def template_youtube_comments(comments, locale, thin_mode) if child["creatorHeart"]? if !thin_mode - creator_thumbnail = "/ggpht#{URI.parse(child["creatorHeart"]["creatorThumbnail"].as_s).full_path}" + creator_thumbnail = "/ggpht#{URI.parse(child["creatorHeart"]["creatorThumbnail"].as_s).request_target}" else creator_thumbnail = "" end @@ -406,7 +433,7 @@ def template_youtube_comments(comments, locale, thin_mode) <div class="pure-u-1"> <p> <a href="javascript:void(0)" data-continuation="#{comments["continuation"]}" - data-onclick="get_youtube_replies" data-load-more>#{translate(locale, "Load more")}</a> + data-onclick="get_youtube_replies" data-load-more #{"data-load-replies" if is_replies}>#{translate(locale, "Load more")}</a> </p> </div> </div> @@ -468,12 +495,16 @@ def replace_links(html) html.xpath_nodes(%q(//a)).each do |anchor| url = URI.parse(anchor["href"]) - if {"www.youtube.com", "m.youtube.com", "youtu.be"}.includes?(url.host) - if url.path == "/redirect" - params = HTTP::Params.parse(url.query.not_nil!) - anchor["href"] = params["q"]? + if url.host.nil? || url.host.not_nil!.ends_with?("youtube.com") || url.host.not_nil!.ends_with?("youtu.be") + if url.host.try &.ends_with? "youtu.be" + url = "/watch?v=#{url.path.lstrip('/')}#{url.query_params}" else - anchor["href"] = url.full_path + if url.path == "/redirect" + params = HTTP::Params.parse(url.query.not_nil!) + anchor["href"] = params["q"]? + else + anchor["href"] = url.request_target + end end elsif url.to_s == "#" begin @@ -482,8 +513,12 @@ def replace_links(html) length_seconds = decode_time(anchor.content) end - anchor["href"] = "javascript:void(0)" - anchor["onclick"] = "player.currentTime(#{length_seconds})" + if length_seconds > 0 + anchor["href"] = "javascript:void(0)" + anchor["onclick"] = "player.currentTime(#{length_seconds})" + else + anchor["href"] = url.request_target + end end end @@ -522,11 +557,7 @@ end def content_to_comment_html(content) comment_html = content.map do |run| - text = HTML.escape(run["text"].as_s) - - if run["text"] == "\n" - text = "<br>" - end + text = HTML.escape(run["text"].as_s).gsub("\n", "<br>") if run["bold"]? text = "<b>#{text}</b>" @@ -540,11 +571,13 @@ def content_to_comment_html(content) if url = run["navigationEndpoint"]["urlEndpoint"]?.try &.["url"].as_s url = URI.parse(url) - if !url.host || {"m.youtube.com", "www.youtube.com", "youtu.be"}.includes? url.host + if url.host == "youtu.be" + url = "/watch?v=#{url.request_target.lstrip('/')}" + elsif url.host.nil? || url.host.not_nil!.ends_with?("youtube.com") if url.path == "/redirect" url = HTTP::Params.parse(url.query.not_nil!)["q"] else - url = url.full_path + url = url.request_target end end @@ -553,12 +586,12 @@ def content_to_comment_html(content) length_seconds = watch_endpoint["startTimeSeconds"]? video_id = watch_endpoint["videoId"].as_s - if length_seconds + if length_seconds && length_seconds.as_i > 0 text = %(<a href="javascript:void(0)" data-onclick="jump_to_time" data-jump-time="#{length_seconds}">#{text}</a>) else text = %(<a href="/watch?v=#{video_id}">#{text}</a>) end - elsif url = run["navigationEndpoint"]["commandMetadata"]?.try &.["webCommandMetadata"]["url"].as_s + elsif url = run.dig?("navigationEndpoint", "commandMetadata", "webCommandMetadata", "url").try &.as_s text = %(<a href="#{url}">#{text}</a>) end end @@ -569,16 +602,6 @@ def content_to_comment_html(content) return comment_html end -def extract_comment_cursor(continuation) - cursor = URI.decode_www_form(continuation) - .try { |i| Base64.decode(i) } - .try { |i| IO::Memory.new(i) } - .try { |i| Protodec::Any.parse(i) } - .try { |i| i["6:2:embedded"]["1:0:string"].as_s } - - return cursor -end - def produce_comment_continuation(video_id, cursor = "", sort_by = "top") object = { "2:embedded" => { diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr new file mode 100644 index 00000000..bf56d826 --- /dev/null +++ b/src/invidious/helpers/crystal_class_overrides.cr @@ -0,0 +1,70 @@ +# Override of the TCPSocket and HTTP::Client classes in order to allow an +# IP family to be selected for domains that resolve to both IPv4 and +# IPv6 addresses. +# +class TCPSocket + def initialize(host : String, port, dns_timeout = nil, connect_timeout = nil, family = Socket::Family::UNSPEC) + Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo| + super(addrinfo.family, addrinfo.type, addrinfo.protocol) + connect(addrinfo, timeout: connect_timeout) do |error| + close + error + end + end + end +end + +# :ditto: +class HTTP::Client + property family : Socket::Family = Socket::Family::UNSPEC + + private def io + io = @io + return io if io + unless @reconnect + raise "This HTTP::Client cannot be reconnected" + end + + hostname = @host.starts_with?('[') && @host.ends_with?(']') ? @host[1..-2] : @host + io = TCPSocket.new hostname, @port, @dns_timeout, @connect_timeout, @family + io.read_timeout = @read_timeout if @read_timeout + io.write_timeout = @write_timeout if @write_timeout + io.sync = false + + {% if !flag?(:without_openssl) %} + if tls = @tls + tcp_socket = io + begin + io = OpenSSL::SSL::Socket::Client.new(tcp_socket, context: tls, sync_close: true, hostname: @host) + rescue exc + # don't leak the TCP socket when the SSL connection failed + tcp_socket.close + raise exc + end + end + {% end %} + + @io = io + end +end + +# Mute the ClientError exception raised when a connection is flushed. +# This happends when the connection is unexpectedly closed by the client. +# +class HTTP::Server::Response + class Output + private def unbuffered_flush + @io.flush + rescue ex : IO::Error + unbuffered_close + end + end +end + +# TODO: Document this override +# +class PG::ResultSet + def field(index = @column_index) + @fields.not_nil![index] + end +end diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr index 2c62d44b..e1d02563 100644 --- a/src/invidious/helpers/errors.cr +++ b/src/invidious/helpers/errors.cr @@ -7,7 +7,7 @@ class InfoException < Exception end macro error_template(*args) - error_template_helper(env, config, locale, {{*args}}) + error_template_helper(env, locale, {{*args}}) end def github_details(summary : String, content : String) @@ -22,9 +22,9 @@ def github_details(summary : String, content : String) return HTML.escape(details) end -def error_template_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception) +def error_template_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception) if exception.is_a?(InfoException) - return error_template_helper(env, config, locale, status_code, exception.message || "") + return error_template_helper(env, locale, status_code, exception.message || "") end env.response.content_type = "text/html" env.response.status_code = status_code @@ -40,42 +40,46 @@ def error_template_helper(env : HTTP::Server::Context, config : Config, locale : and include the following text in your message: <pre style="padding: 20px; background: rgba(0, 0, 0, 0.12345);">#{issue_template}</pre> END_HTML + + next_steps = error_redirect_helper(env, locale) + return templated "error" end -def error_template_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String) +def error_template_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String) env.response.content_type = "text/html" env.response.status_code = status_code error_message = translate(locale, message) + next_steps = error_redirect_helper(env, locale) return templated "error" end macro error_atom(*args) - error_atom_helper(env, config, locale, {{*args}}) + error_atom_helper(env, locale, {{*args}}) end -def error_atom_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception) +def error_atom_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception) if exception.is_a?(InfoException) - return error_atom_helper(env, config, locale, status_code, exception.message || "") + return error_atom_helper(env, locale, status_code, exception.message || "") end env.response.content_type = "application/atom+xml" env.response.status_code = status_code return "<error>#{exception.inspect_with_backtrace}</error>" end -def error_atom_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String) +def error_atom_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String) env.response.content_type = "application/atom+xml" env.response.status_code = status_code return "<error>#{message}</error>" end macro error_json(*args) - error_json_helper(env, config, locale, {{*args}}) + error_json_helper(env, locale, {{*args}}) end -def error_json_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception, additional_fields : Hash(String, Object) | Nil) +def error_json_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception, additional_fields : Hash(String, Object) | Nil) if exception.is_a?(InfoException) - return error_json_helper(env, config, locale, status_code, exception.message || "", additional_fields) + return error_json_helper(env, locale, status_code, exception.message || "", additional_fields) end env.response.content_type = "application/json" env.response.status_code = status_code @@ -86,11 +90,11 @@ def error_json_helper(env : HTTP::Server::Context, config : Config, locale : Has return error_message.to_json end -def error_json_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception) - return error_json_helper(env, config, locale, status_code, exception, nil) +def error_json_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception) + return error_json_helper(env, locale, status_code, exception, nil) end -def error_json_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String, additional_fields : Hash(String, Object) | Nil) +def error_json_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String, additional_fields : Hash(String, Object) | Nil) env.response.content_type = "application/json" env.response.status_code = status_code error_message = {"error" => message} @@ -100,6 +104,37 @@ def error_json_helper(env : HTTP::Server::Context, config : Config, locale : Has return error_message.to_json end -def error_json_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String) - error_json_helper(env, config, locale, status_code, message, nil) +def error_json_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String) + error_json_helper(env, locale, status_code, message, nil) +end + +def error_redirect_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil) + request_path = env.request.path + + if request_path.starts_with?("/search") || request_path.starts_with?("/watch") || + request_path.starts_with?("/channel") || request_path.starts_with?("/playlist?list=PL") + next_steps_text = translate(locale, "next_steps_error_message") + refresh = translate(locale, "next_steps_error_message_refresh") + go_to_youtube = translate(locale, "next_steps_error_message_go_to_youtube") + switch_instance = translate(locale, "Switch Invidious Instance") + + return <<-END_HTML + <p style="margin-bottom: 4px;">#{next_steps_text}</p> + <ul> + <li> + <a href="#{env.request.resource}">#{refresh}</a> + </li> + <li> + <a href="/redirect?referer=#{env.get("current_page")}">#{switch_instance}</a> + </li> + <li> + <a href="https://youtube.com#{env.request.resource}">#{go_to_youtube}</a> + </li> + </ul> + END_HTML + + return next_step_html + else + return "" + end end diff --git a/src/invidious/helpers/extractors.cr b/src/invidious/helpers/extractors.cr new file mode 100644 index 00000000..850c93ec --- /dev/null +++ b/src/invidious/helpers/extractors.cr @@ -0,0 +1,566 @@ +# This file contains helper methods to parse the Youtube API json data into +# neat little packages we can use + +# Tuple of Parsers/Extractors so we can easily cycle through them. +private ITEM_CONTAINER_EXTRACTOR = { + Extractors::YouTubeTabs, + Extractors::SearchResults, + Extractors::Continuation, +} + +private ITEM_PARSERS = { + Parsers::VideoRendererParser, + Parsers::ChannelRendererParser, + Parsers::GridPlaylistRendererParser, + Parsers::PlaylistRendererParser, + Parsers::CategoryRendererParser, +} + +record AuthorFallback, name : String, id : String + +# Namespace for logic relating to parsing InnerTube data into various datastructs. +# +# Each of the parsers in this namespace are accessed through the #process() method +# which validates the given data as applicable to itself. If it is applicable the given +# data is passed to the private `#parse()` method which returns a datastruct of the given +# type. Otherwise, nil is returned. +private module Parsers + # Parses a InnerTube videoRenderer into a SearchVideo. Returns nil when the given object isn't a videoRenderer + # + # A videoRenderer renders a video to click on within the YouTube and Invidious UI. It is **not** + # the watchable video itself. + # + # See specs for example. + # + # `videoRenderer`s can be found almost everywhere on YouTube. In categories, search results, channels, etc. + # + module VideoRendererParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = (item["videoRenderer"]? || item["gridVideoRenderer"]?) + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + video_id = item_contents["videoId"].as_s + title = extract_text(item_contents["title"]) || "" + + # Extract author information + if author_info = item_contents.dig?("ownerText", "runs", 0) + author = author_info["text"].as_s + author_id = HelperExtractors.get_browse_id(author_info) + else + author = author_fallback.name + author_id = author_fallback.id + end + + # For live videos (and possibly recently premiered videos) there is no published information. + # Instead, in its place is the amount of people currently watching. This behavior should be replicated + # on Invidious once all features of livestreams are supported. On an unrelated note, defaulting to the current + # time for publishing isn't a good idea. + published = item_contents.dig?("publishedTimeText", "simpleText").try { |t| decode_date(t.as_s) } || Time.local + + # Typically views are stored under a "simpleText" in the "viewCountText". However, for + # livestreams and premiered it is stored under a "runs" array: [{"text":123}, {"text": "watching"}] + # When view count is disabled the "viewCountText" is not present on InnerTube data. + # TODO change default value to nil and typical encoding type to tuple storing type (watchers, views, etc) + # and count + view_count = item_contents.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64 + description_html = item_contents["descriptionSnippet"]?.try { |t| parse_content(t) } || "" + + # The length information *should* only always exist in "lengthText". However, the legacy Invidious code + # extracts from "thumbnailOverlays" when it doesn't. More testing is needed to see if this is + # actually needed + if length_container = item_contents["lengthText"]? + length_seconds = decode_length_seconds(length_container["simpleText"].as_s) + elsif length_container = item_contents["thumbnailOverlays"]?.try &.as_a.find(&.["thumbnailOverlayTimeStatusRenderer"]?) + length_seconds = extract_text(length_container["thumbnailOverlayTimeStatusRenderer"]["text"]).try { |t| decode_length_seconds(t) } || 0 + else + length_seconds = 0 + end + + live_now = false + paid = false + premium = false + + premiere_timestamp = item_contents.dig?("upcomingEventData", "startTime").try { |t| Time.unix(t.as_s.to_i64) } + + item_contents["badges"]?.try &.as_a.each do |badge| + b = badge["metadataBadgeRenderer"] + case b["label"].as_s + when "LIVE NOW" + live_now = true + when "New", "4K", "CC" + # TODO + when "Premium" + # TODO: Potentially available as item_contents["topStandaloneBadge"]["metadataBadgeRenderer"] + premium = true + else nil # Ignore + end + end + + SearchVideo.new({ + title: title, + id: video_id, + author: author, + ucid: author_id, + published: published, + views: view_count, + description_html: description_html, + length_seconds: length_seconds, + live_now: live_now, + premium: premium, + premiere_timestamp: premiere_timestamp, + }) + end + end + + # Parses a InnerTube channelRenderer into a SearchChannel. Returns nil when the given object isn't a channelRenderer + # + # A channelRenderer renders a channel to click on within the YouTube and Invidious UI. It is **not** + # the channel page itself. + # + # See specs for example. + # + # `channelRenderer`s can be found almost everywhere on YouTube. In categories, search results, channels, etc. + # + module ChannelRendererParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = (item["channelRenderer"]? || item["gridChannelRenderer"]?) + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + author = extract_text(item_contents["title"]) || author_fallback.name + author_id = item_contents["channelId"]?.try &.as_s || author_fallback.id + + author_thumbnail = HelperExtractors.get_thumbnails(item_contents) + # When public subscriber count is disabled, the subscriberCountText isn't sent by InnerTube. + # Always simpleText + # TODO change default value to nil + subscriber_count = item_contents.dig?("subscriberCountText", "simpleText") + .try { |s| short_text_to_number(s.as_s.split(" ")[0]) } || 0 + + # Auto-generated channels doesn't have videoCountText + # Taken from: https://github.com/iv-org/invidious/pull/2228#discussion_r717620922 + auto_generated = item_contents["videoCountText"]?.nil? + + video_count = HelperExtractors.get_video_count(item_contents) + description_html = item_contents["descriptionSnippet"]?.try { |t| parse_content(t) } || "" + + SearchChannel.new({ + author: author, + ucid: author_id, + author_thumbnail: author_thumbnail, + subscriber_count: subscriber_count, + video_count: video_count, + description_html: description_html, + auto_generated: auto_generated, + }) + end + end + + # Parses a InnerTube gridPlaylistRenderer into a SearchPlaylist. Returns nil when the given object isn't a gridPlaylistRenderer + # + # A gridPlaylistRenderer renders a playlist, that is located in a grid, to click on within the YouTube and Invidious UI. + # It is **not** the playlist itself. + # + # See specs for example. + # + # `gridPlaylistRenderer`s can be found on the playlist-tabs of channels and expanded categories. + # + module GridPlaylistRendererParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["gridPlaylistRenderer"]? + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + title = extract_text(item_contents["title"]) || "" + plid = item_contents["playlistId"]?.try &.as_s || "" + + video_count = HelperExtractors.get_video_count(item_contents) + playlist_thumbnail = HelperExtractors.get_thumbnails(item_contents) + + SearchPlaylist.new({ + title: title, + id: plid, + author: author_fallback.name, + ucid: author_fallback.id, + video_count: video_count, + videos: [] of SearchPlaylistVideo, + thumbnail: playlist_thumbnail, + }) + end + end + + # Parses a InnerTube playlistRenderer into a SearchPlaylist. Returns nil when the given object isn't a playlistRenderer + # + # A playlistRenderer renders a playlist to click on within the YouTube and Invidious UI. It is **not** the playlist itself. + # + # See specs for example. + # + # `playlistRenderer`s can be found almost everywhere on YouTube. In categories, search results, recommended, etc. + # + module PlaylistRendererParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["playlistRenderer"]? + return self.parse(item_contents) + end + end + + private def self.parse(item_contents) + title = item_contents["title"]["simpleText"]?.try &.as_s || "" + plid = item_contents["playlistId"]?.try &.as_s || "" + + video_count = HelperExtractors.get_video_count(item_contents) + playlist_thumbnail = HelperExtractors.get_thumbnails_plural(item_contents) + + author_info = item_contents.dig("shortBylineText", "runs", 0) + author = author_info["text"].as_s + author_id = HelperExtractors.get_browse_id(author_info) + + videos = item_contents["videos"]?.try &.as_a.map do |v| + v = v["childVideoRenderer"] + v_title = v.dig?("title", "simpleText").try &.as_s || "" + v_id = v["videoId"]?.try &.as_s || "" + v_length_seconds = v.dig?("lengthText", "simpleText").try { |t| decode_length_seconds(t.as_s) } || 0 + SearchPlaylistVideo.new({ + title: v_title, + id: v_id, + length_seconds: v_length_seconds, + }) + end || [] of SearchPlaylistVideo + + # TODO: item_contents["publishedTimeText"]? + + SearchPlaylist.new({ + title: title, + id: plid, + author: author, + ucid: author_id, + video_count: video_count, + videos: videos, + thumbnail: playlist_thumbnail, + }) + end + end + + # Parses a InnerTube shelfRenderer into a Category. Returns nil when the given object isn't a shelfRenderer + # + # A shelfRenderer renders divided sections on YouTube. IE "People also watched" in search results and + # the various organizational sections in the channel home page. A separate one (richShelfRenderer) is used + # for YouTube home. A shelfRenderer can also sometimes be expanded to show more content within it. + # + # See specs for example. + # + # `shelfRenderer`s can be found almost everywhere on YouTube. In categories, search results, channels, etc. + # + module CategoryRendererParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["shelfRenderer"]? + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + title = extract_text(item_contents["title"]?) || "" + url = item_contents.dig?("endpoint", "commandMetadata", "webCommandMetadata", "url") + .try &.as_s + + # Sometimes a category can have badges. + badges = [] of Tuple(String, String) # (Badge style, label) + item_contents["badges"]?.try &.as_a.each do |badge| + badge = badge["metadataBadgeRenderer"] + badges << {badge["style"].as_s, badge["label"].as_s} + end + + # Category description + description_html = item_contents["subtitle"]?.try { |desc| parse_content(desc) } || "" + + # Content parsing + contents = [] of SearchItem + + # Content could be in three locations. + if content_container = item_contents["content"]["horizontalListRenderer"]? + elsif content_container = item_contents["content"]["expandedShelfContentsRenderer"]? + elsif content_container = item_contents["content"]["verticalListRenderer"]? + else + content_container = item_contents["contents"] + end + + raw_contents = content_container["items"].as_a + raw_contents.each do |item| + result = extract_item(item) + if !result.nil? + contents << result + end + end + + Category.new({ + title: title, + contents: contents, + description_html: description_html, + url: url, + badges: badges, + }) + end + end +end + +# The following are the extractors for extracting an array of items from +# the internal Youtube API's JSON response. The result is then packaged into +# a structure we can more easily use via the parsers above. Their internals are +# identical to the item parsers. + +# Namespace for logic relating to extracting InnerTube's initial response to items we can parse. +# +# Each of the extractors in this namespace are accessed through the #process() method +# which validates the given data as applicable to itself. If it is applicable the given +# data is passed to the private `#extract()` method which returns an array of +# parsable items. Otherwise, nil is returned. +# +# NOTE perhaps the result from here should be abstracted into a struct in order to +# get additional metadata regarding the container of the item(s). +private module Extractors + # Extracts items from the selected YouTube tab. + # + # YouTube tabs are typically stored under "twoColumnBrowseResultsRenderer" + # and is structured like this: + # + # "twoColumnBrowseResultsRenderer": { + # {"tabs": [ + # {"tabRenderer": { + # "endpoint": {...} + # "title": "Playlists", + # "selected": true, + # "content": {...}, + # ... + # }} + # ]} + # }] + # + module YouTubeTabs + def self.process(initial_data : Hash(String, JSON::Any)) + if target = initial_data["twoColumnBrowseResultsRenderer"]? + self.extract(target) + end + end + + private def self.extract(target) + raw_items = [] of JSON::Any + content = extract_selected_tab(target["tabs"])["content"] + + content["sectionListRenderer"]["contents"].as_a.each do |renderer_container| + renderer_container_contents = renderer_container["itemSectionRenderer"]["contents"][0] + + # Category extraction + if items_container = renderer_container_contents["shelfRenderer"]? + raw_items << renderer_container_contents + next + elsif items_container = renderer_container_contents["gridRenderer"]? + else + items_container = renderer_container_contents + end + + items_container["items"].as_a.each do |item| + raw_items << item + end + end + + return raw_items + end + end + + # Extracts items from the InnerTube response for search results + # + # Search results are typically stored under "twoColumnSearchResultsRenderer" + # and is structured like this: + # + # "twoColumnSearchResultsRenderer": { + # {"primaryContents": { + # {"sectionListRenderer": { + # "contents": [...], + # ..., + # "subMenu": {...}, + # "hideBottomSeparator": true, + # "targetId": "search-feed" + # }} + # }} + # } + # + module SearchResults + def self.process(initial_data : Hash(String, JSON::Any)) + if target = initial_data["twoColumnSearchResultsRenderer"]? + self.extract(target) + end + end + + private def self.extract(target) + raw_items = [] of Array(JSON::Any) + + target.dig("primaryContents", "sectionListRenderer", "contents").as_a.each do |node| + if node = node["itemSectionRenderer"]? + raw_items << node["contents"].as_a + end + end + + return raw_items.flatten + end + end + + # Extracts continuation items from a InnerTube response + # + # Continuation items (on YouTube) are items which are appended to the + # end of the page for continuous scrolling. As such, in many cases, + # the items are lacking information such as author or category title, + # since the original results has already rendered them on the top of the page. + # + # The way they are structured is too varied to be accurately written down here. + # However, they all eventually lead to an array of parsable items after traversing + # through the JSON structure. + module Continuation + def self.process(initial_data : Hash(String, JSON::Any)) + if target = initial_data["continuationContents"]? + self.extract(target) + elsif target = initial_data["appendContinuationItemsAction"]? + self.extract(target) + end + end + + private def self.extract(target) + raw_items = [] of JSON::Any + if content = target["gridContinuation"]? + raw_items = content["items"].as_a + elsif content = target["continuationItems"]? + raw_items = content.as_a + end + + return raw_items + end + end +end + +# Helper methods to aid in the parsing of InnerTube to data structs. +# +# Mostly used to extract out repeated structures to deal with code +# repetition. +private module HelperExtractors + # Retrieves the amount of videos present within the given InnerTube data. + # + # Returns a 0 when it's unable to do so + def self.get_video_count(container : JSON::Any) : Int32 + if box = container["videoCountText"]? + return extract_text(box).try &.gsub(/\D/, "").to_i || 0 + elsif box = container["videoCount"]? + return box.as_s.to_i + else + return 0 + end + end + + # Retrieve lowest quality thumbnail from InnerTube data + # + # TODO allow configuration of image quality (-1 is highest) + # + # Raises when it's unable to parse from the given JSON data. + def self.get_thumbnails(container : JSON::Any) : String + return container.dig("thumbnail", "thumbnails", 0, "url").as_s + end + + # ditto + # + # YouTube sometimes sends the thumbnail as: + # {"thumbnails": [{"thumbnails": [{"url": "example.com"}, ...]}]} + def self.get_thumbnails_plural(container : JSON::Any) : String + return container.dig("thumbnails", 0, "thumbnails", 0, "url").as_s + end + + # Retrieves the ID required for querying the InnerTube browse endpoint. + # Raises when it's unable to do so + def self.get_browse_id(container) + return container.dig("navigationEndpoint", "browseEndpoint", "browseId").as_s + end +end + +# Extracts text from InnerTube response +# +# InnerTube can package text in three different formats +# "runs": [ +# {"text": "something"}, +# {"text": "cont"}, +# ... +# ] +# +# "SimpleText": "something" +# +# Or sometimes just none at all as with the data returned from +# category continuations. +# +# In order to facilitate calling this function with `#[]?`: +# A nil will be accepted. Of course, since nil cannot be parsed, +# another nil will be returned. +def extract_text(item : JSON::Any?) : String? + if item.nil? + return nil + end + + if text_container = item["simpleText"]? + return text_container.as_s + elsif text_container = item["runs"]? + return text_container.as_a.map(&.["text"].as_s).join("") + else + nil + end +end + +# Parses an item from Youtube's JSON response into a more usable structure. +# The end result can either be a SearchVideo, SearchPlaylist or SearchChannel. +def extract_item(item : JSON::Any, author_fallback : String? = "", + author_id_fallback : String? = "") + # We "allow" nil values but secretly use empty strings instead. This is to save us the + # hassle of modifying every author_fallback and author_id_fallback arg usage + # which is more often than not nil. + author_fallback = AuthorFallback.new(author_fallback || "", author_id_fallback || "") + + # Cycles through all of the item parsers and attempt to parse the raw YT JSON data. + # Each parser automatically validates the data given to see if the data is + # applicable to itself. If not nil is returned and the next parser is attemped. + ITEM_PARSERS.each do |parser| + if result = parser.process(item, author_fallback) + return result + end + end +end + +# Parses multiple items from YouTube's initial JSON response into a more usable structure. +# The end result is an array of SearchItem. +def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, + author_id_fallback : String? = nil) : Array(SearchItem) + items = [] of SearchItem + + if unpackaged_data = initial_data["contents"]?.try &.as_h + elsif unpackaged_data = initial_data["response"]?.try &.as_h + elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 0).try &.as_h + else + unpackaged_data = initial_data + end + + # This is identical to the parser cycling of extract_item(). + ITEM_CONTAINER_EXTRACTOR.each do |extractor| + if container = extractor.process(unpackaged_data) + # Extract items in container + container.each do |item| + if parsed_result = extract_item(item, author_fallback, author_id_fallback) + items << parsed_result + end + end + + break + end + end + + return items +end diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 1f56ec92..99adcd30 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -42,15 +42,19 @@ struct ConfigPreferences property player_style : String = "invidious" property quality : String = "hd720" property quality_dash : String = "auto" - property default_home : String = "Popular" + property default_home : String? = "Popular" property feed_menu : Array(String) = ["Popular", "Trending", "Subscriptions", "Playlists"] + property automatic_instance_redirect : Bool = false property related_videos : Bool = true property sort : String = "published" property speed : Float32 = 1.0_f32 property thin_mode : Bool = false property unseen_only : Bool = false property video_loop : Bool = false + property extend_desc : Bool = false property volume : Int32 = 100 + property vr_mode : Bool = true + property show_nick : Bool = true def to_tuple {% begin %} @@ -64,11 +68,14 @@ end class Config include YAML::Serializable - property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions) - property feed_threads : Int32 = 1 # Number of threads to use for updating feeds - property output : String = "STDOUT" # Log file path or STDOUT - property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr - property db : DBConfig # Database configuration + property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions) + property feed_threads : Int32 = 1 # Number of threads to use for updating feeds + property output : String = "STDOUT" # Log file path or STDOUT + property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr + property db : DBConfig? = nil # Database configuration with separate parameters (username, hostname, etc) + + @[YAML::Field(converter: Preferences::URIConverter)] + property database_url : URI = URI.parse("") # Database configuration using 12-Factor "Database URL" syntax property decrypt_polling : Bool = true # Use polling to keep decryption function up to date property full_refresh : Bool = false # Used for crawling channels: threads should check all videos uploaded by a channel property https_only : Bool? # Used to tell Invidious it is behind a proxy, so links to resources should be https:// @@ -95,6 +102,7 @@ class Config property port : Int32 = 3000 # Port to listen for connections (overrided by command line argument) property host_binding : String = "0.0.0.0" # Host to bind (overrided by command line argument) property pool_size : Int32 = 100 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) + property use_quic : Bool = true # Use quic transport for youtube api @[YAML::Field(converter: Preferences::StringToCookies)] property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format @@ -115,6 +123,80 @@ class Config return false end end + + def self.load + # Load config from file or YAML string env var + env_config_file = "INVIDIOUS_CONFIG_FILE" + env_config_yaml = "INVIDIOUS_CONFIG" + + config_file = ENV.has_key?(env_config_file) ? ENV.fetch(env_config_file) : "config/config.yml" + config_yaml = ENV.has_key?(env_config_yaml) ? ENV.fetch(env_config_yaml) : File.read(config_file) + + config = Config.from_yaml(config_yaml) + + # Update config from env vars (upcased and prefixed with "INVIDIOUS_") + {% for ivar in Config.instance_vars %} + {% env_id = "INVIDIOUS_#{ivar.id.upcase}" %} + + if ENV.has_key?({{env_id}}) + # puts %(Config.{{ivar.id}} : Loading from env var {{env_id}}) + env_value = ENV.fetch({{env_id}}) + success = false + + # Use YAML converter if specified + {% ann = ivar.annotation(::YAML::Field) %} + {% if ann && ann[:converter] %} + puts %(Config.{{ivar.id}} : Parsing "#{env_value}" as {{ivar.type}} with {{ann[:converter]}} converter) + config.{{ivar.id}} = {{ann[:converter]}}.from_yaml(YAML::ParseContext.new, YAML::Nodes.parse(ENV.fetch({{env_id}})).nodes[0]) + puts %(Config.{{ivar.id}} : Set to #{config.{{ivar.id}}}) + success = true + + # Use regular YAML parser otherwise + {% else %} + {% ivar_types = ivar.type.union? ? ivar.type.union_types : [ivar.type] %} + # Sort types to avoid parsing nulls and numbers as strings + {% ivar_types = ivar_types.sort_by { |ivar_type| ivar_type == Nil ? 0 : ivar_type == Int32 ? 1 : 2 } %} + {{ivar_types}}.each do |ivar_type| + if !success + begin + # puts %(Config.{{ivar.id}} : Trying to parse "#{env_value}" as #{ivar_type}) + config.{{ivar.id}} = ivar_type.from_yaml(env_value) + puts %(Config.{{ivar.id}} : Set to #{config.{{ivar.id}}} (#{ivar_type})) + success = true + rescue + # nop + end + end + end + {% end %} + + # Exit on fail + if !success + puts %(Config.{{ivar.id}} failed to parse #{env_value} as {{ivar.type}}) + exit(1) + end + end + {% end %} + + # Build database_url from db.* if it's not set directly + if config.database_url.to_s.empty? + if db = config.db + config.database_url = URI.new( + scheme: "postgres", + user: db.user, + password: db.password, + host: db.host, + port: db.port, + path: db.dbname, + ) + else + puts "Config : Either database_url or db.* is required" + exit(1) + end + end + + return config + end end struct DBConfig @@ -166,173 +248,40 @@ def html_to_content(description_html : String) end def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) - extract_items(initial_data, author_fallback, author_id_fallback).select(&.is_a?(SearchVideo)).map(&.as(SearchVideo)) -end + extracted = extract_items(initial_data, author_fallback, author_id_fallback) -def extract_item(item : JSON::Any, author_fallback : String? = nil, author_id_fallback : String? = nil) - if i = (item["videoRenderer"]? || item["gridVideoRenderer"]?) - video_id = i["videoId"].as_s - title = i["title"].try { |t| t["simpleText"]?.try &.as_s || t["runs"]?.try &.as_a.map(&.["text"].as_s).join("") } || "" - - author_info = i["ownerText"]?.try &.["runs"].as_a[0]? - author = author_info.try &.["text"].as_s || author_fallback || "" - author_id = author_info.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]["browseId"].as_s || author_id_fallback || "" - - published = i["publishedTimeText"]?.try &.["simpleText"]?.try { |t| decode_date(t.as_s) } || Time.local - view_count = i["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64 - description_html = i["descriptionSnippet"]?.try { |t| parse_content(t) } || "" - length_seconds = i["lengthText"]?.try &.["simpleText"]?.try &.as_s.try { |t| decode_length_seconds(t) } || - i["thumbnailOverlays"]?.try &.as_a.find(&.["thumbnailOverlayTimeStatusRenderer"]?).try &.["thumbnailOverlayTimeStatusRenderer"]? - .try &.["text"]?.try &.["simpleText"]?.try &.as_s.try { |t| decode_length_seconds(t) } || 0 - - live_now = false - paid = false - premium = false - - premiere_timestamp = i["upcomingEventData"]?.try &.["startTime"]?.try { |t| Time.unix(t.as_s.to_i64) } - - i["badges"]?.try &.as_a.each do |badge| - b = badge["metadataBadgeRenderer"] - case b["label"].as_s - when "LIVE NOW" - live_now = true - when "New", "4K", "CC" - # TODO - when "Premium" - paid = true - - # TODO: Potentially available as i["topStandaloneBadge"]["metadataBadgeRenderer"] - premium = true - else nil # Ignore - end + target = [] of SearchItem + extracted.each do |i| + if i.is_a?(Category) + i.contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video } + else + target << i end - - SearchVideo.new({ - title: title, - id: video_id, - author: author, - ucid: author_id, - published: published, - views: view_count, - description_html: description_html, - length_seconds: length_seconds, - live_now: live_now, - paid: paid, - premium: premium, - premiere_timestamp: premiere_timestamp, - }) - elsif i = item["channelRenderer"]? - author = i["title"]["simpleText"]?.try &.as_s || author_fallback || "" - author_id = i["channelId"]?.try &.as_s || author_id_fallback || "" - - author_thumbnail = i["thumbnail"]["thumbnails"]?.try &.as_a[0]?.try { |u| "https:#{u["url"]}" } || "" - subscriber_count = i["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s.try { |s| short_text_to_number(s.split(" ")[0]) } || 0 - - auto_generated = false - auto_generated = true if !i["videoCountText"]? - video_count = i["videoCountText"]?.try &.["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0 - description_html = i["descriptionSnippet"]?.try { |t| parse_content(t) } || "" - - SearchChannel.new({ - author: author, - ucid: author_id, - author_thumbnail: author_thumbnail, - subscriber_count: subscriber_count, - video_count: video_count, - description_html: description_html, - auto_generated: auto_generated, - }) - elsif i = item["gridPlaylistRenderer"]? - title = i["title"]["runs"].as_a[0]?.try &.["text"].as_s || "" - plid = i["playlistId"]?.try &.as_s || "" - - video_count = i["videoCountText"]["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0 - playlist_thumbnail = i["thumbnail"]["thumbnails"][0]?.try &.["url"]?.try &.as_s || "" - - SearchPlaylist.new({ - title: title, - id: plid, - author: author_fallback || "", - ucid: author_id_fallback || "", - video_count: video_count, - videos: [] of SearchPlaylistVideo, - thumbnail: playlist_thumbnail, - }) - elsif i = item["playlistRenderer"]? - title = i["title"]["simpleText"]?.try &.as_s || "" - plid = i["playlistId"]?.try &.as_s || "" - - video_count = i["videoCount"]?.try &.as_s.to_i || 0 - playlist_thumbnail = i["thumbnails"].as_a[0]?.try &.["thumbnails"]?.try &.as_a[0]?.try &.["url"].as_s || "" - - author_info = i["shortBylineText"]?.try &.["runs"].as_a[0]? - author = author_info.try &.["text"].as_s || author_fallback || "" - author_id = author_info.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]["browseId"].as_s || author_id_fallback || "" - - videos = i["videos"]?.try &.as_a.map do |v| - v = v["childVideoRenderer"] - v_title = v["title"]["simpleText"]?.try &.as_s || "" - v_id = v["videoId"]?.try &.as_s || "" - v_length_seconds = v["lengthText"]?.try &.["simpleText"]?.try { |t| decode_length_seconds(t.as_s) } || 0 - SearchPlaylistVideo.new({ - title: v_title, - id: v_id, - length_seconds: v_length_seconds, - }) - end || [] of SearchPlaylistVideo - - # TODO: i["publishedTimeText"]? - - SearchPlaylist.new({ - title: title, - id: plid, - author: author, - ucid: author_id, - video_count: video_count, - videos: videos, - thumbnail: playlist_thumbnail, - }) - elsif i = item["radioRenderer"]? # Mix - # TODO - elsif i = item["showRenderer"]? # Show - # TODO - elsif i = item["shelfRenderer"]? - elsif i = item["horizontalCardListRenderer"]? - elsif i = item["searchPyvRenderer"]? # Ad end + return target.select(&.is_a?(SearchVideo)).map(&.as(SearchVideo)) end -def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) - items = [] of SearchItem +def extract_selected_tab(tabs) + # Extract the selected tab from the array of tabs Youtube returns + return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"].as_bool)[0]["tabRenderer"] +end - channel_v2_response = initial_data - .try &.["response"]? - .try &.["continuationContents"]? - .try &.["gridContinuation"]? - .try &.["items"]? +def fetch_continuation_token(items : Array(JSON::Any)) + # Fetches the continuation token from an array of items + return items.last["continuationItemRenderer"]? + .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s +end - if channel_v2_response - channel_v2_response.try &.as_a.each { |item| - extract_item(item, author_fallback, author_id_fallback) - .try { |t| items << t } - } +def fetch_continuation_token(initial_data : Hash(String, JSON::Any)) + # Fetches the continuation token from initial data + if initial_data["onResponseReceivedActions"]? + continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"] else - initial_data.try { |t| t["contents"]? || t["response"]? } - .try { |t| t["twoColumnBrowseResultsRenderer"]?.try &.["tabs"].as_a.select(&.["tabRenderer"]?.try &.["selected"].as_bool)[0]?.try &.["tabRenderer"]["content"] || - t["twoColumnSearchResultsRenderer"]?.try &.["primaryContents"] || - t["continuationContents"]? } - .try { |t| t["sectionListRenderer"]? || t["sectionListContinuation"]? } - .try &.["contents"].as_a - .each { |c| c.try &.["itemSectionRenderer"]?.try &.["contents"].as_a - .try { |t| t[0]?.try &.["shelfRenderer"]?.try &.["content"]["expandedShelfContentsRenderer"]?.try &.["items"].as_a || - t[0]?.try &.["gridRenderer"]?.try &.["items"].as_a || t } - .each { |item| - extract_item(item, author_fallback, author_id_fallback) - .try { |t| items << t } - } } + tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]) + continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"] end - items + return fetch_continuation_token(continuation_items.as_a) end def check_enum(db, enum_name, struct_type = nil) @@ -428,12 +377,6 @@ def check_table(db, table_name, struct_type = nil) end end -class PG::ResultSet - def field(index = @column_index) - @fields.not_nil![index] - end -end - def get_column_array(db, table_name) column_array = [] of String db.query("SELECT * FROM #{table_name} LIMIT 0") do |rs| @@ -602,7 +545,7 @@ def create_notification_stream(env, topics, connection_channel) end def extract_initial_data(body) : Hash(String, JSON::Any) - return JSON.parse(body.match(/(window\["ytInitialData"\]|var\s*ytInitialData)\s*=\s*(?<info>\{.*?\});/mx).try &.["info"] || "{}").as_h + return JSON.parse(body.match(/(window\["ytInitialData"\]|var\s*ytInitialData)\s*=\s*(?<info>{.*?});<\/script>/mx).try &.["info"] || "{}").as_h end def proxy_file(response, env) @@ -618,85 +561,3 @@ def proxy_file(response, env) IO.copy response.body_io, env.response end end - -# See https://github.com/kemalcr/kemal/pull/576 -class HTTP::Server::Response::Output - def close - return if closed? - - unless response.wrote_headers? - response.content_length = @out_count - end - - ensure_headers_written - - super - - if @chunked - @io << "0\r\n\r\n" - @io.flush - end - end -end - -class HTTP::Client::Response - def pipe(io) - HTTP.serialize_body(io, headers, @body, @body_io, @version) - end -end - -# Supports serialize_body without first writing headers -module HTTP - def self.serialize_body(io, headers, body, body_io, version) - if body - io << body - elsif body_io - content_length = content_length(headers) - if content_length - copied = IO.copy(body_io, io) - if copied != content_length - raise ArgumentError.new("Content-Length header is #{content_length} but body had #{copied} bytes") - end - elsif Client::Response.supports_chunked?(version) - headers["Transfer-Encoding"] = "chunked" - serialize_chunked_body(io, body_io) - else - io << body - end - end - end -end - -class HTTP::Client - property family : Socket::Family = Socket::Family::UNSPEC - - private def socket - socket = @socket - return socket if socket - - hostname = @host.starts_with?('[') && @host.ends_with?(']') ? @host[1..-2] : @host - socket = TCPSocket.new hostname, @port, @dns_timeout, @connect_timeout, @family - socket.read_timeout = @read_timeout if @read_timeout - socket.sync = false - - {% if !flag?(:without_openssl) %} - if tls = @tls - socket = OpenSSL::SSL::Socket::Client.new(socket, context: tls, sync_close: true, hostname: @host) - end - {% end %} - - @socket = socket - end -end - -class TCPSocket - def initialize(host, port, dns_timeout = nil, connect_timeout = nil, family = Socket::Family::UNSPEC) - Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo| - super(addrinfo.family, addrinfo.type, addrinfo.protocol) - connect(addrinfo, timeout: connect_timeout) do |error| - close - error - end - end - end -end diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index 0faa2e32..2ed4f150 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -1,3 +1,46 @@ +# "bn_BD" => load_locale("bn_BD"), # Bengali (Bangladesh) [Incomplete] +# "eu" => load_locale("eu"), # Basque [Incomplete] +# "si" => load_locale("si"), # Sinhala [Incomplete] +# "sk" => load_locale("sk"), # Slovak [Incomplete] +# "sr" => load_locale("sr"), # Serbian [Incomplete] +# "sr_Cyrl" => load_locale("sr_Cyrl"), # Serbian (cyrillic) [Incomplete] +LOCALES = { + "ar" => load_locale("ar"), # Arabic + "cs" => load_locale("cs"), # Czech + "da" => load_locale("da"), # Danish + "de" => load_locale("de"), # German + "el" => load_locale("el"), # Greek + "en-US" => load_locale("en-US"), # English (US) + "eo" => load_locale("eo"), # Esperanto + "es" => load_locale("es"), # Spanish + "fa" => load_locale("fa"), # Persian + "fi" => load_locale("fi"), # Finnish + "fr" => load_locale("fr"), # French + "he" => load_locale("he"), # Hebrew + "hr" => load_locale("hr"), # Croatian + "hu-HU" => load_locale("hu-HU"), # Hungarian + "id" => load_locale("id"), # Indonesian + "is" => load_locale("is"), # Icelandic + "it" => load_locale("it"), # Italian + "ja" => load_locale("ja"), # Japanese + "ko" => load_locale("ko"), # Korean + "lt" => load_locale("lt"), # Lithuanian + "nb-NO" => load_locale("nb-NO"), # Norwegian BokmÃ¥l + "nl" => load_locale("nl"), # Dutch + "pl" => load_locale("pl"), # Polish + "pt" => load_locale("pt"), # Portuguese + "pt-BR" => load_locale("pt-BR"), # Portuguese (Brazil) + "pt-PT" => load_locale("pt-PT"), # Portuguese (Portugal) + "ro" => load_locale("ro"), # Romanian + "ru" => load_locale("ru"), # Russian + "sv-SE" => load_locale("sv-SE"), # Swedish + "tr" => load_locale("tr"), # Turkish + "uk" => load_locale("uk"), # Ukrainian + "vi" => load_locale("vi"), # Vietnamese + "zh-CN" => load_locale("zh-CN"), # Chinese (Simplified) + "zh-TW" => load_locale("zh-TW"), # Chinese (Traditional) +} + def load_locale(name) return JSON.parse(File.read("locales/#{name}.json")).as_h end diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr index 7c5b0247..e2e50905 100644 --- a/src/invidious/helpers/logger.cr +++ b/src/invidious/helpers/logger.cr @@ -1,5 +1,3 @@ -require "logger" - enum LogLevel All = 0 Trace = 1 @@ -19,7 +17,19 @@ class Invidious::LogHandler < Kemal::BaseLogHandler elapsed_time = Time.measure { call_next(context) } elapsed_text = elapsed_text(elapsed_time) - info("#{context.response.status_code} #{context.request.method} #{context.request.resource} #{elapsed_text}") + # Default: full path with parameters + requested_url = context.request.resource + + # Try not to log search queries passed as GET parameters during normal use + # (They will still be logged if log level is 'Debug' or 'Trace') + if @level > LogLevel::Debug && ( + requested_url.downcase.includes?("search") || requested_url.downcase.includes?("q=") + ) + # Log only the path + requested_url = context.request.path + end + + info("#{context.response.status_code} #{context.request.method} #{requested_url} #{elapsed_text}") context end diff --git a/src/invidious/helpers/macros.cr b/src/invidious/helpers/macros.cr index 8b74bc86..75df1612 100644 --- a/src/invidious/helpers/macros.cr +++ b/src/invidious/helpers/macros.cr @@ -48,10 +48,20 @@ module JSON::Serializable end end -macro templated(filename, template = "template") +macro templated(filename, template = "template", navbar_search = true) + navbar_search = {{navbar_search}} render "src/invidious/views/#{{{filename}}}.ecr", "src/invidious/views/#{{{template}}}.ecr" end macro rendered(filename) render "src/invidious/views/#{{{filename}}}.ecr" end + +# Similar to Kemals halt method but works in a +# method. +macro haltf(env, status_code = 200, response = "") + {{env}}.response.status_code = {{status_code}} + {{env}}.response.print {{response}} + {{env}}.response.close + return +end diff --git a/src/invidious/helpers/proxy.cr b/src/invidious/helpers/proxy.cr index 7a42ef41..3418d887 100644 --- a/src/invidious/helpers/proxy.cr +++ b/src/invidious/helpers/proxy.cr @@ -71,14 +71,14 @@ end class HTTPClient < HTTP::Client def set_proxy(proxy : HTTPProxy) begin - @socket = proxy.open(host: @host, port: @port, tls: @tls, connection_options: proxy_connection_options) + @io = proxy.open(host: @host, port: @port, tls: @tls, connection_options: proxy_connection_options) rescue IO::Error - @socket = nil + @io = nil end end def unset_proxy - @socket = nil + @io = nil end def proxy_connection_options diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr new file mode 100644 index 00000000..61356555 --- /dev/null +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -0,0 +1,256 @@ +struct SearchVideo + include DB::Serializable + + property title : String + property id : String + property author : String + property ucid : String + property published : Time + property views : Int64 + property description_html : String + property length_seconds : Int32 + property live_now : Bool + property premium : Bool + property premiere_timestamp : Time? + + def to_xml(auto_generated, query_params, xml : XML::Builder) + query_params["v"] = self.id + + xml.element("entry") do + xml.element("id") { xml.text "yt:video:#{self.id}" } + xml.element("yt:videoId") { xml.text self.id } + xml.element("yt:channelId") { xml.text self.ucid } + xml.element("title") { xml.text self.title } + xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}") + + xml.element("author") do + if auto_generated + xml.element("name") { xml.text self.author } + xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } + else + xml.element("name") { xml.text author } + xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" } + end + end + + xml.element("content", type: "xhtml") do + xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do + xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do + xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg") + end + + xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text html_to_content(self.description_html) } + end + end + + xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") } + + xml.element("media:group") do + xml.element("media:title") { xml.text self.title } + xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg", + width: "320", height: "180") + xml.element("media:description") { xml.text html_to_content(self.description_html) } + end + + xml.element("media:community") do + xml.element("media:statistics", views: self.views) + end + end + end + + def to_xml(auto_generated, query_params, xml : XML::Builder | Nil = nil) + if xml + to_xml(HOST_URL, auto_generated, query_params, xml) + else + XML.build do |json| + to_xml(HOST_URL, auto_generated, query_params, xml) + end + end + end + + def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder) + json.object do + json.field "type", "video" + json.field "title", self.title + json.field "videoId", self.id + + json.field "author", self.author + json.field "authorId", self.ucid + json.field "authorUrl", "/channel/#{self.ucid}" + + json.field "videoThumbnails" do + generate_thumbnails(json, self.id) + end + + json.field "description", html_to_content(self.description_html) + json.field "descriptionHtml", self.description_html + + json.field "viewCount", self.views + json.field "published", self.published.to_unix + json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) + json.field "lengthSeconds", self.length_seconds + json.field "liveNow", self.live_now + json.field "premium", self.premium + json.field "isUpcoming", self.is_upcoming + + if self.premiere_timestamp + json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix + end + end + end + + def to_json(locale, json : JSON::Builder | Nil = nil) + if json + to_json(locale, json) + else + JSON.build do |json| + to_json(locale, json) + end + end + end + + def is_upcoming + premiere_timestamp ? true : false + end +end + +struct SearchPlaylistVideo + include DB::Serializable + + property title : String + property id : String + property length_seconds : Int32 +end + +struct SearchPlaylist + include DB::Serializable + + property title : String + property id : String + property author : String + property ucid : String + property video_count : Int32 + property videos : Array(SearchPlaylistVideo) + property thumbnail : String? + + def to_json(locale, json : JSON::Builder) + json.object do + json.field "type", "playlist" + json.field "title", self.title + json.field "playlistId", self.id + json.field "playlistThumbnail", self.thumbnail + + json.field "author", self.author + json.field "authorId", self.ucid + json.field "authorUrl", "/channel/#{self.ucid}" + + json.field "videoCount", self.video_count + json.field "videos" do + json.array do + self.videos.each do |video| + json.object do + json.field "title", video.title + json.field "videoId", video.id + json.field "lengthSeconds", video.length_seconds + + json.field "videoThumbnails" do + generate_thumbnails(json, video.id) + end + end + end + end + end + end + end + + def to_json(locale, json : JSON::Builder | Nil = nil) + if json + to_json(locale, json) + else + JSON.build do |json| + to_json(locale, json) + end + end + end +end + +struct SearchChannel + include DB::Serializable + + property author : String + property ucid : String + property author_thumbnail : String + property subscriber_count : Int32 + property video_count : Int32 + property description_html : String + property auto_generated : Bool + + def to_json(locale, json : JSON::Builder) + json.object do + json.field "type", "channel" + json.field "author", self.author + json.field "authorId", self.ucid + json.field "authorUrl", "/channel/#{self.ucid}" + + json.field "authorThumbnails" do + json.array do + qualities = {32, 48, 76, 100, 176, 512} + + qualities.each do |quality| + json.object do + json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}") + json.field "width", quality + json.field "height", quality + end + end + end + end + + json.field "autoGenerated", self.auto_generated + json.field "subCount", self.subscriber_count + json.field "videoCount", self.video_count + + json.field "description", html_to_content(self.description_html) + json.field "descriptionHtml", self.description_html + end + end + + def to_json(locale, json : JSON::Builder | Nil = nil) + if json + to_json(locale, json) + else + JSON.build do |json| + to_json(locale, json) + end + end + end +end + +class Category + include DB::Serializable + + property title : String + property contents : Array(SearchItem) | Array(Video) + property url : String? + property description_html : String + property badges : Array(Tuple(String, String))? + + def to_json(locale, json : JSON::Builder) + json.object do + json.field "title", self.title + json.field "contents", self.contents + end + end + + def to_json(locale, json : JSON::Builder | Nil = nil) + if json + to_json(locale, json) + else + JSON.build do |json| + to_json(locale, json) + end + end + end +end + +alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index f068b5f2..68ba76f9 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -1,5 +1,5 @@ require "lsquic" -require "pool/connection" +require "db" def add_yt_headers(request) request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" @@ -9,20 +9,22 @@ def add_yt_headers(request) return if request.resource.starts_with? "/sorry/index" request.headers["x-youtube-client-name"] ||= "1" request.headers["x-youtube-client-version"] ||= "2.20200609" + # Preserve original cookies and add new YT consent cookie for EU servers + request.headers["cookie"] = "#{request.headers["cookie"]?}; CONSENT=YES+" if !CONFIG.cookies.empty? request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" end end -struct QUICPool +struct YoutubeConnectionPool property! url : URI property! capacity : Int32 property! timeout : Float64 - property pool : ConnectionPool(QUIC::Client) + property pool : DB::Pool(QUIC::Client | HTTP::Client) - def initialize(url : URI, @capacity = 5, @timeout = 5.0) + def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true) @url = url - @pool = build_pool + @pool = build_pool(use_quic) end def client(region = nil, &block) @@ -41,16 +43,20 @@ struct QUICPool conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" response = yield conn ensure - pool.checkin(conn) + pool.release(conn) end end response end - private def build_pool - ConnectionPool(QUIC::Client).new(capacity: capacity, timeout: timeout) do - conn = QUIC::Client.new(url) + private def build_pool(use_quic) + DB::Pool(QUIC::Client | HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do + if use_quic + conn = QUIC::Client.new(url) + else + conn = HTTP::Client.new(url) + end conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -280,9 +286,9 @@ def arg_array(array, start = 1) return args end -def make_host_url(config, kemal_config) - ssl = config.https_only || kemal_config.ssl - port = config.external_port || kemal_config.port +def make_host_url(kemal_config) + ssl = CONFIG.https_only || kemal_config.ssl + port = CONFIG.external_port || kemal_config.port if ssl scheme = "https://" @@ -292,16 +298,16 @@ def make_host_url(config, kemal_config) # Add if non-standard port if port != 80 && port != 443 - port = ":#{kemal_config.port}" + port = ":#{port}" else port = "" end - if !config.domain + if !CONFIG.domain return "" end - host = config.domain.not_nil!.lchop(".") + host = CONFIG.domain.not_nil!.lchop(".") return "#{scheme}#{host}#{port}" end @@ -329,7 +335,7 @@ def get_referer(env, fallback = "/", unroll = true) end end - referer = referer.full_path + referer = referer.request_target referer = "/" + referer.gsub(/[^\/?@&%=\-_.0-9a-zA-Z]/, "").lstrip("/\\") if referer == env.request.path @@ -345,7 +351,7 @@ def sha256(text) return digest.final.hexstring end -def subscribe_pubsub(topic, key, config) +def subscribe_pubsub(topic, key) case topic when .match(/^UC[A-Za-z0-9_-]{22}$/) topic = "channel_id=#{topic}" @@ -403,3 +409,65 @@ def convert_theme(theme) theme end end + +def fetch_random_instance + begin + instance_api_client = make_client(URI.parse("https://api.invidious.io")) + + # Timeouts + instance_api_client.connect_timeout = 10.seconds + instance_api_client.dns_timeout = 10.seconds + + instance_list = JSON.parse(instance_api_client.get("/instances.json").body).as_a + instance_api_client.close + rescue Socket::ConnectError | IO::TimeoutError | JSON::ParseException + instance_list = [] of JSON::Any + end + + filtered_instance_list = [] of String + + instance_list.each do |data| + # TODO Check if current URL is onion instance and use .onion types if so. + if data[1]["type"] == "https" + # Instances can have statisitics disabled, which is an requirement of version validation. + # as_nil? doesn't exist. Thus we'll have to handle the error rasied if as_nil fails. + begin + data[1]["stats"].as_nil + next + rescue TypeCastError + end + + # stats endpoint could also lack the software dict. + next if data[1]["stats"]["software"]?.nil? + + # Makes sure the instance isn't too outdated. + if remote_version = data[1]["stats"]?.try &.["software"]?.try &.["version"] + remote_commit_date = remote_version.as_s.match(/\d{4}\.\d{2}\.\d{2}/) + next if !remote_commit_date + + remote_commit_date = Time.parse(remote_commit_date[0], "%Y.%m.%d", Time::Location::UTC) + local_commit_date = Time.parse(CURRENT_VERSION, "%Y.%m.%d", Time::Location::UTC) + + next if (remote_commit_date - local_commit_date).abs.days > 30 + + begin + data[1]["monitor"].as_nil + health = data[1]["monitor"].as_h["dailyRatios"][0].as_h["ratio"] + filtered_instance_list << data[0].as_s if health.to_s.to_f > 90 + rescue TypeCastError + # We can't check the health if the monitoring is broken. Thus we'll just add it to the list + # and move on. Ideally we'll ignore any instance that has broken health monitoring but due to the fact that + # it's an error that often occurs with all the instances at the same time, we have to just skip the check. + filtered_instance_list << data[0].as_s + end + end + end + end + + # If for some reason no instances managed to get fetched successfully then we'll just redirect to redirect.invidious.io + if filtered_instance_list.size == 0 + return "redirect.invidious.io" + end + + return filtered_instance_list.sample(1)[0] +end diff --git a/src/invidious/helpers/youtube_api.cr b/src/invidious/helpers/youtube_api.cr new file mode 100644 index 00000000..b3815f6a --- /dev/null +++ b/src/invidious/helpers/youtube_api.cr @@ -0,0 +1,447 @@ +# +# This file contains youtube API wrappers +# + +module YoutubeAPI + extend self + + # Enumerate used to select one of the clients supported by the API + enum ClientType + Web + WebEmbeddedPlayer + WebMobile + WebScreenEmbed + Android + AndroidEmbeddedPlayer + AndroidScreenEmbed + end + + # List of hard-coded values used by the different clients + HARDCODED_CLIENTS = { + ClientType::Web => { + name: "WEB", + version: "2.20210721.00.00", + api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", + screen: "WATCH_FULL_SCREEN", + }, + ClientType::WebEmbeddedPlayer => { + name: "WEB_EMBEDDED_PLAYER", # 56 + version: "1.20210721.1.0", + api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", + screen: "EMBED", + }, + ClientType::WebMobile => { + name: "MWEB", + version: "2.20210726.08.00", + api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", + screen: "", # None + }, + ClientType::WebScreenEmbed => { + name: "WEB", + version: "2.20210721.00.00", + api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", + screen: "EMBED", + }, + ClientType::Android => { + name: "ANDROID", + version: "16.20", + api_key: "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w", + screen: "", # ?? + }, + ClientType::AndroidEmbeddedPlayer => { + name: "ANDROID_EMBEDDED_PLAYER", # 55 + version: "16.20", + api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", + screen: "", # None? + }, + ClientType::AndroidScreenEmbed => { + name: "ANDROID", # 3 + version: "16.20", + api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", + screen: "EMBED", + }, + } + + #################################################################### + # struct ClientConfig + # + # Data structure used to pass a client configuration to the different + # API endpoints handlers. + # + # Use case examples: + # + # ``` + # # Get Norwegian search results + # conf_1 = ClientConfig.new(region: "NO") + # YoutubeAPI::search("Kollektivet", params: "", client_config: conf_1) + # + # # Use the Android client to request video streams URLs + # conf_2 = ClientConfig.new(client_type: ClientType::Android) + # YoutubeAPI::player(video_id: "dQw4w9WgXcQ", client_config: conf_2) + # + # # Proxy request through russian proxies + # conf_3 = ClientConfig.new(proxy_region: "RU") + # YoutubeAPI::next({video_id: "dQw4w9WgXcQ"}, client_config: conf_3) + # ``` + # + struct ClientConfig + # Type of client to emulate. + # See `enum ClientType` and `HARDCODED_CLIENTS`. + property client_type : ClientType + + # Region to provide to youtube, e.g to alter search results + # (this is passed as the `gl` parmeter). + property region : String | Nil + + # ISO code of country where the proxy is located. + # Used in case of geo-restricted videos. + property proxy_region : String | Nil + + # Initialization function + def initialize( + *, + @client_type = ClientType::Web, + @region = "US", + @proxy_region = nil + ) + end + + # Getter functions that provides easy access to hardcoded clients + # parameters (name/version strings and related API key) + def name : String + HARDCODED_CLIENTS[@client_type][:name] + end + + # :ditto: + def version : String + HARDCODED_CLIENTS[@client_type][:version] + end + + # :ditto: + def api_key : String + HARDCODED_CLIENTS[@client_type][:api_key] + end + + # :ditto: + def screen : String + HARDCODED_CLIENTS[@client_type][:screen] + end + + # Convert to string, for logging purposes + def to_s + return { + client_type: self.name, + region: @region, + proxy_region: @proxy_region, + }.to_s + end + end + + # Default client config, used if nothing is passed + DEFAULT_CLIENT_CONFIG = ClientConfig.new + + #################################################################### + # make_context(client_config) + # + # Return, as a Hash, the "context" data required to request the + # youtube API endpoints. + # + private def make_context(client_config : ClientConfig | Nil) : Hash + # Use the default client config if nil is passed + client_config ||= DEFAULT_CLIENT_CONFIG + + client_context = { + "client" => { + "hl" => "en", + "gl" => client_config.region || "US", # Can't be empty! + "clientName" => client_config.name, + "clientVersion" => client_config.version, + }, + } + + # Add some more context if it exists in the client definitions + if !client_config.screen.empty? + client_context["client"]["clientScreen"] = client_config.screen + end + + if client_config.screen == "EMBED" + client_context["thirdParty"] = { + "embedUrl" => "https://www.youtube.com/embed/dQw4w9WgXcQ", + } + end + + return client_context + end + + #################################################################### + # browse(continuation, client_config?) + # browse(browse_id, params, client_config?) + # + # Requests the youtubei/v1/browse endpoint with the required headers + # and POST data in order to get a JSON reply in english that can + # be easily parsed. + # + # Both forms can take an optional ClientConfig parameter (see + # `struct ClientConfig` above for more details). + # + # The requested data can either be: + # + # - A continuation token (ctoken). Depending on this token's + # contents, the returned data can be playlist videos, channel + # community tab content, channel info, ... + # + # - A playlist ID (parameters MUST be an empty string) + # + def browse(continuation : String, client_config : ClientConfig | Nil = nil) + # JSON Request data, required by the API + data = { + "context" => self.make_context(client_config), + "continuation" => continuation, + } + + return self._post_json("/youtubei/v1/browse", data, client_config) + end + + # :ditto: + def browse( + browse_id : String, + *, # Force the following paramters to be passed by name + params : String, + client_config : ClientConfig | Nil = nil + ) + # JSON Request data, required by the API + data = { + "browseId" => browse_id, + "context" => self.make_context(client_config), + } + + # Append the additionnal parameters if those were provided + # (this is required for channel info, playlist and community, e.g) + if params != "" + data["params"] = params + end + + return self._post_json("/youtubei/v1/browse", data, client_config) + end + + #################################################################### + # next(continuation, client_config?) + # next(data, client_config?) + # + # Requests the youtubei/v1/next endpoint with the required headers + # and POST data in order to get a JSON reply in english that can + # be easily parsed. + # + # Both forms can take an optional ClientConfig parameter (see + # `struct ClientConfig` above for more details). + # + # The requested data can be: + # + # - A continuation token (ctoken). Depending on this token's + # contents, the returned data can be videos comments, + # their replies, ... In this case, the string must be passed + # directly to the function. E.g: + # + # ``` + # YoutubeAPI::next("ABCDEFGH_abcdefgh==") + # ``` + # + # - Arbitrary parameters, in Hash form. See examples below for + # known examples of arbitrary data that can be passed to YouTube: + # + # ``` + # # Get the videos related to a specific video ID + # YoutubeAPI::next({"videoId" => "dQw4w9WgXcQ"}) + # + # # Get a playlist video's details + # YoutubeAPI::next({ + # "videoId" => "9bZkp7q19f0", + # "playlistId" => "PL_oFlvgqkrjUVQwiiE3F3k3voF4tjXeP0", + # }) + # ``` + # + def next(continuation : String, *, client_config : ClientConfig | Nil = nil) + # JSON Request data, required by the API + data = { + "context" => self.make_context(client_config), + "continuation" => continuation, + } + + return self._post_json("/youtubei/v1/next", data, client_config) + end + + # :ditto: + def next(data : Hash, *, client_config : ClientConfig | Nil = nil) + # JSON Request data, required by the API + data2 = data.merge({ + "context" => self.make_context(client_config), + }) + + return self._post_json("/youtubei/v1/next", data2, client_config) + end + + # Allow a NamedTuple to be passed, too. + def next(data : NamedTuple, *, client_config : ClientConfig | Nil = nil) + return self.next(data.to_h, client_config: client_config) + end + + #################################################################### + # player(video_id, params, client_config?) + # + # Requests the youtubei/v1/player endpoint with the required headers + # and POST data in order to get a JSON reply. + # + # The requested data is a video ID (`v=` parameter), with some + # additional paramters, formatted as a base64 string. + # + # An optional ClientConfig parameter can be passed, too (see + # `struct ClientConfig` above for more details). + # + def player( + video_id : String, + *, # Force the following paramters to be passed by name + params : String, + client_config : ClientConfig | Nil = nil + ) + # JSON Request data, required by the API + data = { + "videoId" => video_id, + "context" => self.make_context(client_config), + } + + # Append the additionnal parameters if those were provided + if params != "" + data["params"] = params + end + + return self._post_json("/youtubei/v1/player", data, client_config) + end + + #################################################################### + # resolve_url(url, client_config?) + # + # Requests the youtubei/v1/navigation/resolve_url endpoint with the + # required headers and POST data in order to get a JSON reply. + # + # An optional ClientConfig parameter can be passed, too (see + # `struct ClientConfig` above for more details). + # + # Output: + # + # ``` + # # Valid channel "brand URL" gives the related UCID and browse ID + # channel_a = YoutubeAPI.resolve_url("https://youtube.com/c/google") + # channel_a # => { + # "endpoint": { + # "browseEndpoint": { + # "params": "EgC4AQA%3D", + # "browseId":"UCK8sQmJBp8GCxrOtXWBpyEA" + # }, + # ... + # } + # } + # + # # Invalid URL returns throws an InfoException + # channel_b = YoutubeAPI.resolve_url("https://youtube.com/c/invalid") + # ``` + # + def resolve_url(url : String, client_config : ClientConfig | Nil = nil) + data = { + "context" => self.make_context(nil), + "url" => url, + } + + return self._post_json("/youtubei/v1/navigation/resolve_url", data, client_config) + end + + #################################################################### + # search(search_query, params, client_config?) + # + # Requests the youtubei/v1/search endpoint with the required headers + # and POST data in order to get a JSON reply. As the search results + # vary depending on the region, a region code can be specified in + # order to get non-US results. + # + # The requested data is a search string, with some additional + # paramters, formatted as a base64 string. + # + # An optional ClientConfig parameter can be passed, too (see + # `struct ClientConfig` above for more details). + # + def search( + search_query : String, + params : String, + client_config : ClientConfig | Nil = nil + ) + # JSON Request data, required by the API + data = { + "query" => search_query, + "context" => self.make_context(client_config), + "params" => params, + } + + return self._post_json("/youtubei/v1/search", data, client_config) + end + + #################################################################### + # _post_json(endpoint, data, client_config?) + # + # Internal function that does the actual request to youtube servers + # and handles errors. + # + # The requested data is an endpoint (URL without the domain part) + # and the data as a Hash object. + # + def _post_json( + endpoint : String, + data : Hash, + client_config : ClientConfig | Nil + ) : Hash(String, JSON::Any) + # Use the default client config if nil is passed + client_config ||= DEFAULT_CLIENT_CONFIG + + # Query parameters + url = "#{endpoint}?key=#{client_config.api_key}" + + headers = HTTP::Headers{ + "Content-Type" => "application/json; charset=UTF-8", + "Accept-Encoding" => "gzip", + } + + # Logging + LOGGER.debug("YoutubeAPI: Using endpoint: \"#{endpoint}\"") + LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config.to_s}") + LOGGER.trace("YoutubeAPI: POST data: #{data.to_s}") + + # Send the POST request + if client_config.proxy_region + response = YT_POOL.client( + client_config.proxy_region, + &.post(url, headers: headers, body: data.to_json) + ) + else + response = YT_POOL.client &.post( + url, headers: headers, body: data.to_json + ) + end + + # Convert result to Hash + initial_data = JSON.parse(response.body).as_h + + # Error handling + if initial_data.has_key?("error") + code = initial_data["error"]["code"] + message = initial_data["error"]["message"].to_s.sub(/(\\n)+\^$/, "") + + # Logging + LOGGER.error("YoutubeAPI: Got error #{code} when requesting #{endpoint}") + LOGGER.error("YoutubeAPI: #{message}") + LOGGER.info("YoutubeAPI: POST data was: #{data.to_s}") + + raise InfoException.new("Could not extract JSON. Youtube API returned \ + error #{code} with message:<br>\"#{message}\"") + end + + return initial_data + end +end # End of module diff --git a/src/invidious/jobs/bypass_captcha_job.cr b/src/invidious/jobs/bypass_captcha_job.cr index 22c54036..71f8a938 100644 --- a/src/invidious/jobs/bypass_captcha_job.cr +++ b/src/invidious/jobs/bypass_captcha_job.cr @@ -1,13 +1,12 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob - private getter config : Config - - def initialize(@config) - end - def begin loop do begin - {"/watch?v=jNQXAC9IVRw&gl=US&hl=en&has_verified=1&bpctr=9999999999", produce_channel_videos_url(ucid: "UC4QobU6STFB0P71PMvOGN5A")}.each do |path| + random_video = PG_DB.query_one?("select id, ucid from (select id, ucid from channel_videos limit 1000) as s ORDER BY RANDOM() LIMIT 1", as: {id: String, ucid: String}) + if !random_video + random_video = {id: "zj82_v2R6ts", ucid: "UCK87Lox575O_HCHBWaBSyGA"} + end + {"/watch?v=#{random_video["id"]}&gl=US&hl=en&has_verified=1&bpctr=9999999999", produce_channel_videos_url(ucid: random_video["ucid"])}.each do |path| response = YT_POOL.client &.get(path) if response.body.includes?("To continue with your YouTube experience, please fill out the form below.") html = XML.parse_html(response.body) @@ -22,9 +21,9 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob headers = response.cookies.add_request_headers(HTTP::Headers.new) - response = JSON.parse(HTTP::Client.post(config.captcha_api_url + "/createTask", + response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/createTask", headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => config.captcha_key, + "clientKey" => CONFIG.captcha_key, "task" => { "type" => "NoCaptchaTaskProxyless", "websiteURL" => "https://www.youtube.com#{path}", @@ -39,9 +38,9 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob loop do sleep 10.seconds - response = JSON.parse(HTTP::Client.post(config.captcha_api_url + "/getTaskResult", + response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/getTaskResult", headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => config.captcha_key, + "clientKey" => CONFIG.captcha_key, "taskId" => task_id, }.to_json).body) @@ -58,14 +57,14 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob response.cookies .select { |cookie| cookie.name != "PREF" } - .each { |cookie| config.cookies << cookie } + .each { |cookie| CONFIG.cookies << cookie } # Persist cookies between runs - File.write("config/config.yml", config.to_yaml) + File.write("config/config.yml", CONFIG.to_yaml) elsif response.headers["Location"]?.try &.includes?("/sorry/index") location = response.headers["Location"].try { |u| URI.parse(u) } headers = HTTP::Headers{":authority" => location.host.not_nil!} - response = YT_POOL.client &.get(location.full_path, headers) + response = YT_POOL.client &.get(location.request_target, headers) html = XML.parse_html(response.body) form = html.xpath_node(%(//form[@action="index"])).not_nil! @@ -77,11 +76,11 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob inputs[node["name"]] = node["value"] end - captcha_client = HTTPClient.new(URI.parse(config.captcha_api_url)) - captcha_client.family = config.force_resolve || Socket::Family::INET + captcha_client = HTTPClient.new(URI.parse(CONFIG.captcha_api_url)) + captcha_client.family = CONFIG.force_resolve || Socket::Family::INET response = JSON.parse(captcha_client.post("/createTask", headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => config.captcha_key, + "clientKey" => CONFIG.captcha_key, "task" => { "type" => "NoCaptchaTaskProxyless", "websiteURL" => location.to_s, @@ -100,7 +99,7 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob response = JSON.parse(captcha_client.post("/getTaskResult", headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => config.captcha_key, + "clientKey" => CONFIG.captcha_key, "taskId" => task_id, }.to_json).body) @@ -117,12 +116,12 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob headers = HTTP::Headers{ "Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0], } - cookies = HTTP::Cookies.from_headers(headers) + cookies = HTTP::Cookies.from_client_headers(headers) - cookies.each { |cookie| config.cookies << cookie } + cookies.each { |cookie| CONFIG.cookies << cookie } # Persist cookies between runs - File.write("config/config.yml", config.to_yaml) + File.write("config/config.yml", CONFIG.to_yaml) end end rescue ex diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr index 3e94a56e..fbe6d381 100644 --- a/src/invidious/jobs/refresh_channels_job.cr +++ b/src/invidious/jobs/refresh_channels_job.cr @@ -1,12 +1,11 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob private getter db : DB::Database - private getter config : Config - def initialize(@db, @config) + def initialize(@db) end def begin - max_fibers = config.channel_threads + max_fibers = CONFIG.channel_threads lim_fibers = max_fibers active_fibers = 0 active_channel = Channel(Bool).new @@ -31,7 +30,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob spawn do begin LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel") - channel = fetch_channel(id, db, config.full_refresh) + channel = fetch_channel(id, db, CONFIG.full_refresh) lim_fibers = max_fibers diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr index 7b4ccdea..926c27fa 100644 --- a/src/invidious/jobs/refresh_feeds_job.cr +++ b/src/invidious/jobs/refresh_feeds_job.cr @@ -1,12 +1,11 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob private getter db : DB::Database - private getter config : Config - def initialize(@db, @config) + def initialize(@db) end def begin - max_fibers = config.feed_threads + max_fibers = CONFIG.feed_threads active_fibers = 0 active_channel = Channel(Bool).new diff --git a/src/invidious/jobs/statistics_refresh_job.cr b/src/invidious/jobs/statistics_refresh_job.cr index 021671be..6569c0a1 100644 --- a/src/invidious/jobs/statistics_refresh_job.cr +++ b/src/invidious/jobs/statistics_refresh_job.cr @@ -21,9 +21,8 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob } private getter db : DB::Database - private getter config : Config - def initialize(@db, @config, @software_config : Hash(String, String)) + def initialize(@db, @software_config : Hash(String, String)) end def begin @@ -43,7 +42,7 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob "version" => @software_config["version"], "branch" => @software_config["branch"], } - STATISTICS["openRegistration"] = config.registration_enabled + STATISTICS["openRegistrations"] = CONFIG.registration_enabled end private def refresh_stats diff --git a/src/invidious/jobs/subscribe_to_feeds_job.cr b/src/invidious/jobs/subscribe_to_feeds_job.cr index 750aceb8..a431a48a 100644 --- a/src/invidious/jobs/subscribe_to_feeds_job.cr +++ b/src/invidious/jobs/subscribe_to_feeds_job.cr @@ -1,15 +1,14 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob private getter db : DB::Database private getter hmac_key : String - private getter config : Config - def initialize(@db, @config, @hmac_key) + def initialize(@db, @hmac_key) end def begin max_fibers = 1 - if config.use_pubsub_feeds.is_a?(Int32) - max_fibers = config.use_pubsub_feeds.as(Int32) + if CONFIG.use_pubsub_feeds.is_a?(Int32) + max_fibers = CONFIG.use_pubsub_feeds.as(Int32) end active_fibers = 0 @@ -30,7 +29,7 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob spawn do begin - response = subscribe_pubsub(ucid, hmac_key, config) + response = subscribe_pubsub(ucid, hmac_key) if response.status_code >= 400 LOGGER.error("SubscribeToFeedsJob: #{ucid} : #{response.body}") diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 25797a36..f56cc2ea 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -101,6 +101,7 @@ struct Playlist property author_thumbnail : String property ucid : String property description : String + property description_html : String property video_count : Int32 property views : Int64 property updated : Time @@ -163,10 +164,6 @@ struct Playlist def privacy PlaylistPrivacy::Public end - - def description_html - HTML.escape(self.description).gsub("\n", "<br>") - end end enum PlaylistPrivacy @@ -310,23 +307,32 @@ def subscribe_playlist(db, user, playlist) return playlist end -def produce_playlist_url(id, index) +def produce_playlist_continuation(id, index) if id.starts_with? "UC" id = "UU" + id.lchop("UC") end plid = "VL" + id + # Emulate a "request counter" increment, to make perfectly valid + # ctokens, even if at the time of writing, it's ignored by youtube. + request_count = (index / 100).to_i64 || 1_i64 + data = {"1:varint" => index.to_i64} .try { |i| Protodec::Any.cast_json(i) } .try { |i| Protodec::Any.from_json(i) } .try { |i| Base64.urlsafe_encode(i, padding: false) } + data_wrapper = {"1:varint" => request_count, "15:string" => "PT:#{data}"} + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + object = { "80226972:embedded" => { - "2:string" => plid, - "3:base64" => { - "15:string" => "PT:#{data}", - }, + "2:string" => plid, + "3:string" => data_wrapper, + "35:string" => id, }, } @@ -335,7 +341,7 @@ def produce_playlist_url(id, index) .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } - return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" + return continuation end def get_playlist(db, plid, locale, refresh = true, force_refresh = false) @@ -355,16 +361,7 @@ def fetch_playlist(plid, locale) plid = "UU#{plid.lchop("UC")}" end - response = YT_POOL.client &.get("/playlist?list=#{plid}&hl=en") - if response.status_code != 200 - if response.headers["location"]?.try &.includes? "/sorry/index" - raise InfoException.new("Could not extract playlist info. Instance is likely blocked.") - else - raise InfoException.new("Not a playlist.") - end - end - - initial_data = extract_initial_data(response.body) + initial_data = YoutubeAPI.browse("VL" + plid, params: "") playlist_sidebar_renderer = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]? raise InfoException.new("Could not extract playlistSidebarRenderer.") if !playlist_sidebar_renderer @@ -375,7 +372,12 @@ def fetch_playlist(plid, locale) title = playlist_info["title"]?.try &.["runs"][0]?.try &.["text"]?.try &.as_s || "" desc_item = playlist_info["description"]? - description = desc_item.try &.["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || desc_item.try &.["simpleText"]?.try &.as_s || "" + + description_txt = desc_item.try &.["runs"]?.try &.as_a + .map(&.["text"].as_s).join("") || desc_item.try &.["simpleText"]?.try &.as_s || "" + + description_html = desc_item.try &.["runs"]?.try &.as_a + .try { |run| content_to_comment_html(run).try &.to_s } || "<p></p>" thumbnail = playlist_info["thumbnailRenderer"]?.try &.["playlistVideoThumbnailRenderer"]? .try &.["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s @@ -415,7 +417,8 @@ def fetch_playlist(plid, locale) author: author, author_thumbnail: author_thumbnail, ucid: ucid, - description: description, + description: description_txt, + description_html: description_html, video_count: video_count, views: views, updated: updated, @@ -424,47 +427,55 @@ def fetch_playlist(plid, locale) end def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil) - if playlist.is_a? InvidiousPlaylist - db.query_all("SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3", playlist.id, playlist.index, offset, as: PlaylistVideo) - else - fetch_playlist_videos(playlist.id, playlist.video_count, offset, locale, continuation) - end -end - -def fetch_playlist_videos(plid, video_count, offset = 0, locale = nil, continuation = nil) - if continuation - response = YT_POOL.client &.get("/watch?v=#{continuation}&list=#{plid}&gl=US&hl=en") - initial_data = extract_initial_data(response.body) - offset = initial_data["currentVideoEndpoint"]?.try &.["watchEndpoint"]?.try &.["index"]?.try &.as_i64 || offset - end - - if video_count > 100 - url = produce_playlist_url(plid, offset) - - response = YT_POOL.client &.get(url) - initial_data = JSON.parse(response.body).as_a.find(&.as_h.["response"]?).try &.as_h - elsif offset > 100 + # Show empy playlist if requested page is out of range + # (e.g, when a new playlist has been created, offset will be negative) + if offset >= playlist.video_count || offset < 0 return [] of PlaylistVideo - else # Extract first page of videos - response = YT_POOL.client &.get("/playlist?list=#{plid}&gl=US&hl=en") - initial_data = extract_initial_data(response.body) end - return [] of PlaylistVideo if !initial_data - videos = extract_playlist_videos(initial_data) + if playlist.is_a? InvidiousPlaylist + db.query_all("SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3", + playlist.id, playlist.index, offset, as: PlaylistVideo) + else + if offset >= 100 + # Normalize offset to match youtube's behavior (100 videos chunck per request) + offset = (offset / 100).to_i64 * 100_i64 - until videos.empty? || videos[0].index == offset - videos.shift - end + ctoken = produce_playlist_continuation(playlist.id, offset) + initial_data = YoutubeAPI.browse(ctoken) + else + initial_data = YoutubeAPI.browse("VL" + playlist.id, params: "") + end - return videos + return extract_playlist_videos(initial_data) + end end def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) videos = [] of PlaylistVideo - (initial_data["contents"]?.try &.["twoColumnBrowseResultsRenderer"]["tabs"].as_a.select(&.["tabRenderer"]["selected"]?.try &.as_bool)[0]["tabRenderer"]["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["playlistVideoListRenderer"]["contents"].as_a || - initial_data["response"]?.try &.["continuationContents"]["playlistVideoListContinuation"]["contents"].as_a).try &.each do |item| + if initial_data["contents"]? + tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"] + tabs_renderer = tabs.as_a.select(&.["tabRenderer"]["selected"]?.try &.as_bool)[0]["tabRenderer"] + + # Watch out the two versions, with and without "s" + if tabs_renderer["contents"]? || tabs_renderer["content"]? + # Initial playlist data + tabs_contents = tabs_renderer.["contents"]? || tabs_renderer.["content"] + + list_renderer = tabs_contents.["sectionListRenderer"]["contents"][0] + item_renderer = list_renderer.["itemSectionRenderer"]["contents"][0] + contents = item_renderer.["playlistVideoListRenderer"]["contents"].as_a + else + # Continuation data + contents = initial_data["onResponseReceivedActions"][0]? + .try &.["appendContinuationItemsAction"]["continuationItems"].as_a + end + else + contents = initial_data["response"]?.try &.["continuationContents"]["playlistVideoListContinuation"]["contents"].as_a + end + + contents.try &.each do |item| if i = item["playlistVideoRenderer"]? video_id = i["navigationEndpoint"]["watchEndpoint"]["videoId"].as_s plid = i["navigationEndpoint"]["watchEndpoint"]["playlistId"].as_s diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr new file mode 100644 index 00000000..f8963587 --- /dev/null +++ b/src/invidious/routes/api/manifest.cr @@ -0,0 +1,224 @@ +module Invidious::Routes::API::Manifest + # /api/manifest/dash/id/:id + def self.get_dash_video_id(env) + env.response.headers.add("Access-Control-Allow-Origin", "*") + env.response.content_type = "application/dash+xml" + + local = env.params.query["local"]?.try &.== "true" + id = env.params.url["id"] + region = env.params.query["region"]? + + # Since some implementations create playlists based on resolution regardless of different codecs, + # we can opt to only add a source to a representation if it has a unique height within that representation + unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe } + + begin + video = get_video(id, PG_DB, region: region) + rescue ex : VideoRedirect + return env.redirect env.request.resource.gsub(id, ex.video_id) + rescue ex + haltf env, status_code: 403 + end + + if dashmpd = video.dash_manifest_url + manifest = YT_POOL.client &.get(URI.parse(dashmpd).request_target).body + + manifest = manifest.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl| + url = baseurl.lchop("<BaseURL>") + url = url.rchop("</BaseURL>") + + if local + uri = URI.parse(url) + url = "#{uri.request_target}host/#{uri.host}/" + end + + "<BaseURL>#{url}</BaseURL>" + end + + return manifest + end + + adaptive_fmts = video.adaptive_fmts + + if local + adaptive_fmts.each do |fmt| + fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) + end + end + + audio_streams = video.audio_streams + video_streams = video.video_streams.sort_by { |stream| {stream["width"].as_i, stream["fps"].as_i} }.reverse + + manifest = XML.build(indent: " ", encoding: "UTF-8") do |xml| + xml.element("MPD", "xmlns": "urn:mpeg:dash:schema:mpd:2011", + "profiles": "urn:mpeg:dash:profile:full:2011", minBufferTime: "PT1.5S", type: "static", + mediaPresentationDuration: "PT#{video.length_seconds}S") do + xml.element("Period") do + i = 0 + + {"audio/mp4", "audio/webm"}.each do |mime_type| + mime_streams = audio_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type } + next if mime_streams.empty? + + xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true) do + mime_streams.each do |fmt| + codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"') + bandwidth = fmt["bitrate"].as_i + itag = fmt["itag"].as_i + url = fmt["url"].as_s + + xml.element("Representation", id: fmt["itag"], codecs: codecs, bandwidth: bandwidth) do + xml.element("AudioChannelConfiguration", schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011", + value: "2") + xml.element("BaseURL") { xml.text url } + xml.element("SegmentBase", indexRange: "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}") do + xml.element("Initialization", range: "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}") + end + end + end + end + + i += 1 + end + + potential_heights = {4320, 2160, 1440, 1080, 720, 480, 360, 240, 144} + + {"video/mp4", "video/webm"}.each do |mime_type| + mime_streams = video_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type } + next if mime_streams.empty? + + heights = [] of Int32 + xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, scanType: "progressive") do + mime_streams.each do |fmt| + codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"') + bandwidth = fmt["bitrate"].as_i + itag = fmt["itag"].as_i + url = fmt["url"].as_s + width = fmt["width"].as_i + height = fmt["height"].as_i + + # Resolutions reported by YouTube player (may not accurately reflect source) + height = potential_heights.min_by { |i| (height - i).abs } + next if unique_res && heights.includes? height + heights << height + + xml.element("Representation", id: itag, codecs: codecs, width: width, height: height, + startWithSAP: "1", maxPlayoutRate: "1", + bandwidth: bandwidth, frameRate: fmt["fps"]) do + xml.element("BaseURL") { xml.text url } + xml.element("SegmentBase", indexRange: "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}") do + xml.element("Initialization", range: "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}") + end + end + end + end + + i += 1 + end + end + end + end + + return manifest + end + + # /api/manifest/dash/id/videoplayback + def self.get_dash_video_playback(env) + env.response.headers.delete("Content-Type") + env.response.headers["Access-Control-Allow-Origin"] = "*" + env.redirect "/videoplayback?#{env.params.query}" + end + + # /api/manifest/dash/id/videoplayback/* + def self.get_dash_video_playback_greedy(env) + env.response.headers.delete("Content-Type") + env.response.headers["Access-Control-Allow-Origin"] = "*" + env.redirect env.request.path.lchop("/api/manifest/dash/id") + end + + # /api/manifest/dash/id/videoplayback && /api/manifest/dash/id/videoplayback/* + def self.options_dash_video_playback(env) + env.response.headers.delete("Content-Type") + env.response.headers["Access-Control-Allow-Origin"] = "*" + env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" + env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range" + end + + # /api/manifest/hls_playlist/* + def self.get_hls_playlist(env) + response = YT_POOL.client &.get(env.request.path) + + if response.status_code != 200 + haltf env, status_code: response.status_code + end + + local = env.params.query["local"]?.try &.== "true" + + env.response.content_type = "application/x-mpegURL" + env.response.headers.add("Access-Control-Allow-Origin", "*") + + manifest = response.body + + if local + manifest = manifest.gsub(/^https:\/\/\w+---.{11}\.c\.youtube\.com[^\n]*/m) do |match| + path = URI.parse(match).path + + path = path.lchop("/videoplayback/") + path = path.rchop("/") + + path = path.gsub(/mime\/\w+\/\w+/) do |mimetype| + mimetype = mimetype.split("/") + mimetype[0] + "/" + mimetype[1] + "%2F" + mimetype[2] + end + + path = path.split("/") + + raw_params = {} of String => Array(String) + path.each_slice(2) do |pair| + key, value = pair + value = URI.decode_www_form(value) + + if raw_params[key]? + raw_params[key] << value + else + raw_params[key] = [value] + end + end + + raw_params = HTTP::Params.new(raw_params) + if fvip = raw_params["hls_chunk_host"].match(/r(?<fvip>\d+)---/) + raw_params["fvip"] = fvip["fvip"] + end + + raw_params["local"] = "true" + + "#{HOST_URL}/videoplayback?#{raw_params}" + end + end + + manifest + end + + # /api/manifest/hls_variant/* + def self.get_hls_variant(env) + response = YT_POOL.client &.get(env.request.path) + + if response.status_code != 200 + haltf env, status_code: response.status_code + end + + local = env.params.query["local"]?.try &.== "true" + + env.response.content_type = "application/x-mpegURL" + env.response.headers.add("Access-Control-Allow-Origin", "*") + + manifest = response.body + + if local + manifest = manifest.gsub("https://www.youtube.com", HOST_URL) + manifest = manifest.gsub("index.m3u8", "index.m3u8?local=true") + end + + manifest + end +end diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr new file mode 100644 index 00000000..b4e9e9c8 --- /dev/null +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -0,0 +1,415 @@ +module Invidious::Routes::API::V1::Authenticated + # The notification APIs cannot be extracted yet! + # They require the *local* notifications constant defined in invidious.cr + # + # def self.notifications(env) + # env.response.content_type = "text/event-stream" + + # topics = env.params.body["topics"]?.try &.split(",").uniq.first(1000) + # topics ||= [] of String + + # create_notification_stream(env, topics, connection_channel) + # end + + def self.get_preferences(env) + env.response.content_type = "application/json" + user = env.get("user").as(User) + user.preferences.to_json + end + + def self.set_preferences(env) + env.response.content_type = "application/json" + user = env.get("user").as(User) + + begin + preferences = Preferences.from_json(env.request.body || "{}") + rescue + preferences = user.preferences + end + + PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) + + env.response.status_code = 204 + end + + def self.feed(env) + env.response.content_type = "application/json" + + user = env.get("user").as(User) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + max_results = env.params.query["max_results"]?.try &.to_i? + max_results ||= user.preferences.max_results + max_results ||= CONFIG.default_user_preferences.max_results + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) + + JSON.build do |json| + json.object do + json.field "notifications" do + json.array do + notifications.each do |video| + video.to_json(locale, json) + end + end + end + + json.field "videos" do + json.array do + videos.each do |video| + video.to_json(locale, json) + end + end + end + end + end + end + + def self.get_subscriptions(env) + env.response.content_type = "application/json" + user = env.get("user").as(User) + + if user.subscriptions.empty? + values = "'{}'" + else + values = "VALUES #{user.subscriptions.map { |id| %(('#{id}')) }.join(",")}" + end + + subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousChannel) + + JSON.build do |json| + json.array do + subscriptions.each do |subscription| + json.object do + json.field "author", subscription.author + json.field "authorId", subscription.id + end + end + end + end + end + + def self.subscribe_channel(env) + env.response.content_type = "application/json" + user = env.get("user").as(User) + + ucid = env.params.url["ucid"] + + if !user.subscriptions.includes? ucid + get_channel(ucid, PG_DB, false, false) + PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email) + end + + # For Google accounts, access tokens don't have enough information to + # make a request on the user's behalf, which is why we don't sync with + # YouTube. + + env.response.status_code = 204 + end + + def self.unsubscribe_channel(env) + env.response.content_type = "application/json" + user = env.get("user").as(User) + + ucid = env.params.url["ucid"] + + PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_remove(subscriptions, $1) WHERE email = $2", ucid, user.email) + + env.response.status_code = 204 + end + + def self.list_playlists(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + user = env.get("user").as(User) + + playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1", user.email, as: InvidiousPlaylist) + + JSON.build do |json| + json.array do + playlists.each do |playlist| + playlist.to_json(0, locale, json) + end + end + end + end + + def self.create_playlist(env) + env.response.content_type = "application/json" + user = env.get("user").as(User) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + title = env.params.json["title"]?.try &.as(String).delete("<>").byte_slice(0, 150) + if !title + return error_json(400, "Invalid title.") + end + + privacy = env.params.json["privacy"]?.try { |privacy| PlaylistPrivacy.parse(privacy.as(String).downcase) } + if !privacy + return error_json(400, "Invalid privacy setting.") + end + + if PG_DB.query_one("SELECT count(*) FROM playlists WHERE author = $1", user.email, as: Int64) >= 100 + return error_json(400, "User cannot have more than 100 playlists.") + end + + playlist = create_playlist(PG_DB, title, privacy, user) + env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{playlist.id}" + env.response.status_code = 201 + { + "title" => title, + "playlistId" => playlist.id, + }.to_json + end + + def self.update_playlist_attribute(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + user = env.get("user").as(User) + + plid = env.params.url["plid"] + + playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if !playlist || playlist.author != user.email && playlist.privacy.private? + return error_json(404, "Playlist does not exist.") + end + + if playlist.author != user.email + return error_json(403, "Invalid user") + end + + title = env.params.json["title"].try &.as(String).delete("<>").byte_slice(0, 150) || playlist.title + privacy = env.params.json["privacy"]?.try { |privacy| PlaylistPrivacy.parse(privacy.as(String).downcase) } || playlist.privacy + description = env.params.json["description"]?.try &.as(String).delete("\r") || playlist.description + + if title != playlist.title || + privacy != playlist.privacy || + description != playlist.description + updated = Time.utc + else + updated = playlist.updated + end + + PG_DB.exec("UPDATE playlists SET title = $1, privacy = $2, description = $3, updated = $4 WHERE id = $5", title, privacy, description, updated, plid) + env.response.status_code = 204 + end + + def self.delete_playlist(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + user = env.get("user").as(User) + + plid = env.params.url["plid"] + + playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if !playlist || playlist.author != user.email && playlist.privacy.private? + return error_json(404, "Playlist does not exist.") + end + + if playlist.author != user.email + return error_json(403, "Invalid user") + end + + PG_DB.exec("DELETE FROM playlist_videos * WHERE plid = $1", plid) + PG_DB.exec("DELETE FROM playlists * WHERE id = $1", plid) + + env.response.status_code = 204 + end + + def self.insert_video_into_playlist(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + user = env.get("user").as(User) + + plid = env.params.url["plid"] + + playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if !playlist || playlist.author != user.email && playlist.privacy.private? + return error_json(404, "Playlist does not exist.") + end + + if playlist.author != user.email + return error_json(403, "Invalid user") + end + + if playlist.index.size >= 500 + return error_json(400, "Playlist cannot have more than 500 videos") + end + + video_id = env.params.json["videoId"].try &.as(String) + if !video_id + return error_json(403, "Invalid videoId") + end + + begin + video = get_video(video_id, PG_DB) + rescue ex + return error_json(500, ex) + end + + playlist_video = PlaylistVideo.new({ + title: video.title, + id: video.id, + author: video.author, + ucid: video.ucid, + length_seconds: video.length_seconds, + published: video.published, + plid: plid, + live_now: video.live_now, + index: Random::Secure.rand(0_i64..Int64::MAX), + }) + + video_array = playlist_video.to_a + args = arg_array(video_array) + + PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array) + PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, plid) + + env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{plid}/videos/#{playlist_video.index.to_u64.to_s(16).upcase}" + env.response.status_code = 201 + playlist_video.to_json(locale, index: playlist.index.size) + end + + def self.delete_video_in_playlist(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + user = env.get("user").as(User) + + plid = env.params.url["plid"] + index = env.params.url["index"].to_i64(16) + + playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if !playlist || playlist.author != user.email && playlist.privacy.private? + return error_json(404, "Playlist does not exist.") + end + + if playlist.author != user.email + return error_json(403, "Invalid user") + end + + if !playlist.index.includes? index + return error_json(404, "Playlist does not contain index") + end + + PG_DB.exec("DELETE FROM playlist_videos * WHERE index = $1", index) + PG_DB.exec("UPDATE playlists SET index = array_remove(index, $1), video_count = cardinality(index) - 1, updated = $2 WHERE id = $3", index, Time.utc, plid) + + env.response.status_code = 204 + end + + # Invidious::Routing.patch "/api/v1/auth/playlists/:plid/videos/:index" + # def modify_playlist_at(env) + # TODO + # end + + def self.get_tokens(env) + env.response.content_type = "application/json" + user = env.get("user").as(User) + scopes = env.get("scopes").as(Array(String)) + + tokens = PG_DB.query_all("SELECT id, issued FROM session_ids WHERE email = $1", user.email, as: {session: String, issued: Time}) + + JSON.build do |json| + json.array do + tokens.each do |token| + json.object do + json.field "session", token[:session] + json.field "issued", token[:issued].to_unix + end + end + end + end + end + + def self.register_token(env) + user = env.get("user").as(User) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + case env.request.headers["Content-Type"]? + when "application/x-www-form-urlencoded" + scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v } + callback_url = env.params.body["callbackUrl"]? + expire = env.params.body["expire"]?.try &.to_i? + when "application/json" + scopes = env.params.json["scopes"].as(Array).map { |v| v.as_s } + callback_url = env.params.json["callbackUrl"]?.try &.as(String) + expire = env.params.json["expire"]?.try &.as(Int64) + else + return error_json(400, "Invalid or missing header 'Content-Type'") + end + + if callback_url && callback_url.empty? + callback_url = nil + end + + if callback_url + callback_url = URI.parse(callback_url) + end + + if sid = env.get?("sid").try &.as(String) + env.response.content_type = "text/html" + + csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, PG_DB, use_nonce: true) + return templated "authorize_token" + else + env.response.content_type = "application/json" + + superset_scopes = env.get("scopes").as(Array(String)) + + authorized_scopes = [] of String + scopes.each do |scope| + if scopes_include_scope(superset_scopes, scope) + authorized_scopes << scope + end + end + + access_token = generate_token(user.email, authorized_scopes, expire, HMAC_KEY, PG_DB) + + if callback_url + access_token = URI.encode_www_form(access_token) + + if query = callback_url.query + query = HTTP::Params.parse(query.not_nil!) + else + query = HTTP::Params.new + end + + query["token"] = access_token + callback_url.query = query.to_s + + env.redirect callback_url.to_s + else + access_token + end + end + end + + def self.unregister_token(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + env.response.content_type = "application/json" + user = env.get("user").as(User) + scopes = env.get("scopes").as(Array(String)) + + session = env.params.json["session"]?.try &.as(String) + session ||= env.get("session").as(String) + + # Allow tokens to revoke other tokens with correct scope + if session == env.get("session").as(String) + PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", session) + elsif scopes_include_scope(scopes, "GET:tokens") + PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", session) + else + return error_json(400, "Cannot revoke session #{session}") + end + + env.response.status_code = 204 + end +end diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr new file mode 100644 index 00000000..da39661c --- /dev/null +++ b/src/invidious/routes/api/v1/channels.cr @@ -0,0 +1,278 @@ +module Invidious::Routes::API::V1::Channels + def self.home(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + ucid = env.params.url["ucid"] + sort_by = env.params.query["sort_by"]?.try &.downcase + sort_by ||= "newest" + + begin + channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) + return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) + rescue ex + return error_json(500, ex) + end + + page = 1 + if channel.auto_generated + videos = [] of SearchVideo + count = 0 + else + begin + count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) + rescue ex + return error_json(500, ex) + end + end + + JSON.build do |json| + # TODO: Refactor into `to_json` for InvidiousChannel + json.object do + json.field "author", channel.author + json.field "authorId", channel.ucid + json.field "authorUrl", channel.author_url + + json.field "authorBanners" do + json.array do + if channel.banner + qualities = { + {width: 2560, height: 424}, + {width: 2120, height: 351}, + {width: 1060, height: 175}, + } + qualities.each do |quality| + json.object do + json.field "url", channel.banner.not_nil!.gsub("=w1060-", "=w#{quality[:width]}-") + json.field "width", quality[:width] + json.field "height", quality[:height] + end + end + + json.object do + json.field "url", channel.banner.not_nil!.split("=w1060-")[0] + json.field "width", 512 + json.field "height", 288 + end + end + end + end + + json.field "authorThumbnails" do + json.array do + qualities = {32, 48, 76, 100, 176, 512} + + qualities.each do |quality| + json.object do + json.field "url", channel.author_thumbnail.gsub(/=s\d+/, "=s#{quality}") + json.field "width", quality + json.field "height", quality + end + end + end + end + + json.field "subCount", channel.sub_count + json.field "totalViews", channel.total_views + json.field "joined", channel.joined.to_unix + + json.field "autoGenerated", channel.auto_generated + json.field "isFamilyFriendly", channel.is_family_friendly + json.field "description", html_to_content(channel.description_html) + json.field "descriptionHtml", channel.description_html + + json.field "allowedRegions", channel.allowed_regions + + json.field "latestVideos" do + json.array do + videos.each do |video| + video.to_json(locale, json) + end + end + end + + json.field "relatedChannels" do + json.array do + channel.related_channels.each do |related_channel| + json.object do + json.field "author", related_channel.author + json.field "authorId", related_channel.ucid + json.field "authorUrl", related_channel.author_url + + json.field "authorThumbnails" do + json.array do + qualities = {32, 48, 76, 100, 176, 512} + + qualities.each do |quality| + json.object do + json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}") + json.field "width", quality + json.field "height", quality + end + end + end + end + end + end + end + end + end + end + end + + def self.latest(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + ucid = env.params.url["ucid"] + + begin + videos = get_latest_videos(ucid) + rescue ex + return error_json(500, ex) + end + + JSON.build do |json| + json.array do + videos.each do |video| + video.to_json(locale, json) + end + end + end + end + + def self.videos(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + ucid = env.params.url["ucid"] + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + sort_by = env.params.query["sort"]?.try &.downcase + sort_by ||= env.params.query["sort_by"]?.try &.downcase + sort_by ||= "newest" + + begin + channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) + return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) + rescue ex + return error_json(500, ex) + end + + begin + count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) + rescue ex + return error_json(500, ex) + end + + JSON.build do |json| + json.array do + videos.each do |video| + video.to_json(locale, json) + end + end + end + end + + def self.playlists(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + ucid = env.params.url["ucid"] + continuation = env.params.query["continuation"]? + sort_by = env.params.query["sort"]?.try &.downcase || + env.params.query["sort_by"]?.try &.downcase || + "last" + + begin + channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) + return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) + rescue ex + return error_json(500, ex) + end + + items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) + + JSON.build do |json| + json.object do + json.field "playlists" do + json.array do + items.each do |item| + item.to_json(locale, json) if item.is_a?(SearchPlaylist) + end + end + end + + json.field "continuation", continuation + end + end + end + + def self.community(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + ucid = env.params.url["ucid"] + + thin_mode = env.params.query["thin_mode"]? + thin_mode = thin_mode == "true" + + format = env.params.query["format"]? + format ||= "json" + + continuation = env.params.query["continuation"]? + # sort_by = env.params.query["sort_by"]?.try &.downcase + + begin + fetch_channel_community(ucid, continuation, locale, format, thin_mode) + rescue ex + return error_json(500, ex) + end + end + + def self.search(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + ucid = env.params.url["ucid"] + + query = env.params.query["q"]? + query ||= "" + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + count, search_results = channel_search(query, page, ucid) + JSON.build do |json| + json.array do + search_results.each do |item| + item.to_json(locale, json) + end + end + end + end + + # 301 redirect from /api/v1/channels/comments/:ucid + # and /api/v1/channels/:ucid/comments to new /api/v1/channels/:ucid/community and + # corresponding equivalent URL structure of the other one. + def self.channel_comments_redirect(env) + env.response.content_type = "application/json" + ucid = env.params.url["ucid"] + + env.response.headers["Location"] = "/api/v1/channels/#{ucid}/community?#{env.params.query}" + env.response.status_code = 301 + return + end +end diff --git a/src/invidious/routes/api/v1/feeds.cr b/src/invidious/routes/api/v1/feeds.cr new file mode 100644 index 00000000..bb8f661b --- /dev/null +++ b/src/invidious/routes/api/v1/feeds.cr @@ -0,0 +1,45 @@ +module Invidious::Routes::API::V1::Feeds + def self.trending(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + region = env.params.query["region"]? + trending_type = env.params.query["type"]? + + begin + trending, plid = fetch_trending(trending_type, region, locale) + rescue ex + return error_json(500, ex) + end + + videos = JSON.build do |json| + json.array do + trending.each do |video| + video.to_json(locale, json) + end + end + end + + videos + end + + def self.popular(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + if !CONFIG.popular_enabled + error_message = {"error" => "Administrator has disabled this endpoint."}.to_json + haltf env, 400, error_message + end + + JSON.build do |json| + json.array do + popular_videos.each do |video| + video.to_json(locale, json) + end + end + end + end +end diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr new file mode 100644 index 00000000..cf95bd9b --- /dev/null +++ b/src/invidious/routes/api/v1/misc.cr @@ -0,0 +1,136 @@ +module Invidious::Routes::API::V1::Misc + # Stats API endpoint for Invidious + def self.stats(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + env.response.content_type = "application/json" + + if !CONFIG.statistics_enabled + return error_json(400, "Statistics are not enabled.") + end + + Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json + end + + # APIv1 currently uses the same logic for both + # user playlists and Invidious playlists. This means that we can't + # reasonably split them yet. This should be addressed in APIv2 + def self.get_playlist(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + plid = env.params.url["plid"] + + offset = env.params.query["index"]?.try &.to_i? + offset ||= env.params.query["page"]?.try &.to_i?.try { |page| (page - 1) * 100 } + offset ||= 0 + + continuation = env.params.query["continuation"]? + + format = env.params.query["format"]? + format ||= "json" + + if plid.starts_with? "RD" + return env.redirect "/api/v1/mixes/#{plid}" + end + + begin + playlist = get_playlist(PG_DB, plid, locale) + rescue ex : InfoException + return error_json(404, ex) + rescue ex + return error_json(404, "Playlist does not exist.") + end + + user = env.get?("user").try &.as(User) + if !playlist || playlist.privacy.private? && playlist.author != user.try &.email + return error_json(404, "Playlist does not exist.") + end + + response = playlist.to_json(offset, locale, continuation: continuation) + + if format == "html" + response = JSON.parse(response) + playlist_html = template_playlist(response) + index, next_video = response["videos"].as_a.skip(1).select { |video| !video["author"].as_s.empty? }[0]?.try { |v| {v["index"], v["videoId"]} } || {nil, nil} + + response = { + "playlistHtml" => playlist_html, + "index" => index, + "nextVideo" => next_video, + }.to_json + end + + response + end + + def self.mixes(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + rdid = env.params.url["rdid"] + + continuation = env.params.query["continuation"]? + continuation ||= rdid.lchop("RD")[0, 11] + + format = env.params.query["format"]? + format ||= "json" + + begin + mix = fetch_mix(rdid, continuation, locale: locale) + + if !rdid.ends_with? continuation + mix = fetch_mix(rdid, mix.videos[1].id) + index = mix.videos.index(mix.videos.select { |video| video.id == continuation }[0]?) + end + + mix.videos = mix.videos[index..-1] + rescue ex + return error_json(500, ex) + end + + response = JSON.build do |json| + json.object do + json.field "title", mix.title + json.field "mixId", mix.id + + json.field "videos" do + json.array do + mix.videos.each do |video| + json.object do + json.field "title", video.title + json.field "videoId", video.id + json.field "author", video.author + + json.field "authorId", video.ucid + json.field "authorUrl", "/channel/#{video.ucid}" + + json.field "videoThumbnails" do + json.array do + generate_thumbnails(json, video.id) + end + end + + json.field "index", video.index + json.field "lengthSeconds", video.length_seconds + end + end + end + end + end + end + + if format == "html" + response = JSON.parse(response) + playlist_html = template_mix(response) + next_video = response["videos"].as_a.select { |video| !video["author"].as_s.empty? }[0]?.try &.["videoId"] + + response = { + "playlistHtml" => playlist_html, + "nextVideo" => next_video, + }.to_json + end + + response + end +end diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr new file mode 100644 index 00000000..f3a6fa06 --- /dev/null +++ b/src/invidious/routes/api/v1/search.cr @@ -0,0 +1,78 @@ +module Invidious::Routes::API::V1::Search + def self.search(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + region = env.params.query["region"]? + + env.response.content_type = "application/json" + + query = env.params.query["q"]? + query ||= "" + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + sort_by = env.params.query["sort_by"]?.try &.downcase + sort_by ||= "relevance" + + date = env.params.query["date"]?.try &.downcase + date ||= "" + + duration = env.params.query["duration"]?.try &.downcase + duration ||= "" + + features = env.params.query["features"]?.try &.split(",").map { |feature| feature.downcase } + features ||= [] of String + + content_type = env.params.query["type"]?.try &.downcase + content_type ||= "video" + + begin + search_params = produce_search_params(page, sort_by, date, content_type, duration, features) + rescue ex + return error_json(400, ex) + end + + count, search_results = search(query, search_params, region).as(Tuple) + JSON.build do |json| + json.array do + search_results.each do |item| + item.to_json(locale, json) + end + end + end + end + + def self.search_suggestions(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + region = env.params.query["region"]? + + env.response.content_type = "application/json" + + query = env.params.query["q"]? + query ||= "" + + begin + headers = HTTP::Headers{":authority" => "suggestqueries.google.com"} + response = YT_POOL.client &.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback", headers).body + + body = response[35..-2] + body = JSON.parse(body).as_a + suggestions = body[1].as_a[0..-2] + + JSON.build do |json| + json.object do + json.field "query", body[0].as_s + json.field "suggestions" do + json.array do + suggestions.each do |suggestion| + json.string suggestion[0].as_s + end + end + end + end + end + rescue ex + return error_json(500, ex) + end + end +end diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr new file mode 100644 index 00000000..575e6fdf --- /dev/null +++ b/src/invidious/routes/api/v1/videos.cr @@ -0,0 +1,363 @@ +module Invidious::Routes::API::V1::Videos + def self.videos(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + id = env.params.url["id"] + region = env.params.query["region"]? + + begin + video = get_video(id, PG_DB, region: region) + rescue ex : VideoRedirect + env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) + return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) + rescue ex + return error_json(500, ex) + end + + video.to_json(locale) + end + + def self.captions(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + id = env.params.url["id"] + region = env.params.query["region"]? + + # See https://github.com/ytdl-org/youtube-dl/blob/6ab30ff50bf6bd0585927cb73c7421bef184f87a/youtube_dl/extractor/youtube.py#L1354 + # It is possible to use `/api/timedtext?type=list&v=#{id}` and + # `/api/timedtext?type=track&v=#{id}&lang=#{lang_code}` directly, + # but this does not provide links for auto-generated captions. + # + # In future this should be investigated as an alternative, since it does not require + # getting video info. + + begin + video = get_video(id, PG_DB, region: region) + rescue ex : VideoRedirect + env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) + return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) + rescue ex + haltf env, 500 + end + + captions = video.captions + + label = env.params.query["label"]? + lang = env.params.query["lang"]? + tlang = env.params.query["tlang"]? + + if !label && !lang + response = JSON.build do |json| + json.object do + json.field "captions" do + json.array do + captions.each do |caption| + json.object do + json.field "label", caption.name + json.field "languageCode", caption.languageCode + json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}" + end + end + end + end + end + end + + return response + end + + env.response.content_type = "text/vtt; charset=UTF-8" + + if lang + caption = captions.select { |caption| caption.languageCode == lang } + else + caption = captions.select { |caption| caption.name == label } + end + + if caption.empty? + haltf env, 404 + else + caption = caption[0] + end + + url = URI.parse("#{caption.baseUrl}&tlang=#{tlang}").request_target + + # Auto-generated captions often have cues that aren't aligned properly with the video, + # as well as some other markup that makes it cumbersome, so we try to fix that here + if caption.name.includes? "auto-generated" + caption_xml = YT_POOL.client &.get(url).body + caption_xml = XML.parse(caption_xml) + + webvtt = String.build do |str| + str << <<-END_VTT + WEBVTT + Kind: captions + Language: #{tlang || caption.languageCode} + + + END_VTT + + caption_nodes = caption_xml.xpath_nodes("//transcript/text") + caption_nodes.each_with_index do |node, i| + start_time = node["start"].to_f.seconds + duration = node["dur"]?.try &.to_f.seconds + duration ||= start_time + + if caption_nodes.size > i + 1 + end_time = caption_nodes[i + 1]["start"].to_f.seconds + else + end_time = start_time + duration + end + + start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}" + end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}" + + text = HTML.unescape(node.content) + text = text.gsub(/<font color="#[a-fA-F0-9]{6}">/, "") + text = text.gsub(/<\/font>/, "") + if md = text.match(/(?<name>.*) : (?<text>.*)/) + text = "<v #{md["name"]}>#{md["text"]}</v>" + end + + str << <<-END_CUE + #{start_time} --> #{end_time} + #{text} + + + END_CUE + end + end + else + webvtt = YT_POOL.client &.get("#{url}&format=vtt").body + end + + if title = env.params.query["title"]? + # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ + env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}" + end + + webvtt + end + + # Fetches YouTube storyboards + # + # Which are sprites containing x * y preview + # thumbnails for individual scenes in a video. + # See https://support.jwplayer.com/articles/how-to-add-preview-thumbnails + def self.storyboards(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + id = env.params.url["id"] + region = env.params.query["region"]? + + begin + video = get_video(id, PG_DB, region: region) + rescue ex : VideoRedirect + env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) + return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) + rescue ex + haltf env, 500 + end + + storyboards = video.storyboards + width = env.params.query["width"]? + height = env.params.query["height"]? + + if !width && !height + response = JSON.build do |json| + json.object do + json.field "storyboards" do + generate_storyboards(json, id, storyboards) + end + end + end + + return response + end + + env.response.content_type = "text/vtt" + + storyboard = storyboards.select { |storyboard| width == "#{storyboard[:width]}" || height == "#{storyboard[:height]}" } + + if storyboard.empty? + haltf env, 404 + else + storyboard = storyboard[0] + end + + String.build do |str| + str << <<-END_VTT + WEBVTT + END_VTT + + start_time = 0.milliseconds + end_time = storyboard[:interval].milliseconds + + storyboard[:storyboard_count].times do |i| + url = storyboard[:url] + authority = /(i\d?).ytimg.com/.match(url).not_nil![1]? + url = url.gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "") + url = "#{HOST_URL}/sb/#{authority}/#{url}" + + storyboard[:storyboard_height].times do |j| + storyboard[:storyboard_width].times do |k| + str << <<-END_CUE + #{start_time}.000 --> #{end_time}.000 + #{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]} + + + END_CUE + + start_time += storyboard[:interval].milliseconds + end_time += storyboard[:interval].milliseconds + end + end + end + end + end + + def self.annotations(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "text/xml" + + id = env.params.url["id"] + source = env.params.query["source"]? + source ||= "archive" + + if !id.match(/[a-zA-Z0-9_-]{11}/) + haltf env, 400 + end + + annotations = "" + + case source + when "archive" + if CONFIG.cache_annotations && (cached_annotation = PG_DB.query_one?("SELECT * FROM annotations WHERE id = $1", id, as: Annotation)) + annotations = cached_annotation.annotations + else + index = CHARS_SAFE.index(id[0]).not_nil!.to_s.rjust(2, '0') + + # IA doesn't handle leading hyphens, + # so we use https://archive.org/details/youtubeannotations_64 + if index == "62" + index = "64" + id = id.sub(/^-/, 'A') + end + + file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml") + + location = make_client(ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}")) + + if !location.headers["Location"]? + env.response.status_code = location.status_code + end + + response = make_client(URI.parse(location.headers["Location"]), &.get(location.headers["Location"])) + + if response.body.empty? + haltf env, 404 + end + + if response.status_code != 200 + haltf env, response.status_code + end + + annotations = response.body + + cache_annotation(PG_DB, id, annotations) + end + else # "youtube" + response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}") + + if response.status_code != 200 + haltf env, response.status_code + end + + annotations = response.body + end + + etag = sha256(annotations)[0, 16] + if env.request.headers["If-None-Match"]?.try &.== etag + haltf env, 304 + else + env.response.headers["ETag"] = etag + annotations + end + end + + def self.comments(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + region = env.params.query["region"]? + + env.response.content_type = "application/json" + + id = env.params.url["id"] + + source = env.params.query["source"]? + source ||= "youtube" + + thin_mode = env.params.query["thin_mode"]? + thin_mode = thin_mode == "true" + + format = env.params.query["format"]? + format ||= "json" + + action = env.params.query["action"]? + action ||= "action_get_comments" + + continuation = env.params.query["continuation"]? + sort_by = env.params.query["sort_by"]?.try &.downcase + + if source == "youtube" + sort_by ||= "top" + + begin + comments = fetch_youtube_comments(id, continuation, format, locale, thin_mode, region, sort_by: sort_by) + rescue ex + return error_json(500, ex) + end + + return comments + elsif source == "reddit" + sort_by ||= "confidence" + + begin + comments, reddit_thread = fetch_reddit_comments(id, sort_by: sort_by) + content_html = template_reddit_comments(comments, locale) + + content_html = fill_links(content_html, "https", "www.reddit.com") + content_html = replace_links(content_html) + rescue ex + comments = nil + reddit_thread = nil + content_html = "" + end + + if !reddit_thread || !comments + haltf env, 404 + end + + if format == "json" + reddit_thread = JSON.parse(reddit_thread.to_json).as_h + reddit_thread["comments"] = JSON.parse(comments.to_json) + + return reddit_thread.to_json + else + response = { + "title" => reddit_thread.title, + "permalink" => reddit_thread.permalink, + "contentHtml" => content_html, + } + + return response.to_json + end + end + end +end diff --git a/src/invidious/routes/base_route.cr b/src/invidious/routes/base_route.cr deleted file mode 100644 index 37624267..00000000 --- a/src/invidious/routes/base_route.cr +++ /dev/null @@ -1,6 +0,0 @@ -abstract class Invidious::Routes::BaseRoute - private getter config : Config - - def initialize(@config) - end -end diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr new file mode 100644 index 00000000..11c2f869 --- /dev/null +++ b/src/invidious/routes/channels.cr @@ -0,0 +1,173 @@ +{% skip_file if flag?(:api_only) %} + +module Invidious::Routes::Channels + def self.home(env) + self.videos(env) + end + + def self.videos(env) + data = self.fetch_basic_information(env) + if !data.is_a?(Tuple) + return data + end + locale, user, subscriptions, continuation, ucid, channel = data + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + sort_by = env.params.query["sort_by"]?.try &.downcase + + if channel.auto_generated + sort_options = {"last", "oldest", "newest"} + sort_by ||= "last" + + items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) + items.uniq! do |item| + if item.responds_to?(:title) + item.title + elsif item.responds_to?(:author) + item.author + end + end + items = items.select(&.is_a?(SearchPlaylist)).map(&.as(SearchPlaylist)) + items.each { |item| item.author = "" } + else + sort_options = {"newest", "oldest", "popular"} + sort_by ||= "newest" + + count, items = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) + end + + templated "channel" + end + + def self.playlists(env) + data = self.fetch_basic_information(env) + if !data.is_a?(Tuple) + return data + end + locale, user, subscriptions, continuation, ucid, channel = data + + sort_options = {"last", "oldest", "newest"} + sort_by = env.params.query["sort_by"]?.try &.downcase + sort_by ||= "last" + + if channel.auto_generated + return env.redirect "/channel/#{channel.ucid}" + end + + items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) + items = items.select { |item| item.is_a?(SearchPlaylist) }.map { |item| item.as(SearchPlaylist) } + items.each { |item| item.author = "" } + + templated "playlists" + end + + def self.community(env) + data = self.fetch_basic_information(env) + if !data.is_a?(Tuple) + return data + end + locale, user, subscriptions, continuation, ucid, channel = data + + thin_mode = env.params.query["thin_mode"]? || env.get("preferences").as(Preferences).thin_mode + thin_mode = thin_mode == "true" + + continuation = env.params.query["continuation"]? + # sort_by = env.params.query["sort_by"]?.try &.downcase + + if !channel.tabs.includes? "community" + return env.redirect "/channel/#{channel.ucid}" + end + + begin + items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode)) + rescue ex : InfoException + env.response.status_code = 500 + error_message = ex.message + rescue ex + return error_template(500, ex) + end + + templated "community" + end + + def self.about(env) + data = self.fetch_basic_information(env) + if !data.is_a?(Tuple) + return data + end + locale, user, subscriptions, continuation, ucid, channel = data + + env.redirect "/channel/#{ucid}" + end + + # Redirects brand url channels to a normal /channel/:ucid route + def self.brand_redirect(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + # /attribution_link endpoint needs both the `a` and `u` parameter + # and in order to avoid detection from YouTube we should only send the required ones + # without any of the additional url parameters that only Invidious uses. + yt_url_params = URI::Params.encode(env.params.query.to_h.select(["a", "u", "user"])) + + # Retrieves URL params that only Invidious uses + invidious_url_params = URI::Params.encode(env.params.query.to_h.select!(["a", "u", "user"])) + + begin + resolved_url = YoutubeAPI.resolve_url("https://youtube.com#{env.request.path}#{yt_url_params.size > 0 ? "?#{yt_url_params}" : ""}") + ucid = resolved_url["endpoint"]["browseEndpoint"]["browseId"] + rescue ex : InfoException | KeyError + raise InfoException.new(translate(locale, "This channel does not exist.")) + end + + selected_tab = env.request.path.split("/")[-1] + if ["home", "videos", "playlists", "community", "channels", "about"].includes? selected_tab + url = "/channel/#{ucid}/#{selected_tab}" + else + url = "/channel/#{ucid}" + end + + env.redirect url + end + + # Handles redirects for the /profile endpoint + def self.profile(env) + # The /profile endpoint is special. If passed into the resolve_url + # endpoint YouTube would return a sign in page instead of an /channel/:ucid + # thus we'll add an edge case and handle it here. + + uri_params = env.params.query.size > 0 ? "?#{env.params.query}" : "" + + user = env.params.query["user"]? + if !user + raise InfoException.new("This channel does not exist.") + else + env.redirect "/user/#{user}#{uri_params}" + end + end + + private def self.fetch_basic_information(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + if user + user = user.as(User) + subscriptions = user.subscriptions + end + subscriptions ||= [] of String + + ucid = env.params.url["ucid"] + continuation = env.params.query["continuation"]? + + begin + channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + return env.redirect env.request.resource.gsub(ucid, ex.channel_id) + rescue ex + return error_template(500, ex) + end + + return {locale, user, subscriptions, continuation, ucid, channel} + end +end diff --git a/src/invidious/routes/embed/show.cr b/src/invidious/routes/embed.cr index 8a655556..80d09789 100644 --- a/src/invidious/routes/embed/show.cr +++ b/src/invidious/routes/embed.cr @@ -1,5 +1,31 @@ -class Invidious::Routes::Embed::Show < Invidious::Routes::BaseRoute - def handle(env) +{% skip_file if flag?(:api_only) %} + +module Invidious::Routes::Embed + def self.redirect(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") + begin + playlist = get_playlist(PG_DB, plid, locale: locale) + offset = env.params.query["index"]?.try &.to_i? || 0 + videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale) + rescue ex + return error_template(500, ex) + end + + url = "/embed/#{videos[0].id}?#{env.params.query}" + + if env.params.query.size > 0 + url += "?#{env.params.query}" + end + else + url = "/" + end + + env.redirect url + end + + def self.show(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? id = env.params.url["id"] @@ -120,8 +146,8 @@ class Invidious::Routes::Embed::Show < Invidious::Routes::BaseRoute adaptive_fmts = video.adaptive_fmts if params.local - fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) } - adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) } + fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) } + adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) } end video_streams = video.video_streams @@ -141,11 +167,11 @@ class Invidious::Routes::Embed::Show < Invidious::Routes::BaseRoute captions = video.captions preferred_captions = captions.select { |caption| - params.preferred_captions.includes?(caption.name.simpleText) || + params.preferred_captions.includes?(caption.name) || params.preferred_captions.includes?(caption.languageCode.split("-")[0]) } preferred_captions.sort_by! { |caption| - (params.preferred_captions.index(caption.name.simpleText) || + (params.preferred_captions.index(caption.name) || params.preferred_captions.index(caption.languageCode.split("-")[0])).not_nil! } captions = captions - preferred_captions diff --git a/src/invidious/routes/embed/index.cr b/src/invidious/routes/embed/index.cr deleted file mode 100644 index 32a4966b..00000000 --- a/src/invidious/routes/embed/index.cr +++ /dev/null @@ -1,25 +0,0 @@ -class Invidious::Routes::Embed::Index < Invidious::Routes::BaseRoute - def handle(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") - begin - playlist = get_playlist(PG_DB, plid, locale: locale) - offset = env.params.query["index"]?.try &.to_i? || 0 - videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale) - rescue ex - return error_template(500, ex) - end - - url = "/embed/#{videos[0].id}?#{env.params.query}" - - if env.params.query.size > 0 - url += "?#{env.params.query}" - end - else - url = "/" - end - - env.redirect url - end -end diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr new file mode 100644 index 00000000..d9280529 --- /dev/null +++ b/src/invidious/routes/feeds.cr @@ -0,0 +1,433 @@ +{% skip_file if flag?(:api_only) %} + +module Invidious::Routes::Feeds + def self.view_all_playlists_redirect(env) + env.redirect "/feed/playlists" + end + + def self.playlists(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + referer = get_referer(env) + + return env.redirect "/" if user.nil? + + user = user.as(User) + + items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + items_created.map! do |item| + item.author = "" + item + end + + items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + items_saved.map! do |item| + item.author = "" + item + end + + templated "feeds/playlists" + end + + def self.popular(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + if CONFIG.popular_enabled + templated "feeds/popular" + else + message = translate(locale, "The Popular feed has been disabled by the administrator.") + templated "message" + end + end + + def self.trending(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + trending_type = env.params.query["type"]? + trending_type ||= "Default" + + region = env.params.query["region"]? + region ||= "US" + + begin + trending, plid = fetch_trending(trending_type, region, locale) + rescue ex + return error_template(500, ex) + end + + templated "feeds/trending" + end + + def self.subscriptions(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + sid = sid.as(String) + token = user.token + + if user.preferences.unseen_only + env.set "show_watched", true + end + + # Refresh account + headers = HTTP::Headers.new + headers["Cookie"] = env.request.headers["Cookie"] + + if !user.password + user, sid = get_user(sid, headers, PG_DB) + end + + max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE) + max_results ||= user.preferences.max_results + max_results ||= CONFIG.default_user_preferences.max_results + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) + + # "updated" here is used for delivering new notifications, so if + # we know a user has looked at their feed e.g. in the past 10 minutes, + # they've already seen a video posted 20 minutes ago, and don't need + # to be notified. + PG_DB.exec("UPDATE users SET notifications = $1, updated = $2 WHERE email = $3", [] of String, Time.utc, + user.email) + user.notifications = [] of String + env.set "user", user + + templated "feeds/subscriptions" + end + + def self.history(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + referer = get_referer(env) + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + if !user + return env.redirect referer + end + + user = user.as(User) + + max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE) + max_results ||= user.preferences.max_results + max_results ||= CONFIG.default_user_preferences.max_results + + if user.watched[(page - 1) * max_results]? + watched = user.watched.reverse[(page - 1) * max_results, max_results] + end + watched ||= [] of String + + templated "feeds/history" + end + + # RSS feeds + + def self.rss_channel(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.headers["Content-Type"] = "application/atom+xml" + env.response.content_type = "application/atom+xml" + + ucid = env.params.url["ucid"] + + params = HTTP::Params.parse(env.params.query["params"]? || "") + + begin + channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + return env.redirect env.request.resource.gsub(ucid, ex.channel_id) + rescue ex + return error_atom(500, ex) + end + + response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}") + rss = XML.parse_html(response.body) + + videos = rss.xpath_nodes("//feed/entry").map do |entry| + video_id = entry.xpath_node("videoid").not_nil!.content + title = entry.xpath_node("title").not_nil!.content + + published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) + updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) + + author = entry.xpath_node("author/name").not_nil!.content + ucid = entry.xpath_node("channelid").not_nil!.content + description_html = entry.xpath_node("group/description").not_nil!.to_s + views = entry.xpath_node("group/community/statistics").not_nil!.["views"].to_i64 + + SearchVideo.new({ + title: title, + id: video_id, + author: author, + ucid: ucid, + published: published, + views: views, + description_html: description_html, + length_seconds: 0, + live_now: false, + paid: false, + premium: false, + premiere_timestamp: nil, + }) + end + + XML.build(indent: " ", encoding: "UTF-8") do |xml| + xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", + "xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom", + "xml:lang": "en-US") do + xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}") + xml.element("id") { xml.text "yt:channel:#{channel.ucid}" } + xml.element("yt:channelId") { xml.text channel.ucid } + xml.element("icon") { xml.text channel.author_thumbnail } + xml.element("title") { xml.text channel.author } + xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{channel.ucid}") + + xml.element("author") do + xml.element("name") { xml.text channel.author } + xml.element("uri") { xml.text "#{HOST_URL}/channel/#{channel.ucid}" } + end + + videos.each do |video| + video.to_xml(channel.auto_generated, params, xml) + end + end + end + end + + def self.rss_private(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.headers["Content-Type"] = "application/atom+xml" + env.response.content_type = "application/atom+xml" + + token = env.params.query["token"]? + + if !token + haltf env, status_code: 403 + end + + user = PG_DB.query_one?("SELECT * FROM users WHERE token = $1", token.strip, as: User) + if !user + haltf env, status_code: 403 + end + + max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE) + max_results ||= user.preferences.max_results + max_results ||= CONFIG.default_user_preferences.max_results + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + params = HTTP::Params.parse(env.params.query["params"]? || "") + + videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) + + XML.build(indent: " ", encoding: "UTF-8") do |xml| + xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", + "xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom", + "xml:lang": "en-US") do + xml.element("link", "type": "text/html", rel: "alternate", href: "#{HOST_URL}/feed/subscriptions") + xml.element("link", "type": "application/atom+xml", rel: "self", + href: "#{HOST_URL}#{env.request.resource}") + xml.element("title") { xml.text translate(locale, "Invidious Private Feed for `x`", user.email) } + + (notifications + videos).each do |video| + video.to_xml(locale, params, xml) + end + end + end + end + + def self.rss_playlist(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.headers["Content-Type"] = "application/atom+xml" + env.response.content_type = "application/atom+xml" + + plid = env.params.url["plid"] + + params = HTTP::Params.parse(env.params.query["params"]? || "") + path = env.request.path + + if plid.starts_with? "IV" + if playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + videos = get_playlist_videos(PG_DB, playlist, offset: 0, locale: locale) + + return XML.build(indent: " ", encoding: "UTF-8") do |xml| + xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", + "xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom", + "xml:lang": "en-US") do + xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}") + xml.element("id") { xml.text "iv:playlist:#{plid}" } + xml.element("iv:playlistId") { xml.text plid } + xml.element("title") { xml.text playlist.title } + xml.element("link", rel: "alternate", href: "#{HOST_URL}/playlist?list=#{plid}") + + xml.element("author") do + xml.element("name") { xml.text playlist.author } + end + + videos.each do |video| + video.to_xml(false, xml) + end + end + end + else + haltf env, status_code: 404 + end + end + + response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}") + document = XML.parse(response.body) + + document.xpath_nodes(%q(//*[@href]|//*[@url])).each do |node| + node.attributes.each do |attribute| + case attribute.name + when "url", "href" + request_target = URI.parse(node[attribute.name]).request_target + query_string_opt = request_target.starts_with?("/watch?v=") ? "&#{params}" : "" + node[attribute.name] = "#{HOST_URL}#{request_target}#{query_string_opt}" + else nil # Skip + end + end + end + + document = document.to_xml(options: XML::SaveOptions::NO_DECL) + + document.scan(/<uri>(?<url>[^<]+)<\/uri>/).each do |match| + content = "#{HOST_URL}#{URI.parse(match["url"]).request_target}" + document = document.gsub(match[0], "<uri>#{content}</uri>") + end + document + end + + def self.rss_videos(env) + if ucid = env.params.query["channel_id"]? + env.redirect "/feed/channel/#{ucid}" + elsif user = env.params.query["user"]? + env.redirect "/feed/channel/#{user}" + elsif plid = env.params.query["playlist_id"]? + env.redirect "/feed/playlist/#{plid}" + end + end + + # Push notifications via PubSub + + def self.push_notifications_get(env) + verify_token = env.params.url["token"] + + mode = env.params.query["hub.mode"]? + topic = env.params.query["hub.topic"]? + challenge = env.params.query["hub.challenge"]? + + if !mode || !topic || !challenge + haltf env, status_code: 400 + else + mode = mode.not_nil! + topic = topic.not_nil! + challenge = challenge.not_nil! + end + + case verify_token + when .starts_with? "v1" + _, time, nonce, signature = verify_token.split(":") + data = "#{time}:#{nonce}" + when .starts_with? "v2" + time, signature = verify_token.split(":") + data = "#{time}" + else + haltf env, status_code: 400 + end + + # The hub will sometimes check if we're still subscribed after delivery errors, + # so we reply with a 200 as long as the request hasn't expired + if Time.utc.to_unix - time.to_i > 432000 + haltf env, status_code: 400 + end + + if OpenSSL::HMAC.hexdigest(:sha1, HMAC_KEY, data) != signature + haltf env, status_code: 400 + end + + if ucid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["channel_id"]? + PG_DB.exec("UPDATE channels SET subscribed = $1 WHERE id = $2", Time.utc, ucid) + elsif plid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["playlist_id"]? + PG_DB.exec("UPDATE playlists SET subscribed = $1 WHERE id = $2", Time.utc, ucid) + else + haltf env, status_code: 400 + end + + env.response.status_code = 200 + challenge + end + + def self.push_notifications_post(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + token = env.params.url["token"] + body = env.request.body.not_nil!.gets_to_end + signature = env.request.headers["X-Hub-Signature"].lchop("sha1=") + + if signature != OpenSSL::HMAC.hexdigest(:sha1, HMAC_KEY, body) + LOGGER.error("/feed/webhook/#{token} : Invalid signature") + haltf env, status_code: 200 + end + + spawn do + rss = XML.parse_html(body) + rss.xpath_nodes("//feed/entry").each do |entry| + id = entry.xpath_node("videoid").not_nil!.content + author = entry.xpath_node("author/name").not_nil!.content + published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) + updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) + + video = get_video(id, PG_DB, force_refresh: true) + + # Deliver notifications to `/api/v1/auth/notifications` + payload = { + "topic" => video.ucid, + "videoId" => video.id, + "published" => published.to_unix, + }.to_json + PG_DB.exec("NOTIFY notifications, E'#{payload}'") + + video = ChannelVideo.new({ + id: id, + title: video.title, + published: published, + updated: updated, + ucid: video.ucid, + author: author, + length_seconds: video.length_seconds, + live_now: video.live_now, + premiere_timestamp: video.premiere_timestamp, + views: video.views, + }) + + was_insert = PG_DB.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, + updated = $4, ucid = $5, author = $6, length_seconds = $7, + live_now = $8, premiere_timestamp = $9, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) + + PG_DB.exec("UPDATE users SET notifications = array_append(notifications, $1), + feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert + end + end + + env.response.status_code = 200 + end +end diff --git a/src/invidious/routes/home.cr b/src/invidious/routes/home.cr deleted file mode 100644 index 486a7344..00000000 --- a/src/invidious/routes/home.cr +++ /dev/null @@ -1,28 +0,0 @@ -class Invidious::Routes::Home < Invidious::Routes::BaseRoute - def handle(env) - preferences = env.get("preferences").as(Preferences) - locale = LOCALES[preferences.locale]? - user = env.get? "user" - - case preferences.default_home - when "Popular" - env.redirect "/feed/popular" - when "Trending" - env.redirect "/feed/trending" - when "Subscriptions" - if user - env.redirect "/feed/subscriptions" - else - env.redirect "/feed/popular" - end - when "Playlists" - if user - env.redirect "/view_all_playlists" - else - env.redirect "/feed/popular" - end - else - templated "empty" - end - end -end diff --git a/src/invidious/routes/licenses.cr b/src/invidious/routes/licenses.cr deleted file mode 100644 index 38fde7bb..00000000 --- a/src/invidious/routes/licenses.cr +++ /dev/null @@ -1,6 +0,0 @@ -class Invidious::Routes::Licenses < Invidious::Routes::BaseRoute - def handle(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - rendered "licenses" - end -end diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index 45a6d4d8..e7aef289 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -1,12 +1,14 @@ -class Invidious::Routes::Login < Invidious::Routes::BaseRoute - def login_page(env) +{% skip_file if flag?(:api_only) %} + +module Invidious::Routes::Login + def self.login_page(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? user = env.get? "user" return env.redirect "/feed/subscriptions" if user - if !config.login_enabled + if !CONFIG.login_enabled return error_template(400, "Login has been disabled by administrator.") end @@ -28,12 +30,12 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute templated "login" end - def login(env) + def self.login(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? referer = get_referer(env, "/feed/subscriptions") - if !config.login_enabled + if !CONFIG.login_enabled return error_template(403, "Login has been disabled by administrator.") end @@ -238,7 +240,7 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute traceback << "Logging in..." location = URI.parse(challenge_results[0][-1][2].to_s) - cookies = HTTP::Cookies.from_headers(headers) + cookies = HTTP::Cookies.from_client_headers(headers) headers.delete("Content-Type") headers.delete("Google-Accounts-XSRF") @@ -255,13 +257,13 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute traceback << "Unhandled dialog /b/0/SmsAuthInterstitial." end - login = client.get(location.full_path, headers) + login = client.get(location.request_target, headers) headers = login.cookies.add_request_headers(headers) location = login.headers["Location"]?.try { |u| URI.parse(u) } end - cookies = HTTP::Cookies.from_headers(headers) + cookies = HTTP::Cookies.from_client_headers(headers) sid = cookies["SID"]?.try &.value if !sid raise "Couldn't get SID." @@ -274,14 +276,14 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute host = URI.parse(env.request.headers["Host"]).host - if Kemal.config.ssl || config.https_only + if Kemal.config.ssl || CONFIG.https_only secure = true else secure = false end cookies.each do |cookie| - if Kemal.config.ssl || config.https_only + if Kemal.config.ssl || CONFIG.https_only cookie.secure = secure else cookie.secure = secure @@ -330,14 +332,14 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) - if Kemal.config.ssl || config.https_only + if Kemal.config.ssl || CONFIG.https_only secure = true else secure = false end - if config.domain - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{config.domain}", value: sid, expires: Time.utc + 2.years, + if CONFIG.domain + env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{CONFIG.domain}", value: sid, expires: Time.utc + 2.years, secure: secure, http_only: true) else env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.utc + 2.years, @@ -354,7 +356,7 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute env.response.cookies << cookie end else - if !config.registration_enabled + if !CONFIG.registration_enabled return error_template(400, "Registration has been disabled by administrator.") end @@ -369,7 +371,7 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute password = password.byte_slice(0, 55) - if config.captcha_enabled + if CONFIG.captcha_enabled captcha_type = env.params.body["captcha_type"]? answer = env.params.body["answer"]? change_type = env.params.body["change_type"]? @@ -434,6 +436,13 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) user, sid = create_user(sid, email, password) + + if language_header = env.request.headers["Accept-Language"]? + if language = ANG.language_negotiator.best(language_header, LOCALES.keys) + user.preferences.locale = language.header + end + end + user_array = user.to_a user_array[4] = user_array[4].to_json # User preferences @@ -445,14 +454,14 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute view_name = "subscriptions_#{sha256(user.email)}" PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") - if Kemal.config.ssl || config.https_only + if Kemal.config.ssl || CONFIG.https_only secure = true else secure = false end - if config.domain - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{config.domain}", value: sid, expires: Time.utc + 2.years, + if CONFIG.domain + env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{CONFIG.domain}", value: sid, expires: Time.utc + 2.years, secure: secure, http_only: true) else env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.utc + 2.years, @@ -475,7 +484,7 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute end end - def signout(env) + def self.signout(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? user = env.get? "user" diff --git a/src/invidious/routes/misc.cr b/src/invidious/routes/misc.cr new file mode 100644 index 00000000..0e6356d0 --- /dev/null +++ b/src/invidious/routes/misc.cr @@ -0,0 +1,51 @@ +{% skip_file if flag?(:api_only) %} + +module Invidious::Routes::Misc + def self.home(env) + preferences = env.get("preferences").as(Preferences) + locale = LOCALES[preferences.locale]? + user = env.get? "user" + + case preferences.default_home + when "Popular" + env.redirect "/feed/popular" + when "Trending" + env.redirect "/feed/trending" + when "Subscriptions" + if user + env.redirect "/feed/subscriptions" + else + env.redirect "/feed/popular" + end + when "Playlists" + if user + env.redirect "/feed/playlists" + else + env.redirect "/feed/popular" + end + else + templated "search_homepage", navbar_search: false + end + end + + def self.privacy(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + templated "privacy" + end + + def self.licenses(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + rendered "licenses" + end + + def self.cross_instance_redirect(env) + referer = get_referer(env) + + if !env.get("preferences").as(Preferences).automatic_instance_redirect + return env.redirect("https://redirect.invidious.io#{referer}") + end + + instance_url = fetch_random_instance + env.redirect "https://#{instance_url}#{referer}" + end +end diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index 6c899054..5ab15093 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -1,30 +1,7 @@ -class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute - def index(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - referer = get_referer(env) - - return env.redirect "/" if user.nil? - - user = user.as(User) - - items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) - items_created.map! do |item| - item.author = "" - item - end - - items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) - items_saved.map! do |item| - item.author = "" - item - end - - templated "view_all_playlists" - end +{% skip_file if flag?(:api_only) %} - def new(env) +module Invidious::Routes::Playlists + def self.new(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? user = env.get? "user" @@ -40,7 +17,7 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute templated "create_playlist" end - def create(env) + def self.create(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? user = env.get? "user" @@ -78,7 +55,7 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute env.redirect "/playlist?list=#{playlist.id}" end - def subscribe(env) + def self.subscribe(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? user = env.get? "user" @@ -95,7 +72,7 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute env.redirect "/playlist?list=#{playlist.id}" end - def delete_page(env) + def self.delete_page(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? user = env.get? "user" @@ -118,7 +95,7 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute templated "delete_playlist" end - def delete(env) + def self.delete(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? user = env.get? "user" @@ -148,10 +125,10 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute PG_DB.exec("DELETE FROM playlist_videos * WHERE plid = $1", plid) PG_DB.exec("DELETE FROM playlists * WHERE id = $1", plid) - env.redirect "/view_all_playlists" + env.redirect "/feed/playlists" end - def edit(env) + def self.edit(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? user = env.get? "user" @@ -191,7 +168,7 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute templated "edit_playlist" end - def update(env) + def self.update(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? user = env.get? "user" @@ -235,7 +212,7 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute env.redirect "/playlist?list=#{plid}" end - def add_playlist_items_page(env) + def self.add_playlist_items_page(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? user = env.get? "user" @@ -267,7 +244,7 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute query = env.params.query["q"]? if query begin - search_query, count, items = process_search_query(query, page, user, region: nil) + search_query, count, items, operators = process_search_query(query, page, user, region: nil) videos = items.select { |item| item.is_a? SearchVideo }.map { |item| item.as(SearchVideo) } rescue ex videos = [] of SearchVideo @@ -282,7 +259,7 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute templated "add_playlist_items" end - def playlist_ajax(env) + def self.playlist_ajax(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? user = env.get? "user" @@ -409,7 +386,7 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute end end - def show(env) + def self.show(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? user = env.get?("user").try &.as(User) @@ -433,6 +410,13 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute return error_template(500, ex) end + page_count = (playlist.video_count / 100).to_i + page_count += 1 if (playlist.video_count % 100) > 0 + + if page > page_count + return env.redirect "/playlist?list=#{plid}&page=#{page_count}" + end + if playlist.privacy == PlaylistPrivacy::Private && playlist.author != user.try &.email return error_template(403, "This playlist is private.") end @@ -440,7 +424,7 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute begin videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale) rescue ex - videos = [] of PlaylistVideo + return error_template(500, "Error encountered while retrieving playlist videos.<br>#{ex.message}") end if playlist.author == user.try &.email @@ -450,7 +434,7 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute templated "playlist" end - def mix(env) + def self.mix(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? rdid = env.params.query["list"]? diff --git a/src/invidious/routes/user_preferences.cr b/src/invidious/routes/preferences.cr index 7f334115..9410ac30 100644 --- a/src/invidious/routes/user_preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -1,5 +1,7 @@ -class Invidious::Routes::UserPreferences < Invidious::Routes::BaseRoute - def show(env) +{% skip_file if flag?(:api_only) %} + +module Invidious::Routes::PreferencesRoute + def self.show(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? referer = get_referer(env) @@ -9,7 +11,7 @@ class Invidious::Routes::UserPreferences < Invidious::Routes::BaseRoute templated "preferences" end - def update(env) + def self.update(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? referer = get_referer(env) @@ -60,6 +62,18 @@ class Invidious::Routes::UserPreferences < Invidious::Routes::BaseRoute volume = env.params.body["volume"]?.try &.as(String).to_i? volume ||= CONFIG.default_user_preferences.volume + extend_desc = env.params.body["extend_desc"]?.try &.as(String) + extend_desc ||= "off" + extend_desc = extend_desc == "on" + + vr_mode = env.params.body["vr_mode"]?.try &.as(String) + vr_mode ||= "off" + vr_mode = vr_mode == "on" + + show_nick = env.params.body["show_nick"]?.try &.as(String) + show_nick ||= "off" + show_nick = show_nick == "on" + comments = [] of String 2.times do |i| comments << (env.params.body["comments[#{i}]"]?.try &.as(String) || CONFIG.default_user_preferences.comments[i]) @@ -84,6 +98,10 @@ class Invidious::Routes::UserPreferences < Invidious::Routes::BaseRoute end end + automatic_instance_redirect = env.params.body["automatic_instance_redirect"]?.try &.as(String) + automatic_instance_redirect ||= "off" + automatic_instance_redirect = automatic_instance_redirect == "on" + locale = env.params.body["locale"]?.try &.as(String) locale ||= CONFIG.default_user_preferences.locale @@ -114,40 +132,44 @@ class Invidious::Routes::UserPreferences < Invidious::Routes::BaseRoute # Convert to JSON and back again to take advantage of converters used for compatability preferences = Preferences.from_json({ - annotations: annotations, - annotations_subscribed: annotations_subscribed, - autoplay: autoplay, - captions: captions, - comments: comments, - continue: continue, - continue_autoplay: continue_autoplay, - dark_mode: dark_mode, - latest_only: latest_only, - listen: listen, - local: local, - locale: locale, - max_results: max_results, - notifications_only: notifications_only, - player_style: player_style, - quality: quality, - quality_dash: quality_dash, - default_home: default_home, - feed_menu: feed_menu, - related_videos: related_videos, - sort: sort, - speed: speed, - thin_mode: thin_mode, - unseen_only: unseen_only, - video_loop: video_loop, - volume: volume, + annotations: annotations, + annotations_subscribed: annotations_subscribed, + autoplay: autoplay, + captions: captions, + comments: comments, + continue: continue, + continue_autoplay: continue_autoplay, + dark_mode: dark_mode, + latest_only: latest_only, + listen: listen, + local: local, + locale: locale, + max_results: max_results, + notifications_only: notifications_only, + player_style: player_style, + quality: quality, + quality_dash: quality_dash, + default_home: default_home, + feed_menu: feed_menu, + automatic_instance_redirect: automatic_instance_redirect, + related_videos: related_videos, + sort: sort, + speed: speed, + thin_mode: thin_mode, + unseen_only: unseen_only, + video_loop: video_loop, + volume: volume, + extend_desc: extend_desc, + vr_mode: vr_mode, + show_nick: show_nick, }.to_json).to_json if user = env.get? "user" user = user.as(User) PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences, user.email) - if config.admins.includes? user.email - config.default_user_preferences.default_home = env.params.body["admin_default_home"]?.try &.as(String) || config.default_user_preferences.default_home + if CONFIG.admins.includes? user.email + CONFIG.default_user_preferences.default_home = env.params.body["admin_default_home"]?.try &.as(String) || CONFIG.default_user_preferences.default_home admin_feed_menu = [] of String 4.times do |index| @@ -156,43 +178,42 @@ class Invidious::Routes::UserPreferences < Invidious::Routes::BaseRoute admin_feed_menu << option end end - config.default_user_preferences.feed_menu = admin_feed_menu + CONFIG.default_user_preferences.feed_menu = admin_feed_menu popular_enabled = env.params.body["popular_enabled"]?.try &.as(String) popular_enabled ||= "off" - config.popular_enabled = popular_enabled == "on" + CONFIG.popular_enabled = popular_enabled == "on" captcha_enabled = env.params.body["captcha_enabled"]?.try &.as(String) captcha_enabled ||= "off" - config.captcha_enabled = captcha_enabled == "on" + CONFIG.captcha_enabled = captcha_enabled == "on" login_enabled = env.params.body["login_enabled"]?.try &.as(String) login_enabled ||= "off" - config.login_enabled = login_enabled == "on" + CONFIG.login_enabled = login_enabled == "on" registration_enabled = env.params.body["registration_enabled"]?.try &.as(String) registration_enabled ||= "off" - config.registration_enabled = registration_enabled == "on" + CONFIG.registration_enabled = registration_enabled == "on" statistics_enabled = env.params.body["statistics_enabled"]?.try &.as(String) statistics_enabled ||= "off" - config.statistics_enabled = statistics_enabled == "on" + CONFIG.statistics_enabled = statistics_enabled == "on" - CONFIG.default_user_preferences = config.default_user_preferences - File.write("config/config.yml", config.to_yaml) + File.write("config/config.yml", CONFIG.to_yaml) end else - if Kemal.config.ssl || config.https_only + if Kemal.config.ssl || CONFIG.https_only secure = true else secure = false end - if config.domain - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{config.domain}", value: preferences, expires: Time.utc + 2.years, + if CONFIG.domain + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, secure: secure, http_only: true) else - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: preferences, expires: Time.utc + 2.years, + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, secure: secure, http_only: true) end end @@ -200,7 +221,7 @@ class Invidious::Routes::UserPreferences < Invidious::Routes::BaseRoute env.redirect referer end - def toggle_theme(env) + def self.toggle_theme(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? referer = get_referer(env, unroll: false) @@ -234,17 +255,17 @@ class Invidious::Routes::UserPreferences < Invidious::Routes::BaseRoute preferences = preferences.to_json - if Kemal.config.ssl || config.https_only + if Kemal.config.ssl || CONFIG.https_only secure = true else secure = false end - if config.domain - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{config.domain}", value: preferences, expires: Time.utc + 2.years, + if CONFIG.domain + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, secure: secure, http_only: true) else - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: preferences, expires: Time.utc + 2.years, + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, secure: secure, http_only: true) end end diff --git a/src/invidious/routes/privacy.cr b/src/invidious/routes/privacy.cr deleted file mode 100644 index 4565c94c..00000000 --- a/src/invidious/routes/privacy.cr +++ /dev/null @@ -1,6 +0,0 @@ -class Invidious::Routes::Privacy < Invidious::Routes::BaseRoute - def handle(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - templated "privacy" - end -end diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index 48446161..3f1e219f 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -1,5 +1,7 @@ -class Invidious::Routes::Search < Invidious::Routes::BaseRoute - def opensearch(env) +{% skip_file if flag?(:api_only) %} + +module Invidious::Routes::Search + def self.opensearch(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? env.response.content_type = "application/opensearchdescription+xml" @@ -15,45 +17,56 @@ class Invidious::Routes::Search < Invidious::Routes::BaseRoute end end - def results(env) + def self.results(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? query = env.params.query["search_query"]? query ||= env.params.query["q"]? - query ||= "" - page = env.params.query["page"]?.try &.to_i? - page ||= 1 + page = env.params.query["page"]? - if query - env.redirect "/search?q=#{URI.encode_www_form(query)}&page=#{page}" + if query && !query.empty? + if page && !page.empty? + env.redirect "/search?q=" + URI.encode_www_form(query) + "&page=" + page + else + env.redirect "/search?q=" + URI.encode_www_form(query) + end else - env.redirect "/" + env.redirect "/search" end end - def search(env) + def self.search(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? region = env.params.query["region"]? query = env.params.query["search_query"]? query ||= env.params.query["q"]? - query ||= "" - return env.redirect "/" if query.empty? + if !query || query.empty? + # Display the full page search box implemented in #1977 + env.set "search", "" + templated "search_homepage", navbar_search: false + else + page = env.params.query["page"]?.try &.to_i? + page ||= 1 - page = env.params.query["page"]?.try &.to_i? - page ||= 1 + user = env.get? "user" - user = env.get? "user" + begin + search_query, count, videos, operators = process_search_query(query, page, user, region: region) + rescue ex + return error_template(500, ex) + end - begin - search_query, count, videos = process_search_query(query, page, user, region: nil) - rescue ex - return error_template(500, ex) - end + operator_hash = {} of String => String + operators.each do |operator| + key, value = operator.downcase.split(":") + operator_hash[key] = value + end - env.set "search", query - templated "search" + env.set "search", query + templated "search" + end end end diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr new file mode 100644 index 00000000..acbf62b4 --- /dev/null +++ b/src/invidious/routes/video_playback.cr @@ -0,0 +1,280 @@ +module Invidious::Routes::VideoPlayback + # /videoplayback + def self.get_video_playback(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + query_params = env.params.query + + fvip = query_params["fvip"]? || "3" + mns = query_params["mn"]?.try &.split(",") + mns ||= [] of String + + if query_params["region"]? + region = query_params["region"] + query_params.delete("region") + end + + if query_params["host"]? && !query_params["host"].empty? + host = "https://#{query_params["host"]}" + query_params.delete("host") + else + host = "https://r#{fvip}---#{mns.pop}.googlevideo.com" + end + + url = "/videoplayback?#{query_params.to_s}" + + headers = HTTP::Headers.new + REQUEST_HEADERS_WHITELIST.each do |header| + if env.request.headers[header]? + headers[header] = env.request.headers[header] + end + end + + client = make_client(URI.parse(host), region) + response = HTTP::Client::Response.new(500) + error = "" + 5.times do + begin + response = client.head(url, headers) + + if response.headers["Location"]? + location = URI.parse(response.headers["Location"]) + env.response.headers["Access-Control-Allow-Origin"] = "*" + + new_host = "#{location.scheme}://#{location.host}" + if new_host != host + host = new_host + client.close + client = make_client(URI.parse(new_host), region) + end + + url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" + else + break + end + rescue Socket::Addrinfo::Error + if !mns.empty? + mn = mns.pop + end + fvip = "3" + + host = "https://r#{fvip}---#{mn}.googlevideo.com" + client = make_client(URI.parse(host), region) + rescue ex + error = ex.message + end + end + + if response.status_code >= 400 + env.response.content_type = "text/plain" + haltf env, response.status_code + end + + if url.includes? "&file=seg.ts" + if CONFIG.disabled?("livestreams") + return error_template(403, "Administrator has disabled this endpoint.") + end + + begin + client.get(url, headers) do |response| + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value + end + end + + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if location = response.headers["Location"]? + location = URI.parse(location) + location = "#{location.request_target}&host=#{location.host}" + + if region + location += "®ion=#{region}" + end + + return env.redirect location + end + + IO.copy(response.body_io, env.response) + end + rescue ex + end + else + if query_params["title"]? && CONFIG.disabled?("downloads") || + CONFIG.disabled?("dash") + return error_template(403, "Administrator has disabled this endpoint.") + end + + content_length = nil + first_chunk = true + range_start, range_end = parse_range(env.request.headers["Range"]?) + chunk_start = range_start + chunk_end = range_end + + if !chunk_end || chunk_end - chunk_start > HTTP_CHUNK_SIZE + chunk_end = chunk_start + HTTP_CHUNK_SIZE - 1 + end + + # TODO: Record bytes written so we can restart after a chunk fails + while true + if !range_end && content_length + range_end = content_length + end + + if range_end && chunk_start > range_end + break + end + + if range_end && chunk_end > range_end + chunk_end = range_end + end + + headers["Range"] = "bytes=#{chunk_start}-#{chunk_end}" + + begin + client.get(url, headers) do |response| + if first_chunk + if !env.request.headers["Range"]? && response.status_code == 206 + env.response.status_code = 200 + else + env.response.status_code = response.status_code + end + + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) && key.downcase != "content-range" + env.response.headers[key] = value + end + end + + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if location = response.headers["Location"]? + location = URI.parse(location) + location = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" + + env.redirect location + break + end + + if title = query_params["title"]? + # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ + env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}" + end + + if !response.headers.includes_word?("Transfer-Encoding", "chunked") + content_length = response.headers["Content-Range"].split("/")[-1].to_i64 + if env.request.headers["Range"]? + env.response.headers["Content-Range"] = "bytes #{range_start}-#{range_end || (content_length - 1)}/#{content_length}" + env.response.content_length = ((range_end.try &.+ 1) || content_length) - range_start + else + env.response.content_length = content_length + end + end + end + + proxy_file(response, env) + end + rescue ex + if ex.message != "Error reading socket: Connection reset by peer" + break + else + client.close + client = make_client(URI.parse(host), region) + end + end + + chunk_start = chunk_end + 1 + chunk_end += HTTP_CHUNK_SIZE + first_chunk = false + end + end + client.close + end + + # /videoplayback/* + def self.get_video_playback_greedy(env) + path = env.request.path + + path = path.lchop("/videoplayback/") + path = path.rchop("/") + + path = path.gsub(/mime\/\w+\/\w+/) do |mimetype| + mimetype = mimetype.split("/") + mimetype[0] + "/" + mimetype[1] + "%2F" + mimetype[2] + end + + path = path.split("/") + + raw_params = {} of String => Array(String) + path.each_slice(2) do |pair| + key, value = pair + value = URI.decode_www_form(value) + + if raw_params[key]? + raw_params[key] << value + else + raw_params[key] = [value] + end + end + + query_params = HTTP::Params.new(raw_params) + + env.response.headers["Access-Control-Allow-Origin"] = "*" + return env.redirect "/videoplayback?#{query_params}" + end + + # /videoplayback/* && /videoplayback/* + def self.options_video_playback(env) + env.response.headers.delete("Content-Type") + env.response.headers["Access-Control-Allow-Origin"] = "*" + env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" + env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range" + end + + # /latest_version + # + # YouTube /videoplayback links expire after 6 hours, + # so we have a mechanism here to redirect to the latest version + def self.latest_version(env) + if env.params.query["download_widget"]? + download_widget = JSON.parse(env.params.query["download_widget"]) + + id = download_widget["id"].as_s + title = download_widget["title"].as_s + + if label = download_widget["label"]? + return env.redirect "/api/v1/captions/#{id}?label=#{label}&title=#{title}" + else + itag = download_widget["itag"].as_s.to_i + local = "true" + end + end + + id ||= env.params.query["id"]? + itag ||= env.params.query["itag"]?.try &.to_i + + region = env.params.query["region"]? + + local ||= env.params.query["local"]? + local ||= "false" + local = local == "true" + + if !id || !itag + haltf env, status_code: 400, response: "TESTING" + end + + video = get_video(id, PG_DB, region: region) + + fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag } + url = fmt.try &.["url"]?.try &.as_s + + if !url + haltf env, status_code: 404 + end + + url = URI.parse(url).request_target.not_nil! if local + url = "#{url}&title=#{title}" if title + + return env.redirect url + end +end diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 65604a88..2db133ee 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -1,5 +1,7 @@ -class Invidious::Routes::Watch < Invidious::Routes::BaseRoute - def handle(env) +{% skip_file if flag?(:api_only) %} + +module Invidious::Routes::Watch + def self.handle(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? region = env.params.query["region"]? @@ -92,7 +94,7 @@ class Invidious::Routes::Watch < Invidious::Routes::BaseRoute if source == "youtube" begin - comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] + comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] rescue ex if preferences.comments[1] == "reddit" comments, reddit_thread = fetch_reddit_comments(id) @@ -111,12 +113,12 @@ class Invidious::Routes::Watch < Invidious::Routes::BaseRoute comment_html = replace_links(comment_html) rescue ex if preferences.comments[1] == "youtube" - comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] + comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] end end end else - comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] + comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] end comment_html ||= "" @@ -126,8 +128,8 @@ class Invidious::Routes::Watch < Invidious::Routes::BaseRoute adaptive_fmts = video.adaptive_fmts if params.local - fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) } - adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) } + fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) } + adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) } end video_streams = video.video_streams @@ -150,11 +152,11 @@ class Invidious::Routes::Watch < Invidious::Routes::BaseRoute captions = video.captions preferred_captions = captions.select { |caption| - params.preferred_captions.includes?(caption.name.simpleText) || + params.preferred_captions.includes?(caption.name) || params.preferred_captions.includes?(caption.languageCode.split("-")[0]) } preferred_captions.sort_by! { |caption| - (params.preferred_captions.index(caption.name.simpleText) || + (params.preferred_captions.index(caption.name) || params.preferred_captions.index(caption.languageCode.split("-")[0])).not_nil! } captions = captions - preferred_captions @@ -167,9 +169,11 @@ class Invidious::Routes::Watch < Invidious::Routes::BaseRoute if params.listen url = audio_streams[0]["url"].as_s - audio_streams.each do |fmt| - if fmt["bitrate"].as_i == params.quality.rchop("k").to_i - url = fmt["url"].as_s + if params.quality.ends_with? "k" + audio_streams.each do |fmt| + if fmt["bitrate"].as_i == params.quality.rchop("k").to_i + url = fmt["url"].as_s + end end end else @@ -187,4 +191,13 @@ class Invidious::Routes::Watch < Invidious::Routes::BaseRoute templated "watch" end + + def self.redirect(env) + url = "/watch?v=#{env.params.url["id"]}" + if env.params.query.size > 0 + url += "&#{env.params.query}" + end + + return env.redirect url + end end diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index 593c7372..7551f22d 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -1,15 +1,100 @@ module Invidious::Routing - macro get(path, controller, method = :handle) - get {{ path }} do |env| - controller_instance = {{ controller }}.new(config) - controller_instance.{{ method.id }}(env) - end - end + {% for http_method in {"get", "post", "delete", "options", "patch", "put", "head"} %} - macro post(path, controller, method = :handle) - post {{ path }} do |env| - controller_instance = {{ controller }}.new(config) - controller_instance.{{ method.id }}(env) + macro {{http_method.id}}(path, controller, method = :handle) + {{http_method.id}} \{{ path }} do |env| + \{{ controller }}.\{{ method.id }}(env) + end end - end + + {% end %} +end + +macro define_v1_api_routes + {{namespace = Invidious::Routes::API::V1}} + # Videos + Invidious::Routing.get "/api/v1/videos/:id", {{namespace}}::Videos, :videos + Invidious::Routing.get "/api/v1/storyboards/:id", {{namespace}}::Videos, :storyboards + Invidious::Routing.get "/api/v1/captions/:id", {{namespace}}::Videos, :captions + Invidious::Routing.get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations + Invidious::Routing.get "/api/v1/comments/:id", {{namespace}}::Videos, :comments + + # Feeds + Invidious::Routing.get "/api/v1/trending", {{namespace}}::Feeds, :trending + Invidious::Routing.get "/api/v1/popular", {{namespace}}::Feeds, :popular + + # Channels + Invidious::Routing.get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home + {% for route in {"videos", "latest", "playlists", "community", "search"} %} + Invidious::Routing.get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}} + Invidious::Routing.get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}} + {% end %} + + # 301 redirects to new /api/v1/channels/community/:ucid and /:ucid/community + Invidious::Routing.get "/api/v1/channels/comments/:ucid", {{namespace}}::Channels, :channel_comments_redirect + Invidious::Routing.get "/api/v1/channels/:ucid/comments", {{namespace}}::Channels, :channel_comments_redirect + + + # Search + Invidious::Routing.get "/api/v1/search", {{namespace}}::Search, :search + Invidious::Routing.get "/api/v1/search/suggestions", {{namespace}}::Search, :search_suggestions + + # Authenticated + + # The notification APIs cannot be extracted yet! They require the *local* notifications constant defined in invidious.cr + # + # Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + # Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + + Invidious::Routing.get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences + Invidious::Routing.post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences + + Invidious::Routing.get "/api/v1/auth/feed", {{namespace}}::Authenticated, :feed + + Invidious::Routing.get "/api/v1/auth/subscriptions", {{namespace}}::Authenticated, :get_subscriptions + Invidious::Routing.post "/api/v1/auth/subscriptions/:ucid", {{namespace}}::Authenticated, :subscribe_channel + Invidious::Routing.delete "/api/v1/auth/subscriptions/:ucid", {{namespace}}::Authenticated, :unsubscribe_channel + + + Invidious::Routing.get "/api/v1/auth/playlists", {{namespace}}::Authenticated, :list_playlists + Invidious::Routing.post "/api/v1/auth/playlists", {{namespace}}::Authenticated, :create_playlist + Invidious::Routing.patch "/api/v1/auth/playlists/:plid",{{namespace}}:: Authenticated, :update_playlist_attribute + Invidious::Routing.delete "/api/v1/auth/playlists/:plid", {{namespace}}::Authenticated, :delete_playlist + + + Invidious::Routing.post "/api/v1/auth/playlists/:plid/videos", {{namespace}}::Authenticated, :insert_video_into_playlist + Invidious::Routing.delete "/api/v1/auth/playlists/:plid/videos/:index", {{namespace}}::Authenticated, :delete_video_in_playlist + + Invidious::Routing.get "/api/v1/auth/tokens", {{namespace}}::Authenticated, :get_tokens + Invidious::Routing.post "/api/v1/auth/tokens/register", {{namespace}}::Authenticated, :register_token + Invidious::Routing.post "/api/v1/auth/tokens/unregister", {{namespace}}::Authenticated, :unregister_token + + # Misc + Invidious::Routing.get "/api/v1/stats", {{namespace}}::Misc, :stats + Invidious::Routing.get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist + Invidious::Routing.get "/api/v1/auth/playlists/:plid", {{namespace}}::Misc, :get_playlist + Invidious::Routing.get "/api/v1/mixes/:rdid", {{namespace}}::Misc, :mixes +end + +macro define_api_manifest_routes + Invidious::Routing.get "/api/manifest/dash/id/:id", Invidious::Routes::API::Manifest, :get_dash_video_id + + Invidious::Routing.get "/api/manifest/dash/id/videoplayback", Invidious::Routes::API::Manifest, :get_dash_video_playback + Invidious::Routing.get "/api/manifest/dash/id/videoplayback/*", Invidious::Routes::API::Manifest, :get_dash_video_playback_greedy + + Invidious::Routing.options "/api/manifest/dash/id/videoplayback", Invidious::Routes::API::Manifest, :options_dash_video_playback + Invidious::Routing.options "/api/manifest/dash/id/videoplayback/*", Invidious::Routes::API::Manifest, :options_dash_video_playback + + Invidious::Routing.get "/api/manifest/hls_playlist/*", Invidious::Routes::API::Manifest, :get_hls_playlist + Invidious::Routing.get "/api/manifest/hls_variant/*", Invidious::Routes::API::Manifest, :get_hls_variant +end + +macro define_video_playback_routes + Invidious::Routing.get "/videoplayback", Invidious::Routes::VideoPlayback, :get_video_playback + Invidious::Routing.get "/videoplayback/*", Invidious::Routes::VideoPlayback, :get_video_playback_greedy + + Invidious::Routing.options "/videoplayback", Invidious::Routes::VideoPlayback, :options_video_playback + Invidious::Routing.options "/videoplayback/*", Invidious::Routes::VideoPlayback, :options_video_playback + + Invidious::Routing.get "/latest_version", Invidious::Routes::VideoPlayback, :latest_version end diff --git a/src/invidious/search.cr b/src/invidious/search.cr index 85fd024a..d95d802e 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -1,273 +1,49 @@ -struct SearchVideo - include DB::Serializable - - property title : String - property id : String - property author : String - property ucid : String - property published : Time - property views : Int64 - property description_html : String - property length_seconds : Int32 - property live_now : Bool - property paid : Bool - property premium : Bool - property premiere_timestamp : Time? - - def to_xml(auto_generated, query_params, xml : XML::Builder) - query_params["v"] = self.id - - xml.element("entry") do - xml.element("id") { xml.text "yt:video:#{self.id}" } - xml.element("yt:videoId") { xml.text self.id } - xml.element("yt:channelId") { xml.text self.ucid } - xml.element("title") { xml.text self.title } - xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}") - - xml.element("author") do - if auto_generated - xml.element("name") { xml.text self.author } - xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } - else - xml.element("name") { xml.text author } - xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" } - end - end - - xml.element("content", type: "xhtml") do - xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do - xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do - xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg") - end - - xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text html_to_content(self.description_html) } - end - end - - xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") } - - xml.element("media:group") do - xml.element("media:title") { xml.text self.title } - xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg", - width: "320", height: "180") - xml.element("media:description") { xml.text html_to_content(self.description_html) } - end - - xml.element("media:community") do - xml.element("media:statistics", views: self.views) - end - end - end - - def to_xml(auto_generated, query_params, xml : XML::Builder | Nil = nil) - if xml - to_xml(HOST_URL, auto_generated, query_params, xml) - else - XML.build do |json| - to_xml(HOST_URL, auto_generated, query_params, xml) - end - end - end - - def to_json(locale, json : JSON::Builder) - json.object do - json.field "type", "video" - json.field "title", self.title - json.field "videoId", self.id - - json.field "author", self.author - json.field "authorId", self.ucid - json.field "authorUrl", "/channel/#{self.ucid}" - - json.field "videoThumbnails" do - generate_thumbnails(json, self.id) - end - - json.field "description", html_to_content(self.description_html) - json.field "descriptionHtml", self.description_html - - json.field "viewCount", self.views - json.field "published", self.published.to_unix - json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) - json.field "lengthSeconds", self.length_seconds - json.field "liveNow", self.live_now - json.field "paid", self.paid - json.field "premium", self.premium - json.field "isUpcoming", self.is_upcoming - - if self.premiere_timestamp - json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix - end - end - end - - def to_json(locale, json : JSON::Builder | Nil = nil) - if json - to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end - end - end - - def is_upcoming - premiere_timestamp ? true : false - end -end - -struct SearchPlaylistVideo - include DB::Serializable - - property title : String - property id : String - property length_seconds : Int32 -end - -struct SearchPlaylist - include DB::Serializable - - property title : String - property id : String - property author : String - property ucid : String - property video_count : Int32 - property videos : Array(SearchPlaylistVideo) - property thumbnail : String? - - def to_json(locale, json : JSON::Builder) - json.object do - json.field "type", "playlist" - json.field "title", self.title - json.field "playlistId", self.id - json.field "playlistThumbnail", self.thumbnail - - json.field "author", self.author - json.field "authorId", self.ucid - json.field "authorUrl", "/channel/#{self.ucid}" - - json.field "videoCount", self.video_count - json.field "videos" do - json.array do - self.videos.each do |video| - json.object do - json.field "title", video.title - json.field "videoId", video.id - json.field "lengthSeconds", video.length_seconds - - json.field "videoThumbnails" do - generate_thumbnails(json, video.id) - end - end - end - end - end - end - end - - def to_json(locale, json : JSON::Builder | Nil = nil) - if json - to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end - end - end -end - -struct SearchChannel - include DB::Serializable - - property author : String - property ucid : String - property author_thumbnail : String - property subscriber_count : Int32 - property video_count : Int32 - property description_html : String - property auto_generated : Bool - - def to_json(locale, json : JSON::Builder) - json.object do - json.field "type", "channel" - json.field "author", self.author - json.field "authorId", self.ucid - json.field "authorUrl", "/channel/#{self.ucid}" - - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - - qualities.each do |quality| - json.object do - json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", quality - end - end - end - end - - json.field "autoGenerated", self.auto_generated - json.field "subCount", self.subscriber_count - json.field "videoCount", self.video_count - - json.field "description", html_to_content(self.description_html) - json.field "descriptionHtml", self.description_html - end - end - - def to_json(locale, json : JSON::Builder | Nil = nil) - if json - to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end - end +def channel_search(query, page, channel) + response = YT_POOL.client &.get("/channel/#{channel}") + + if response.status_code == 404 + response = YT_POOL.client &.get("/user/#{channel}") + response = YT_POOL.client &.get("/c/#{channel}") if response.status_code == 404 + initial_data = extract_initial_data(response.body) + ucid = initial_data["header"]["c4TabbedHeaderRenderer"]?.try &.["channelId"].as_s? + raise InfoException.new("Impossible to extract channel ID from page") if !ucid + else + ucid = channel end -end -alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist + continuation = produce_channel_search_continuation(ucid, query, page) + response_json = YoutubeAPI.browse(continuation) -def channel_search(query, page, channel) - response = YT_POOL.client &.get("/channel/#{channel}?hl=en&gl=US") - response = YT_POOL.client &.get("/user/#{channel}?hl=en&gl=US") if response.headers["location"]? - response = YT_POOL.client &.get("/c/#{channel}?hl=en&gl=US") if response.headers["location"]? + continuationItems = response_json["onResponseReceivedActions"]? + .try &.[0]["appendContinuationItemsAction"]["continuationItems"] - ucid = response.body.match(/\\"channelId\\":\\"(?<ucid>[^\\]+)\\"/).try &.["ucid"]? + return 0, [] of SearchItem if !continuationItems - return 0, [] of SearchItem if !ucid - - url = produce_channel_search_url(ucid, query, page) - response = YT_POOL.client &.get(url) - initial_data = JSON.parse(response.body).as_a.find &.["response"]? - return 0, [] of SearchItem if !initial_data - author = initial_data["response"]?.try &.["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s - items = extract_items(initial_data.as_h, author, ucid) + items = [] of SearchItem + continuationItems.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each { |item| + extract_item(item["itemSectionRenderer"]["contents"].as_a[0]) + .try { |t| items << t } + } return items.size, items end -def search(query, page = 1, search_params = produce_search_params(content_type: "all"), region = nil) +def search(query, search_params = produce_search_params(content_type: "all"), region = nil) return 0, [] of SearchItem if query.empty? - body = YT_POOL.client(region, &.get("/results?q=#{URI.encode_www_form(query)}&page=#{page}&sp=#{search_params}&hl=en").body) - return 0, [] of SearchItem if body.empty? - - initial_data = extract_initial_data(body) + client_config = YoutubeAPI::ClientConfig.new(region: region) + initial_data = YoutubeAPI.search(query, search_params, client_config: client_config) items = extract_items(initial_data) - # initial_data["estimatedResults"]?.try &.as_s.to_i64 - return items.size, items end -def produce_search_params(sort : String = "relevance", date : String = "", content_type : String = "", +def produce_search_params(page = 1, sort : String = "relevance", date : String = "", content_type : String = "", duration : String = "", features : Array(String) = [] of String) object = { "1:varint" => 0_i64, "2:embedded" => {} of String => Int64, + "9:varint" => ((page - 1) * 20).to_i64, } case sort @@ -360,17 +136,28 @@ def produce_search_params(sort : String = "relevance", date : String = "", conte return params end -def produce_channel_search_url(ucid, query, page) +def produce_channel_search_continuation(ucid, query, page) + if page <= 1 + idx = 0_i64 + else + idx = 30_i64 * (page - 1) + end + object = { "80226972:embedded" => { "2:string" => ucid, "3:base64" => { "2:string" => "search", + "6:varint" => 1_i64, "7:varint" => 1_i64, - "15:string" => "#{page}", + "12:varint" => 1_i64, + "15:base64" => { + "3:varint" => idx, + }, "23:varint" => 0_i64, }, "11:string" => query, + "35:string" => "browse-feed#{ucid}search", }, } @@ -379,7 +166,7 @@ def produce_channel_search_url(ucid, query, page) .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } - return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" + return continuation end def process_search_query(query, page, user, region) @@ -439,11 +226,26 @@ def process_search_query(query, page, user, region) count = 0 end else - search_params = produce_search_params(sort: sort, date: date, content_type: content_type, + search_params = produce_search_params(page: page, sort: sort, date: date, content_type: content_type, duration: duration, features: features) - count, items = search(search_query, page, search_params, region).as(Tuple) + count, items = search(search_query, search_params, region).as(Tuple) + end + + # Light processing to flatten search results out of Categories. + # They should ideally be supported in the future. + items_without_category = [] of SearchItem | ChannelVideo + items.each do |i| + if i.is_a? Category + i.contents.each do |nest_i| + if !nest_i.is_a? Video + items_without_category << nest_i + end + end + else + items_without_category << i + end end - {search_query, count, items} + {search_query, items_without_category.size, items_without_category, operators} end diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index 8d078387..25bab4d2 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr @@ -2,33 +2,20 @@ def fetch_trending(trending_type, region, locale) region ||= "US" region = region.upcase - trending = "" plid = nil - if trending_type && trending_type != "Default" - trending_type = trending_type.downcase.capitalize - - response = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en").body - - initial_data = extract_initial_data(response) - - tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"][0]["tabRenderer"]["content"]["sectionListRenderer"]["subMenu"]["channelListSubMenuRenderer"]["contents"].as_a - url = tabs.select { |tab| tab["channelListSubMenuAvatarRenderer"]["title"]["simpleText"] == trending_type }[0]? - - if url - url["channelListSubMenuAvatarRenderer"]["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"] - url = url["channelListSubMenuAvatarRenderer"]["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s - url = "#{url}&gl=#{region}&hl=en" - trending = YT_POOL.client &.get(url).body - plid = extract_plid(url) - else - trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en").body - end - else - trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en").body + if trending_type == "Music" + params = "4gINGgt5dG1hX2NoYXJ0cw%3D%3D" + elsif trending_type == "Gaming" + params = "4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D" + elsif trending_type == "Movies" + params = "4gIKGgh0cmFpbGVycw%3D%3D" + else # Default + params = "" end - initial_data = extract_initial_data(trending) + client_config = YoutubeAPI::ClientConfig.new(region: region) + initial_data = YoutubeAPI.browse("FEtrending", params: params, client_config: client_config) trending = extract_videos(initial_data) return {trending, plid} diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 153e3b6a..aff76b53 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -36,6 +36,7 @@ struct Preferences property annotations : Bool = CONFIG.default_user_preferences.annotations property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed property autoplay : Bool = CONFIG.default_user_preferences.autoplay + property automatic_instance_redirect : Bool = CONFIG.default_user_preferences.automatic_instance_redirect @[JSON::Field(converter: Preferences::StringToArray)] @[YAML::Field(converter: Preferences::StringToArray)] @@ -53,6 +54,8 @@ struct Preferences property latest_only : Bool = CONFIG.default_user_preferences.latest_only property listen : Bool = CONFIG.default_user_preferences.listen property local : Bool = CONFIG.default_user_preferences.local + property vr_mode : Bool = CONFIG.default_user_preferences.vr_mode + property show_nick : Bool = CONFIG.default_user_preferences.show_nick @[JSON::Field(converter: Preferences::ProcessString)] property locale : String = CONFIG.default_user_preferences.locale @@ -68,7 +71,7 @@ struct Preferences property quality : String = CONFIG.default_user_preferences.quality @[JSON::Field(converter: Preferences::ProcessString)] property quality_dash : String = CONFIG.default_user_preferences.quality_dash - property default_home : String = CONFIG.default_user_preferences.default_home + property default_home : String? = CONFIG.default_user_preferences.default_home property feed_menu : Array(String) = CONFIG.default_user_preferences.feed_menu property related_videos : Bool = CONFIG.default_user_preferences.related_videos @@ -78,6 +81,7 @@ struct Preferences property thin_mode : Bool = CONFIG.default_user_preferences.thin_mode property unseen_only : Bool = CONFIG.default_user_preferences.unseen_only property video_loop : Bool = CONFIG.default_user_preferences.video_loop + property extend_desc : Bool = CONFIG.default_user_preferences.extend_desc property volume : Int32 = CONFIG.default_user_preferences.volume module BoolToString @@ -173,6 +177,20 @@ struct Preferences end end + module URIConverter + def self.to_yaml(value : URI, yaml : YAML::Nodes::Builder) + yaml.scalar value.normalize! + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : URI + if node.is_a?(YAML::Nodes::Scalar) + URI.parse node.value + else + node.raise "Expected scalar, not #{node.class}" + end + end + end + module ProcessString def self.to_json(value : String, json : JSON::Builder) json.string value @@ -446,7 +464,7 @@ def subscribe_ajax(channel_id, action, env_headers) html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers) - cookies = HTTP::Cookies.from_headers(headers) + cookies = HTTP::Cookies.from_client_headers(headers) html.cookies.each do |cookie| if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name if cookies[cookie.name]? diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 74edc156..0e6bd77c 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -242,8 +242,10 @@ struct VideoPreferences property speed : Float32 | Float64 property video_end : Float64 | Int32 property video_loop : Bool + property extend_desc : Bool property video_start : Float64 | Int32 property volume : Int32 + property vr_mode : Bool end struct Video @@ -273,7 +275,7 @@ struct Video end end - def to_json(locale, json : JSON::Builder) + def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder) json.object do json.field "type", "video" @@ -423,9 +425,9 @@ struct Video json.array do self.captions.each do |caption| json.object do - json.field "label", caption.name.simpleText + json.field "label", caption.name json.field "languageCode", caption.languageCode - json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name.simpleText)}" + json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}" end end end @@ -517,24 +519,21 @@ struct Video end def published : Time - info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["publishDate"]?.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location.local) } || Time.local + info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["publishDate"]?.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc end def published=(other : Time) info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d")) end - def cookie - info["cookie"]?.try &.as_h.map { |k, v| "#{k}=#{v}" }.join("; ") || "" - end - def allow_ratings r = info["videoDetails"]["allowRatings"]?.try &.as_bool r.nil? ? false : r end def live_now - info["videoDetails"]["isLiveContent"]?.try &.as_bool || false + info["microformat"]?.try &.["playerMicroformatRenderer"]? + .try &.["liveBroadcastDetails"]?.try &.["isLiveNow"]?.try &.as_bool || false end def is_listed @@ -703,8 +702,12 @@ struct Video def captions : Array(Caption) return @captions.as(Array(Caption)) if @captions captions = info["captions"]?.try &.["playerCaptionsTracklistRenderer"]?.try &.["captionTracks"]?.try &.as_a.map do |caption| - caption = Caption.from_json(caption.to_json) - caption.name.simpleText = caption.name.simpleText.split(" - ")[0] + name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"] + languageCode = caption["languageCode"].to_s + baseUrl = caption["baseUrl"].to_s + + caption = Caption.new(name.to_s, languageCode, baseUrl) + caption.name = caption.name.split(" - ")[0] caption end captions ||= [] of Caption @@ -758,35 +761,41 @@ struct Video info["microformat"]?.try &.["playerMicroformatRenderer"]["isFamilySafe"]?.try &.as_bool || false end + def is_vr : Bool? + projection_type = info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s + return {"EQUIRECTANGULAR", "MESH"}.includes? projection_type + end + + def projection_type : String? + return info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s + end + def wilson_score : Float64 ci_lower_bound(likes, likes + dislikes).round(4) end def engagement : Float64 - ((likes + dislikes) / views).round(4) + (((likes + dislikes) / views) * 100).round(4) end def reason : String? info["reason"]?.try &.as_s end - - def session_token : String? - info["sessionToken"]?.try &.as_s? - end end -struct CaptionName - include JSON::Serializable +struct Caption + property name + property languageCode + property baseUrl - property simpleText : String -end + getter name : String + getter languageCode : String + getter baseUrl : String -struct Caption - include JSON::Serializable + setter name - property name : CaptionName - property baseUrl : String - property languageCode : String + def initialize(@name, @languageCode, @baseUrl) + end end class VideoRedirect < Exception @@ -815,44 +824,61 @@ def parse_related(r : JSON::Any) : JSON::Any? JSON::Any.new(rv) end -def extract_polymer_config(body) +def extract_video_info(video_id : String, proxy_region : String? = nil, context_screen : String? = nil) params = {} of String => JSON::Any - player_response = body.match(/(window\["ytInitialPlayerResponse"\]|var\sytInitialPlayerResponse)\s*=\s*(?<info>{.*?});/m) - .try { |r| JSON.parse(r["info"]).as_h } - - if body.includes?("To continue with your YouTube experience, please fill out the form below.") || - body.includes?("https://www.google.com/sorry/index") - params["reason"] = JSON::Any.new("Could not extract video info. Instance is likely blocked.") - elsif !player_response - params["reason"] = JSON::Any.new("Video unavailable.") - elsif player_response["playabilityStatus"]?.try &.["status"]?.try &.as_s != "OK" - reason = player_response["playabilityStatus"]["errorScreen"]?.try &.["playerErrorMessageRenderer"]?.try &.["subreason"]?.try { |s| s["simpleText"]?.try &.as_s || s["runs"].as_a.map { |r| r["text"] }.join("") } || - player_response["playabilityStatus"]["reason"].as_s + + client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region) + if context_screen == "embed" + client_config.client_type = YoutubeAPI::ClientType::WebScreenEmbed + end + + player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) + + if player_response["playabilityStatus"]?.try &.["status"]?.try &.as_s != "OK" + reason = player_response["playabilityStatus"]["errorScreen"]?.try &.["playerErrorMessageRenderer"]?.try &.["subreason"]?.try { |s| + s["simpleText"]?.try &.as_s || s["runs"].as_a.map { |r| r["text"] }.join("") + } || player_response["playabilityStatus"]["reason"].as_s params["reason"] = JSON::Any.new(reason) end - session_token_json_encoded = body.match(/"XSRF_TOKEN":"(?<session_token>[^"]+)"/).try &.["session_token"]? || "" - params["sessionToken"] = JSON.parse(%({"key": "#{session_token_json_encoded}"}))["key"] - params["shortDescription"] = JSON::Any.new(body.match(/"og:description" content="(?<description>[^"]+)"/).try &.["description"]?) + params["shortDescription"] = player_response.dig?("videoDetails", "shortDescription") || JSON::Any.new(nil) - return params if !player_response + # Don't fetch the next endpoint if the video is unavailable. + if !params["reason"]? + next_response = YoutubeAPI.next({"videoId": video_id, "params": ""}) + player_response = player_response.merge(next_response) + end + + # Fetch the video streams using an Android client in order to get the decrypted URLs and + # maybe fix throttling issues (#2194).See for the explanation about the decrypted URLs: + # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 + if !params["reason"]? + if context_screen == "embed" + client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed + else + client_config.client_type = YoutubeAPI::ClientType::Android + end + stream_data = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) + params["streamingData"] = stream_data["streamingData"]? || JSON::Any.new("") + end {"captions", "microformat", "playabilityStatus", "storyboards", "videoDetails"}.each do |f| params[f] = player_response[f] if player_response[f]? end - yt_initial_data = extract_initial_data(body) + params["relatedVideos"] = ( + player_response + .dig?("playerOverlays", "playerOverlayRenderer", "endScreen", "watchNextEndScreenRenderer", "results") + .try &.as_a.compact_map { |r| parse_related r } || \ + player_response + .dig?("webWatchNextResponseExtensionData", "relatedVideoArgs") + .try &.as_s.split(",").map { |r| + r = HTTP::Params.parse(r).to_h + JSON::Any.new(Hash.zip(r.keys, r.values.map { |v| JSON::Any.new(v) })) + } + ).try { |a| JSON::Any.new(a) } || JSON::Any.new([] of JSON::Any) - params["relatedVideos"] = yt_initial_data.try &.["playerOverlays"]?.try &.["playerOverlayRenderer"]? - .try &.["endScreen"]?.try &.["watchNextEndScreenRenderer"]?.try &.["results"]?.try &.as_a.compact_map { |r| - parse_related r - }.try { |a| JSON::Any.new(a) } || yt_initial_data.try &.["webWatchNextResponseExtensionData"]?.try &.["relatedVideoArgs"]? - .try &.as_s.split(",").map { |r| - r = HTTP::Params.parse(r).to_h - JSON::Any.new(Hash.zip(r.keys, r.values.map { |v| JSON::Any.new(v) })) - }.try { |a| JSON::Any.new(a) } || JSON::Any.new([] of JSON::Any) - - primary_results = yt_initial_data.try &.["contents"]?.try &.["twoColumnWatchNextResults"]?.try &.["results"]? + primary_results = player_response.try &.["contents"]?.try &.["twoColumnWatchNextResults"]?.try &.["results"]? .try &.["results"]?.try &.["contents"]? sentiment_bar = primary_results.try &.as_a.select { |object| object["videoPrimaryInfoRenderer"]? }[0]? .try &.["videoPrimaryInfoRenderer"]? @@ -912,20 +938,6 @@ def extract_polymer_config(body) params["subCountText"] = JSON::Any.new(author_info.try &.["subscriberCountText"]? .try { |t| t["simpleText"]? || t["runs"]?.try &.[0]?.try &.["text"]? }.try &.as_s.split(" ", 2)[0] || "-") - initial_data = body.match(/ytplayer\.config\s*=\s*(?<info>.*?);ytplayer\.web_player_context_config/) - .try { |r| JSON.parse(r["info"]) }.try &.["args"]["player_response"]? - .try &.as_s?.try &.try { |r| JSON.parse(r).as_h } - - if initial_data - {"playabilityStatus", "streamingData"}.each do |f| - params[f] = initial_data[f] if initial_data[f]? - end - else - {"playabilityStatus", "streamingData"}.each do |f| - params[f] = player_response[f] if player_response[f]? - end - end - params end @@ -956,52 +968,27 @@ def get_video(id, db, refresh = true, region = nil, force_refresh = false) end def fetch_video(id, region) - response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&has_verified=1&bpctr=9999999999")) + info = extract_video_info(video_id: id) - if md = response.headers["location"]?.try &.match(/v=(?<id>[a-zA-Z0-9_-]{11})/) - raise VideoRedirect.new(video_id: md["id"]) - end - - info = extract_polymer_config(response.body) - info["cookie"] = JSON::Any.new(response.cookies.to_h.transform_values { |v| JSON::Any.new(v.value) }) - allowed_regions = info["microformat"]?.try &.["playerMicroformatRenderer"]["availableCountries"]?.try &.as_a.map &.as_s || [] of String + allowed_regions = info + .dig?("microformat", "playerMicroformatRenderer", "availableCountries") + .try &.as_a.map &.as_s || [] of String # Check for region-blocks if info["reason"]?.try &.as_s.includes?("your country") bypass_regions = PROXY_LIST.keys & allowed_regions if !bypass_regions.empty? region = bypass_regions[rand(bypass_regions.size)] - response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&has_verified=1&bpctr=9999999999")) - - region_info = extract_polymer_config(response.body) + region_info = extract_video_info(video_id: id, proxy_region: region) region_info["region"] = JSON::Any.new(region) if region - region_info["cookie"] = JSON::Any.new(response.cookies.to_h.transform_values { |v| JSON::Any.new(v.value) }) info = region_info if !region_info["reason"]? end end - # Try to pull streams from embed URL + # Try to fetch video info using an embedded client if info["reason"]? - embed_page = YT_POOL.client &.get("/embed/#{id}").body - sts = embed_page.match(/"sts"\s*:\s*(?<sts>\d+)/).try &.["sts"]? || "" - embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?html5=1&video_id=#{id}&eurl=https://youtube.googleapis.com/v/#{id}&gl=US&hl=en&sts=#{sts}").body) - - if embed_info["player_response"]? - player_response = JSON.parse(embed_info["player_response"]) - {"captions", "microformat", "playabilityStatus", "streamingData", "videoDetails", "storyboards"}.each do |f| - info[f] = player_response[f] if player_response[f]? - end - end - - initial_data = JSON.parse(embed_info["watch_next_response"]) if embed_info["watch_next_response"]? - - info["relatedVideos"] = initial_data.try &.["playerOverlays"]?.try &.["playerOverlayRenderer"]? - .try &.["endScreen"]?.try &.["watchNextEndScreenRenderer"]?.try &.["results"]?.try &.as_a.compact_map { |r| - parse_related r - }.try { |a| JSON::Any.new(a) } || embed_info["rvs"]?.try &.split(",").map { |r| - r = HTTP::Params.parse(r).to_h - JSON::Any.new(Hash.zip(r.keys, r.values.map { |v| JSON::Any.new(v) })) - }.try { |a| JSON::Any.new(a) } || JSON::Any.new([] of JSON::Any) + embed_info = extract_video_info(video_id: id, context_screen: "embed") + info = embed_info if !embed_info["reason"]? end raise InfoException.new(info["reason"]?.try &.as_s || "") if !info["videoDetails"]? @@ -1049,7 +1036,9 @@ def process_video_params(query, preferences) related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe } speed = query["speed"]?.try &.rchop("x").to_f? video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe } + extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe } volume = query["volume"]?.try &.to_i? + vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe } if preferences # region ||= preferences.region @@ -1067,7 +1056,9 @@ def process_video_params(query, preferences) related_videos ||= preferences.related_videos.to_unsafe speed ||= preferences.speed video_loop ||= preferences.video_loop.to_unsafe + extend_desc ||= preferences.extend_desc.to_unsafe volume ||= preferences.volume + vr_mode ||= preferences.vr_mode.to_unsafe end annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe @@ -1084,7 +1075,9 @@ def process_video_params(query, preferences) related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe speed ||= CONFIG.default_user_preferences.speed video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe + extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe volume ||= CONFIG.default_user_preferences.volume + vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe annotations = annotations == 1 autoplay = autoplay == 1 @@ -1094,6 +1087,8 @@ def process_video_params(query, preferences) local = local == 1 related_videos = related_videos == 1 video_loop = video_loop == 1 + extend_desc = extend_desc == 1 + vr_mode = vr_mode == 1 if CONFIG.disabled?("dash") && quality == "dash" quality = "high" @@ -1140,8 +1135,10 @@ def process_video_params(query, preferences) speed: speed, video_end: video_end, video_loop: video_loop, + extend_desc: extend_desc, video_start: video_start, volume: volume, + vr_mode: vr_mode, }) return params @@ -1149,15 +1146,15 @@ end def build_thumbnails(id) return { - {name: "maxres", host: "#{HOST_URL}", url: "maxres", height: 720, width: 1280}, - {name: "maxresdefault", host: "https://i.ytimg.com", url: "maxresdefault", height: 720, width: 1280}, - {name: "sddefault", host: "https://i.ytimg.com", url: "sddefault", height: 480, width: 640}, - {name: "high", host: "https://i.ytimg.com", url: "hqdefault", height: 360, width: 480}, - {name: "medium", host: "https://i.ytimg.com", url: "mqdefault", height: 180, width: 320}, - {name: "default", host: "https://i.ytimg.com", url: "default", height: 90, width: 120}, - {name: "start", host: "https://i.ytimg.com", url: "1", height: 90, width: 120}, - {name: "middle", host: "https://i.ytimg.com", url: "2", height: 90, width: 120}, - {name: "end", host: "https://i.ytimg.com", url: "3", height: 90, width: 120}, + {host: HOST_URL, height: 720, width: 1280, name: "maxres", url: "maxres"}, + {host: HOST_URL, height: 720, width: 1280, name: "maxresdefault", url: "maxresdefault"}, + {host: HOST_URL, height: 480, width: 640, name: "sddefault", url: "sddefault"}, + {host: HOST_URL, height: 360, width: 480, name: "high", url: "hqdefault"}, + {host: HOST_URL, height: 180, width: 320, name: "medium", url: "mqdefault"}, + {host: HOST_URL, height: 90, width: 120, name: "default", url: "default"}, + {host: HOST_URL, height: 90, width: 120, name: "start", url: "1"}, + {host: HOST_URL, height: 90, width: 120, name: "middle", url: "2"}, + {host: HOST_URL, height: 90, width: 120, name: "end", url: "3"}, } end diff --git a/src/invidious/views/authorize_token.ecr b/src/invidious/views/authorize_token.ecr index 8ea99010..2dc948d9 100644 --- a/src/invidious/views/authorize_token.ecr +++ b/src/invidious/views/authorize_token.ecr @@ -9,13 +9,13 @@ <%= translate(locale, "Token") %> </h3> </div> - <div class="pure-u-1-3" style="text-align:center"> - <h3> + <div class="pure-u-1-3"> + <h3 style="text-align:center"> <a href="/token_manager"><%= translate(locale, "Token manager") %></a> </h3> </div> - <div class="pure-u-1-3" style="text-align:right"> - <h3> + <div class="pure-u-1-3"> + <h3 style="text-align:right"> <a href="/preferences"><%= translate(locale, "Preferences") %></a> </h3> </div> diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr index caa0ad0e..09cfb76e 100644 --- a/src/invidious/views/channel.ecr +++ b/src/invidious/views/channel.ecr @@ -1,11 +1,14 @@ +<% ucid = channel.ucid %> +<% author = HTML.escape(channel.author) %> + <% content_for "header" do %> -<title><%= channel.author %> - Invidious</title> -<link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/channel/<%= channel.ucid %>" /> +<title><%= author %> - Invidious</title> +<link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/channel/<%= ucid %>" /> <% end %> <% if channel.banner %> <div class="h-box"> - <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).full_path %>"> + <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>"> </div> <div class="h-box"> @@ -16,31 +19,34 @@ <div class="pure-g h-box"> <div class="pure-u-2-3"> <div class="channel-profile"> - <img src="/ggpht<%= URI.parse(channel.author_thumbnail).full_path %>"> - <span><%= channel.author %></span> + <img src="/ggpht<%= URI.parse(channel.author_thumbnail).request_target %>"> + <span><%= author %></span> </div> </div> - <div class="pure-u-1-3" style="text-align:right"> - <h3> - <a href="/feed/channel/<%= channel.ucid %>"><i class="icon ion-logo-rss"></i></a> + <div class="pure-u-1-3"> + <h3 style="text-align:right"> + <a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a> </h3> </div> </div> <div class="h-box"> - <p><span style="white-space:pre-wrap"><%= channel.description_html %></span></p> + <div id="descriptionWrapper"> + <p><span style="white-space:pre-wrap"><%= channel.description_html %></span></p> + </div> </div> <div class="h-box"> - <% ucid = channel.ucid %> - <% author = channel.author %> <% sub_count_text = number_to_short_text(channel.sub_count) %> <%= rendered "components/subscribe_widget" %> </div> <div class="pure-g h-box"> <div class="pure-u-1-3"> - <a href="https://www.youtube.com/channel/<%= channel.ucid %>"><%= translate(locale, "View channel on YouTube") %></a> + <a href="https://www.youtube.com/channel/<%= ucid %>"><%= translate(locale, "View channel on YouTube") %></a> + <div class="pure-u-1 pure-md-1-3"> + <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a> + </div> <% if !channel.auto_generated %> <div class="pure-u-1 pure-md-1-3"> <b><%= translate(locale, "Videos") %></b> @@ -50,12 +56,12 @@ <% if channel.auto_generated %> <b><%= translate(locale, "Playlists") %></b> <% else %> - <a href="/channel/<%= channel.ucid %>/playlists"><%= translate(locale, "Playlists") %></a> + <a href="/channel/<%= ucid %>/playlists"><%= translate(locale, "Playlists") %></a> <% end %> </div> <div class="pure-u-1 pure-md-1-3"> <% if channel.tabs.includes? "community" %> - <a href="/channel/<%= channel.ucid %>/community"><%= translate(locale, "Community") %></a> + <a href="/channel/<%= ucid %>/community"><%= translate(locale, "Community") %></a> <% end %> </div> </div> @@ -67,7 +73,7 @@ <% if sort_by == sort %> <b><%= translate(locale, sort) %></b> <% else %> - <a href="/channel/<%= channel.ucid %>?page=<%= page %>&sort_by=<%= sort %>"> + <a href="/channel/<%= ucid %>?page=<%= page %>&sort_by=<%= sort %>"> <%= translate(locale, sort) %> </a> <% end %> @@ -82,17 +88,15 @@ </div> <div class="pure-g"> - <% items.each_slice(4) do |slice| %> - <% slice.each do |item| %> - <%= rendered "components/item" %> - <% end %> - <% end %> +<% items.each do |item| %> + <%= rendered "components/item" %> +<% end %> </div> <div class="pure-g h-box"> <div class="pure-u-1 pure-u-lg-1-5"> <% if page > 1 %> - <a href="/channel/<%= channel.ucid %>?page=<%= page - 1 %><% if sort_by != "newest" %>&sort_by=<%= HTML.escape(sort_by) %><% end %>"> + <a href="/channel/<%= ucid %>?page=<%= page - 1 %><% if sort_by != "newest" %>&sort_by=<%= HTML.escape(sort_by) %><% end %>"> <%= translate(locale, "Previous page") %> </a> <% end %> @@ -100,7 +104,7 @@ <div class="pure-u-1 pure-u-lg-3-5"></div> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> <% if count == 60 %> - <a href="/channel/<%= channel.ucid %>?page=<%= page + 1 %><% if sort_by != "newest" %>&sort_by=<%= HTML.escape(sort_by) %><% end %>"> + <a href="/channel/<%= ucid %>?page=<%= page + 1 %><% if sort_by != "newest" %>&sort_by=<%= HTML.escape(sort_by) %><% end %>"> <%= translate(locale, "Next page") %> </a> <% end %> diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr index 69724390..15d8ed1e 100644 --- a/src/invidious/views/community.ecr +++ b/src/invidious/views/community.ecr @@ -1,10 +1,13 @@ +<% ucid = channel.ucid %> +<% author = HTML.escape(channel.author) %> + <% content_for "header" do %> -<title><%= channel.author %> - Invidious</title> +<title><%= author %> - Invidious</title> <% end %> <% if channel.banner %> <div class="h-box"> - <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).full_path %>"> + <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>"> </div> <div class="h-box"> @@ -15,24 +18,24 @@ <div class="pure-g h-box"> <div class="pure-u-2-3"> <div class="channel-profile"> - <img src="/ggpht<%= URI.parse(channel.author_thumbnail).full_path %>"> - <span><%= channel.author %></span> + <img src="/ggpht<%= URI.parse(channel.author_thumbnail).request_target %>"> + <span><%= author %></span> </div> </div> <div class="pure-u-1-3" style="text-align:right"> - <h3> + <h3 style="text-align:right"> <a href="/feed/channel/<%= channel.ucid %>"><i class="icon ion-logo-rss"></i></a> </h3> </div> </div> <div class="h-box"> - <p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %></span></p> + <div id="descriptionWrapper"> + <p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %></span></p> + </div> </div> <div class="h-box"> - <% ucid = channel.ucid %> - <% author = channel.author %> <% sub_count_text = number_to_short_text(channel.sub_count) %> <%= rendered "components/subscribe_widget" %> </div> @@ -40,6 +43,9 @@ <div class="pure-g h-box"> <div class="pure-u-1-3"> <a href="https://www.youtube.com/channel/<%= channel.ucid %>/community"><%= translate(locale, "View channel on YouTube") %></a> + <div class="pure-u-1 pure-md-1-3"> + <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a> + </div> <% if !channel.auto_generated %> <div class="pure-u-1 pure-md-1-3"> <a href="/channel/<%= channel.ucid %>"><%= translate(locale, "Videos") %></a> @@ -74,7 +80,7 @@ <script id="community_data" type="application/json"> <%= { - "ucid" => channel.ucid, + "ucid" => ucid, "youtube_comments_text" => HTML.escape(translate(locale, "View YouTube comments")), "comments_text" => HTML.escape(translate(locale, "View `x` comments", "{commentCount}")), "hide_replies_text" => HTML.escape(translate(locale, "Hide replies")), diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index e4a60697..84da1091 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -2,20 +2,20 @@ <div class="h-box"> <% case item when %> <% when SearchChannel %> - <a style="width:100%" href="/channel/<%= item.ucid %>"> + <a href="/channel/<%= item.ucid %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <center> - <img style="width:56.25%" src="/ggpht<%= URI.parse(item.author_thumbnail).full_path %>"/> + <img style="width:56.25%" src="/ggpht<%= URI.parse(item.author_thumbnail).request_target.gsub(/=s\d+/, "=s176") %>"/> </center> <% end %> - <p><%= item.author %></p> + <p dir="auto"><%= HTML.escape(item.author) %></p> </a> <p><%= translate(locale, "`x` subscribers", number_with_separator(item.subscriber_count)) %></p> <% if !item.auto_generated %><p><%= translate(locale, "`x` videos", number_with_separator(item.video_count)) %></p><% end %> <h5><%= item.description_html %></h5> <% when SearchPlaylist, InvidiousPlaylist %> <% if item.id.starts_with? "RD" %> - <% url = "/mix?list=#{item.id}&continuation=#{URI.parse(item.thumbnail || "/vi/-----------").full_path.split("/")[2]}" %> + <% url = "/mix?list=#{item.id}&continuation=#{URI.parse(item.thumbnail || "/vi/-----------").request_target.split("/")[2]}" %> <% else %> <% url = "/playlist?list=#{item.id}" %> <% end %> @@ -23,19 +23,17 @@ <a style="width:100%" href="<%= url %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> - <img class="thumbnail" src="<%= URI.parse(item.thumbnail || "/").full_path %>"/> + <img class="thumbnail" src="<%= URI.parse(item.thumbnail || "/").request_target %>"/> <p class="length"><%= number_with_separator(item.video_count) %> videos</p> </div> <% end %> - <p><%= item.title %></p> + <p dir="auto"><%= HTML.escape(item.title) %></p> + </a> + <a href="/channel/<%= item.ucid %>"> + <p dir="auto"><b><%= HTML.escape(item.author) %></b></p> </a> - <p> - <b> - <a style="width:100%" href="/channel/<%= item.ucid %>"><%= item.author %></a> - </b> - </p> <% when MixVideo %> - <a style="width:100%" href="/watch?v=<%= item.id %>&list=<%= item.rdid %>"> + <a href="/watch?v=<%= item.id %>&list=<%= item.rdid %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> <img class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/> @@ -44,13 +42,11 @@ <% end %> </div> <% end %> - <p><%= HTML.escape(item.title) %></p> + <p dir="auto"><%= HTML.escape(item.title) %></p> + </a> + <a href="/channel/<%= item.ucid %>"> + <p dir="auto"><b><%= HTML.escape(item.author) %></b></p> </a> - <p> - <b> - <a style="width:100%" href="/channel/<%= item.ucid %>"><%= item.author %></a> - </b> - </p> <% when PlaylistVideo %> <a style="width:100%" href="/watch?v=<%= item.id %>&list=<%= item.plid %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> @@ -76,30 +72,47 @@ <% end %> </div> <% end %> - <p><a href="/watch?v=<%= item.id %>"><%= HTML.escape(item.title) %></a></p> + <p dir="auto"><%= HTML.escape(item.title) %></p> </a> - <p> - <b> - <a style="width:100%" href="/channel/<%= item.ucid %>"><%= item.author %></a> - </b> - </p> - <h5 class="pure-g"> - <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp.try &.> Time.utc %> - <div class="pure-u-2-3"><%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %></div> - <% elsif Time.utc - item.published > 1.minute %> - <div class="pure-u-2-3"><%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %></div> - <% else %> - <div class="pure-u-2-3"></div> - <% end %> + <div class="video-card-row flexible"> + <div class="flex-left"><a href="/channel/<%= item.ucid %>"> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %></p> + </a></div> + <div class="flex-right"> + <div class="icon-buttons"> + <a title="<%=translate(locale, "Watch on YouTube")%>" href="https://www.youtube.com/watch?v=<%= item.id %>&list=<%= item.plid %>"> + <i class="icon ion-logo-youtube"></i> + </a> + <a title="<%=translate(locale, "Audio mode")%>" href="/watch?v=<%= item.id %>&list=<%= item.plid %>&listen=1"> + <i class="icon ion-md-headset"></i> + </a> + <a title="<%=translate(locale, "Switch Invidious Instance")%>" href="/redirect?referer=<%=HTML.escape("watch?v=#{item.id}&list=#{item.plid}")%>"> + <i class="icon ion-md-jet"></i> + </a> + </div> + </div> + </div> - <div class="pure-u-1-3" style="text-align:right"> - <%= item.responds_to?(:views) && item.views ? translate(locale, "`x` views", number_to_short_text(item.views || 0)) : "" %> + <div class="video-card-row flexible"> + <div class="flex-left"> + <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp.try &.> Time.utc %> + <p dir="auto"><%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %></p> + <% elsif Time.utc - item.published > 1.minute %> + <p dir="auto"><%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %></p> + <% end %> </div> - </h5> + + <% if item.responds_to?(:views) && item.views %> + <div class="flex-right"> + <p dir="auto"><%= translate(locale, "`x` views", number_to_short_text(item.views || 0)) %></p> + </div> + <% end %> + </div> + <% when Category %> <% else %> - <% if !env.get("preferences").as(Preferences).thin_mode %> - <a style="width:100%" href="/watch?v=<%= item.id %>"> + <a style="width:100%" href="/watch?v=<%= item.id %>"> + <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> <img class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/> <% if env.get? "show_watched" %> @@ -129,36 +142,49 @@ <% end %> <% if item.responds_to?(:live_now) && item.live_now %> - <p class="length"><i class="icon ion-ios-play-circle"></i> <%= translate(locale, "LIVE") %></p> + <p class="length" dir="auto"><i class="icon ion-ios-play-circle"></i> <%= translate(locale, "LIVE") %></p> <% elsif item.length_seconds != 0 %> <p class="length"><%= recode_length_seconds(item.length_seconds) %></p> <% end %> </div> - </a> - <% end %> - <p><a href="/watch?v=<%= item.id %>"><%= HTML.escape(item.title) %></a></p> - <p style="display: flex;"> - <b style="flex: 1;"> - <a style="width:100%" href="/channel/<%= item.ucid %>"><%= item.author %></a> - </b> - <a title="Audio mode" href="/watch?v=<%= item.id %>&listen=1"> - <i class="icon ion-md-headset"></i> - </a> - </p> - - <h5 class="pure-g"> - <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp.try &.> Time.utc %> - <div class="pure-u-2-3"><%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %></div> - <% elsif Time.utc - item.published > 1.minute %> - <div class="pure-u-2-3"><%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %></div> - <% else %> - <div class="pure-u-2-3"></div> <% end %> + <p dir="auto"><%= HTML.escape(item.title) %></p> + </a> - <div class="pure-u-1-3" style="text-align:right"> - <%= item.responds_to?(:views) && item.views ? translate(locale, "`x` views", number_to_short_text(item.views || 0)) : "" %> + <div class="video-card-row flexible"> + <div class="flex-left"><a href="/channel/<%= item.ucid %>"> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %></p> + </a></div> + <div class="flex-right"> + <div class="icon-buttons"> + <a title="<%=translate(locale, "Watch on YouTube")%>" href="https://www.youtube.com/watch?v=<%= item.id %>"> + <i class="icon ion-logo-youtube"></i> + </a> + <a title="<%=translate(locale, "Audio mode")%>" href="/watch?v=<%= item.id %>&listen=1"> + <i class="icon ion-md-headset"></i> + </a> + <a title="<%=translate(locale, "Switch Invidious Instance")%>" href="/redirect?referer=<%=HTML.escape("watch?v=#{item.id}")%>"> + <i class="icon ion-md-jet"></i> + </a> + </div> </div> - </h5> + </div> + + <div class="video-card-row flexible"> + <div class="flex-left"> + <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp.try &.> Time.utc %> + <p class="video-data" dir="auto"><%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %></p> + <% elsif Time.utc - item.published > 1.minute %> + <p class="video-data" dir="auto"><%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %></p> + <% end %> + </div> + + <% if item.responds_to?(:views) && item.views %> + <div class="flex-right"> + <p class="video-data" dir="auto"><%= translate(locale, "`x` views", number_to_short_text(item.views || 0)) %></p> + </div> + <% end %> + </div> <% end %> </div> </div> diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index 625c6fee..6418f66b 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -4,34 +4,41 @@ <% if params.video_loop %>loop<% end %> <% if params.controls %>controls<% end %>> <% if (hlsvp = video.hls_manifest_url) && !CONFIG.disabled?("livestreams") %> - <source src="<%= URI.parse(hlsvp).full_path %><% if params.local %>?local=true<% end %>" type="application/x-mpegURL" label="livestream"> + <source src="<%= URI.parse(hlsvp).request_target %><% if params.local %>?local=true<% end %>" type="application/x-mpegURL" label="livestream"> <% else %> <% if params.listen %> <% audio_streams.each_with_index do |fmt, i| %> <source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["mimeType"] %>' label="<%= fmt["bitrate"] %>k" selected="<%= i == 0 ? true : false %>"> <% end %> - <% else %> + <% else %> <% if params.quality == "dash" %> <source src="/api/manifest/dash/id/<%= video.id %>?local=true&unique_res=1" type='application/dash+xml' label="dash"> <% end %> - <% fmt_stream.each_with_index do |fmt, i| %> - <% if params.quality %> - <source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["mimeType"] %>' label="<%= fmt["quality"] %>" selected="<%= params.quality == fmt["quality"] %>"> - <% else %> - <source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["mimeType"] %>' label="<%= fmt["quality"] %>" selected="<%= i == 0 ? true : false %>"> - <% end %> + <% + fmt_stream.reject! { |f| f["itag"] == 17 } + fmt_stream.sort_by! {|f| params.quality == f["quality"] ? 0 : 1 } + fmt_stream.each_with_index do |fmt, i| + src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" + src_url += "&local=true" if params.local + + quality = fmt["quality"] + mimetype = HTML.escape(fmt["mimeType"].as_s) + + selected = params.quality ? (params.quality == quality) : (i == 0) + %> + <source src="<%= src_url %>" type="<%= mimetype %>" label="<%= quality %>" selected="<%= selected %>"> <% end %> <% end %> - <% preferred_captions.each_with_index do |caption, i| %> - <track kind="captions" src="/api/v1/captions/<%= video.id %>?label=<%= caption.name.simpleText %>&hl=<%= env.get("preferences").as(Preferences).locale %>" - label="<%= caption.name.simpleText %>" <% if i == 0 %>default<% end %>> + <% preferred_captions.each do |caption| %> + <track kind="captions" src="/api/v1/captions/<%= video.id %>?label=<%= caption.name %>&hl=<%= env.get("preferences").as(Preferences).locale %>" + label="<%= caption.name %>"> <% end %> <% captions.each do |caption| %> - <track kind="captions" src="/api/v1/captions/<%= video.id %>?label=<%= caption.name.simpleText %>&hl=<%= env.get("preferences").as(Preferences).locale %>" - label="<%= caption.name.simpleText %>"> + <track kind="captions" src="/api/v1/captions/<%= video.id %>?label=<%= caption.name %>&hl=<%= env.get("preferences").as(Preferences).locale %>" + label="<%= caption.name %>"> <% end %> <% end %> </video> @@ -42,7 +49,8 @@ "aspect_ratio" => aspect_ratio, "title" => video.title, "description" => HTML.escape(video.short_description), - "thumbnail" => thumbnail + "thumbnail" => thumbnail, + "preferred_caption_found" => !preferred_captions.empty? }.to_pretty_json %> </script> diff --git a/src/invidious/views/components/player_sources.ecr b/src/invidious/views/components/player_sources.ecr index 8162546e..0d97d35a 100644 --- a/src/invidious/views/components/player_sources.ecr +++ b/src/invidious/views/components/player_sources.ecr @@ -3,14 +3,18 @@ <link rel="stylesheet" href="/css/videojs.markers.min.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/videojs-share.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/videojs-vtt-thumbnails.css?v=<%= ASSET_COMMIT %>"> -<script src="/js/global.js?v=<%= ASSET_COMMIT %>"></script> +<link rel="stylesheet" href="/css/videojs-mobile-ui.css?v=<%= ASSET_COMMIT %>"> +<link rel="stylesheet" href="/css/player.css?v=<%= ASSET_COMMIT %>"> + <script src="/js/video.min.js?v=<%= ASSET_COMMIT %>"></script> +<script src="/js/videojs-mobile-ui.min.js?v=<%= ASSET_COMMIT %>"></script> <script src="/js/videojs-contrib-quality-levels.min.js?v=<%= ASSET_COMMIT %>"></script> <script src="/js/videojs-http-source-selector.min.js?v=<%= ASSET_COMMIT %>"></script> <script src="/js/videojs-markers.min.js?v=<%= ASSET_COMMIT %>"></script> <script src="/js/videojs-share.min.js?v=<%= ASSET_COMMIT %>"></script> <script src="/js/videojs-vtt-thumbnails.min.js?v=<%= ASSET_COMMIT %>"></script> + <% if params.annotations %> <link rel="stylesheet" href="/css/videojs-youtube-annotations.min.css?v=<%= ASSET_COMMIT %>"> <script src="/js/videojs-youtube-annotations.min.js?v=<%= ASSET_COMMIT %>"></script> @@ -20,3 +24,8 @@ <link rel="stylesheet" href="/css/quality-selector.css?v=<%= ASSET_COMMIT %>"> <script src="/js/silvermine-videojs-quality-selector.min.js?v=<%= ASSET_COMMIT %>"></script> <% end %> + +<% if !params.listen && params.vr_mode %> + <link rel="stylesheet" href="/css/videojs-vr.css?v=<%= ASSET_COMMIT %>"> + <script src="/js/videojs-vr.js?v=<%= ASSET_COMMIT %>"></script> +<% end %> diff --git a/src/invidious/views/edit_playlist.ecr b/src/invidious/views/edit_playlist.ecr index bd8d6207..5046abc1 100644 --- a/src/invidious/views/edit_playlist.ecr +++ b/src/invidious/views/edit_playlist.ecr @@ -1,14 +1,16 @@ +<% title = HTML.escape(playlist.title) %> + <% content_for "header" do %> -<title><%= playlist.title %> - Invidious</title> +<title><%= title %> - Invidious</title> <link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/playlist/<%= plid %>" /> <% end %> <form class="pure-form" action="/edit_playlist?list=<%= plid %>" method="post"> <div class="pure-g h-box"> <div class="pure-u-2-3"> - <h3><input class="pure-input-1" maxlength="150" name="title" type="text" value="<%= playlist.title %>"></h3> + <h3><input class="pure-input-1" maxlength="150" name="title" type="text" value="<%= title %>"></h3> <b> - <%= playlist.author %> | + <%= HTML.escape(playlist.author) %> | <%= translate(locale, "`x` videos", "#{playlist.video_count}") %> | <%= translate(locale, "Updated `x` ago", recode_date(playlist.updated, locale)) %> | <i class="icon <%= {"ion-md-globe", "ion-ios-unlock", "ion-ios-lock"}[playlist.privacy.value] %>"></i> @@ -55,11 +57,9 @@ </div> <div class="pure-g"> - <% videos.each_slice(4) do |slice| %> - <% slice.each do |item| %> - <%= rendered "components/item" %> - <% end %> - <% end %> +<% videos.each do |item| %> + <%= rendered "components/item" %> +<% end %> </div> <div class="pure-g h-box"> diff --git a/src/invidious/views/empty.ecr b/src/invidious/views/empty.ecr deleted file mode 100644 index c10c097e..00000000 --- a/src/invidious/views/empty.ecr +++ /dev/null @@ -1,8 +0,0 @@ -<% content_for "header" do %> -<meta name="description" content="<%= translate(locale, "An alternative front-end to YouTube") %>"> -<title> - Invidious -</title> -<% end %> - -<%= rendered "components/feed_menu" %> diff --git a/src/invidious/views/error.ecr b/src/invidious/views/error.ecr index d0752e5b..04eb74d5 100644 --- a/src/invidious/views/error.ecr +++ b/src/invidious/views/error.ecr @@ -4,4 +4,5 @@ <div class="h-box"> <%= error_message %> + <%= next_steps %> </div> diff --git a/src/invidious/views/feeds/history.ecr b/src/invidious/views/feeds/history.ecr new file mode 100644 index 00000000..40584979 --- /dev/null +++ b/src/invidious/views/feeds/history.ecr @@ -0,0 +1,71 @@ +<% content_for "header" do %> +<title><%= translate(locale, "History") %> - Invidious</title> +<% end %> + +<div class="pure-g h-box"> + <div class="pure-u-1-3"> + <h3><%= translate(locale, "`x` videos", %(<span id="count">#{user.watched.size}</span>)) %></h3> + </div> + <div class="pure-u-1-3"> + <h3 style="text-align:center"> + <a href="/feed/subscriptions"><%= translate(locale, "`x` subscriptions", %(<span id="count">#{user.subscriptions.size}</span>)) %></a> + </h3> + </div> + <div class="pure-u-1-3"> + <h3 style="text-align:right"> + <a href="/clear_watch_history"><%= translate(locale, "Clear watch history") %></a> + </h3> + </div> +</div> + +<script id="watched_data" type="application/json"> +<%= +{ + "csrf_token" => URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") +}.to_pretty_json +%> +</script> +<script src="/js/watched_widget.js"></script> + +<div class="pure-g"> + <% watched.each do |item| %> + <div class="pure-u-1 pure-u-md-1-4"> + <div class="h-box"> + <a style="width:100%" href="/watch?v=<%= item %>"> + <% if !env.get("preferences").as(Preferences).thin_mode %> + <div class="thumbnail"> + <img class="thumbnail" src="/vi/<%= item %>/mqdefault.jpg"/> + <form data-onsubmit="return_false" action="/watch_ajax?action_mark_unwatched=1&id=<%= item %>&referer=<%= env.get("current_page") %>" method="post"> + <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> + <p class="watched"> + <a data-onclick="mark_unwatched" data-id="<%= item %>" href="javascript:void(0)"> + <button type="submit" style="all:unset"><i class="icon ion-md-trash"></i></button> + </a> + </p> + </form> + </div> + <p></p> + <% end %> + </a> + </div> + </div> + <% end %> +</div> + +<div class="pure-g h-box"> + <div class="pure-u-1 pure-u-lg-1-5"> + <% if page > 1 %> + <a href="/feed/history?page=<%= page - 1 %><% if env.params.query["max_results"]? %>&max_results=<%= max_results %><% end %>"> + <%= translate(locale, "Previous page") %> + </a> + <% end %> + </div> + <div class="pure-u-1 pure-u-lg-3-5"></div> + <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> + <% if watched.size >= max_results %> + <a href="/feed/history?page=<%= page + 1 %><% if env.params.query["max_results"]? %>&max_results=<%= max_results %><% end %>"> + <%= translate(locale, "Next page") %> + </a> + <% end %> + </div> +</div> diff --git a/src/invidious/views/view_all_playlists.ecr b/src/invidious/views/feeds/playlists.ecr index 5ec6aa31..868cfeda 100644 --- a/src/invidious/views/view_all_playlists.ecr +++ b/src/invidious/views/feeds/playlists.ecr @@ -16,11 +16,9 @@ </div> <div class="pure-g"> - <% items_created.each_slice(4) do |slice| %> - <% slice.each do |item| %> - <%= rendered "components/item" %> - <% end %> - <% end %> +<% items_created.each do |item| %> + <%= rendered "components/item" %> +<% end %> </div> <div class="pure-g h-box"> @@ -30,9 +28,7 @@ </div> <div class="pure-g"> - <% items_saved.each_slice(4) do |slice| %> - <% slice.each do |item| %> - <%= rendered "components/item" %> - <% end %> - <% end %> +<% items_saved.each do |item| %> + <%= rendered "components/item" %> +<% end %> </div> diff --git a/src/invidious/views/popular.ecr b/src/invidious/views/feeds/popular.ecr index 62abb12a..e77f35b9 100644 --- a/src/invidious/views/popular.ecr +++ b/src/invidious/views/feeds/popular.ecr @@ -12,9 +12,7 @@ <%= rendered "components/feed_menu" %> <div class="pure-g"> - <% popular_videos.each_slice(4) do |slice| %> - <% slice.each do |item| %> - <%= rendered "components/item" %> - <% end %> - <% end %> +<% popular_videos.each do |item| %> + <%= rendered "components/item" %> +<% end %> </div> diff --git a/src/invidious/views/subscriptions.ecr b/src/invidious/views/feeds/subscriptions.ecr index af1d4fbc..97184e2b 100644 --- a/src/invidious/views/subscriptions.ecr +++ b/src/invidious/views/feeds/subscriptions.ecr @@ -11,13 +11,13 @@ <a href="/subscription_manager"><%= translate(locale, "Manage subscriptions") %></a> </h3> </div> - <div class="pure-u-1-3" style="text-align:center"> - <h3> + <div class="pure-u-1-3"> + <h3 style="text-align:center"> <a href="/feed/history"><%= translate(locale, "Watch history") %></a> </h3> </div> - <div class="pure-u-1-3" style="text-align:right"> - <h3> + <div class="pure-u-1-3"> + <h3 style="text-align:right"> <a href="/feed/private?token=<%= token %>"><i class="icon ion-logo-rss"></i></a> </h3> </div> @@ -34,11 +34,9 @@ <% end %> <div class="pure-g"> - <% notifications.each_slice(4) do |slice| %> - <% slice.each do |item| %> - <%= rendered "components/item" %> - <% end %> - <% end %> +<% notifications.each do |item| %> + <%= rendered "components/item" %> +<% end %> </div> <div class="h-box"> @@ -55,11 +53,9 @@ <script src="/js/watched_widget.js"></script> <div class="pure-g"> - <% videos.each_slice(4) do |slice| %> - <% slice.each do |item| %> - <%= rendered "components/item" %> - <% end %> - <% end %> +<% videos.each do |item| %> + <%= rendered "components/item" %> +<% end %> </div> <div class="pure-g h-box"> diff --git a/src/invidious/views/trending.ecr b/src/invidious/views/feeds/trending.ecr index 42acb15c..a35c4ee3 100644 --- a/src/invidious/views/trending.ecr +++ b/src/invidious/views/feeds/trending.ecr @@ -21,7 +21,7 @@ </div> <div class="pure-u-1-3"> <div class="pure-g" style="text-align:right"> - <% {"Default", "Music", "Gaming", "News", "Movies"}.each do |option| %> + <% {"Default", "Music", "Gaming", "Movies"}.each do |option| %> <div class="pure-u-1 pure-md-1-3"> <% if trending_type == option %> <b><%= translate(locale, option) %></b> @@ -41,9 +41,7 @@ </div> <div class="pure-g"> - <% trending.each_slice(4) do |slice| %> - <% slice.each do |item| %> - <%= rendered "components/item" %> - <% end %> - <% end %> +<% trending.each do |item| %> + <%= rendered "components/item" %> +<% end %> </div> diff --git a/src/invidious/views/history.ecr b/src/invidious/views/history.ecr deleted file mode 100644 index fe8c70b9..00000000 --- a/src/invidious/views/history.ecr +++ /dev/null @@ -1,75 +0,0 @@ -<% content_for "header" do %> -<title><%= translate(locale, "History") %> - Invidious</title> -<% end %> - -<div class="pure-g h-box"> - <div class="pure-u-1-3"> - <h3><%= translate(locale, "`x` videos", %(<span id="count">#{user.watched.size}</span>)) %></h3> - </div> - <div class="pure-u-1-3" style="text-align:center"> - <h3> - <a href="/feed/subscriptions"><%= translate(locale, "`x` subscriptions", %(<span id="count">#{user.subscriptions.size}</span>)) %></a> - </h3> - </div> - <div class="pure-u-1-3" style="text-align:right"> - <h3> - <a href="/clear_watch_history"><%= translate(locale, "Clear watch history") %></a> - </h3> - </div> -</div> - -<script id="watched_data" type="application/json"> -<%= -{ - "csrf_token" => URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") -}.to_pretty_json -%> -</script> -<script src="/js/watched_widget.js"></script> - -<div class="pure-g"> - <% watched.each_slice(4) do |slice| %> - <% slice.each do |item| %> - <div class="pure-u-1 pure-u-md-1-4"> - <div class="h-box"> - <a style="width:100%" href="/watch?v=<%= item %>"> - <% if !env.get("preferences").as(Preferences).thin_mode %> - <div class="thumbnail"> - <img class="thumbnail" src="/vi/<%= item %>/mqdefault.jpg"/> - <form data-onsubmit="return_false" action="/watch_ajax?action_mark_unwatched=1&id=<%= item %>&referer=<%= env.get("current_page") %>" method="post"> - <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> - <p class="watched"> - <a data-onclick="mark_unwatched" data-id="<%= item %>" href="javascript:void(0)"> - <button type="submit" style="all:unset"> - <i class="icon ion-md-trash"></i> - </button> - </a> - </p> - </form> - </div> - <p></p> - <% end %> - </a> - </div> - </div> - <% end %> - <% end %> -</div> - -<div class="pure-g h-box"> - <div class="pure-u-1 pure-u-lg-1-5"> - <% if page > 1 %> - <a href="/feed/history?page=<%= page - 1 %><% if env.params.query["max_results"]? %>&max_results=<%= max_results %><% end %>"> - <%= translate(locale, "Previous page") %> - </a> - <% end %> - </div> - <div class="pure-u-1 pure-u-lg-3-5"></div> - <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> - <% if watched.size >= max_results %> - <a href="/feed/history?page=<%= page + 1 %><% if env.params.query["max_results"]? %>&max_results=<%= max_results %><% end %>"> - <%= translate(locale, "Next page") %> - </a> - <% end %> - </div> -</div> diff --git a/src/invidious/views/licenses.ecr b/src/invidious/views/licenses.ecr index aae8bb19..9f5bcbdd 100644 --- a/src/invidious/views/licenses.ecr +++ b/src/invidious/views/licenses.ecr @@ -151,6 +151,20 @@ <tr> <td> + <a href="/js/videojs-mobile-ui.min.js?v=<%= ASSET_COMMIT %>">videojs-mobile-ui.min.js</a> + </td> + + <td> + <a href="https://choosealicense.com/licenses/mit/">MIT</a> + </td> + + <td> + <a href="https://github.com/mister-ben/videojs-mobile-ui"><%= translate(locale, "source") %></a> + </td> + </tr> + + <tr> + <td> <a href="/js/videojs-markers.min.js?v=<%= ASSET_COMMIT %>">videojs-markers.min.js</a> </td> @@ -221,6 +235,20 @@ <tr> <td> + <a href="/js/videojs-vr.js?v=<%= ASSET_COMMIT %>">videojs-vr.js</a> + </td> + + <td> + <a href="https://choosealicense.com/licenses/mit">MIT</a> + </td> + + <td> + <a href="https://github.com/videojs/videojs-vr"><%= translate(locale, "source") %></a> + </td> + </tr> + + <tr> + <td> <a href="/js/video.min.js?v=<%= ASSET_COMMIT %>">video.min.js</a> </td> diff --git a/src/invidious/views/login.ecr b/src/invidious/views/login.ecr index b6e8117b..e2963e9f 100644 --- a/src/invidious/views/login.ecr +++ b/src/invidious/views/login.ecr @@ -6,27 +6,12 @@ <div class="pure-u-1 pure-u-lg-1-5"></div> <div class="pure-u-1 pure-u-lg-3-5"> <div class="h-box"> - <div class="pure-g"> - <div class="pure-u-1-2"> - <a class="pure-button <% if account_type == "invidious" %>pure-button-disabled<% end %>" href="/login?type=invidious"> - <%= translate(locale, "Log in/register") %> - </a> - </div> - <div class="pure-u-1-2"> - <a class="pure-button <% if account_type == "google" %>pure-button-disabled<% end %>" href="/login?type=google"> - <%= translate(locale, "Log in with Google") %> - </a> - </div> - </div> - - <hr> - <% case account_type when %> <% when "google" %> <form class="pure-form pure-form-stacked" action="/login?referer=<%= URI.encode_www_form(referer) %>&type=google" method="post"> <fieldset> <% if email %> - <input name="email" type="hidden" value="<%= email %>"> + <input name="email" type="hidden" value="<%= HTML.escape(email) %>"> <% else %> <label for="email"><%= translate(locale, "E-mail") %> :</label> <input required class="pure-input-1" name="email" type="email" placeholder="<%= translate(locale, "E-mail") %>"> @@ -62,7 +47,7 @@ <form class="pure-form pure-form-stacked" action="/login?referer=<%= URI.encode_www_form(referer) %>&type=invidious" method="post"> <fieldset> <% if email %> - <input name="email" type="hidden" value="<%= email %>"> + <input name="email" type="hidden" value="<%= HTML.escape(email) %>"> <% else %> <label for="email"><%= translate(locale, "User ID") %> :</label> <input required class="pure-input-1" name="email" type="text" placeholder="<%= translate(locale, "User ID") %>"> diff --git a/src/invidious/views/mix.ecr b/src/invidious/views/mix.ecr index e9c0dcbc..e55b00f8 100644 --- a/src/invidious/views/mix.ecr +++ b/src/invidious/views/mix.ecr @@ -1,22 +1,20 @@ <% content_for "header" do %> -<title><%= mix.title %> - Invidious</title> +<title><%= HTML.escape(mix.title) %> - Invidious</title> <% end %> <div class="pure-g h-box"> <div class="pure-u-2-3"> - <h3><%= mix.title %></h3> + <h3><%= HTML.escape(mix.title) %></h3> </div> - <div class="pure-u-1-3" style="text-align:right"> - <h3> + <div class="pure-u-1-3"> + <h3 style="text-align:right"> <a href="/feed/playlist/<%= mix.id %>"><i class="icon ion-logo-rss"></i></a> </h3> </div> </div> <div class="pure-g"> - <% mix.videos.each_slice(4) do |slice| %> - <% slice.each do |item| %> - <%= rendered "components/item" %> - <% end %> - <% end %> +<% mix.videos.each do |item| %> + <%= rendered "components/item" %> +<% end %> </div> diff --git a/src/invidious/views/playlist.ecr b/src/invidious/views/playlist.ecr index bb721c3a..12f93a72 100644 --- a/src/invidious/views/playlist.ecr +++ b/src/invidious/views/playlist.ecr @@ -1,17 +1,20 @@ +<% title = HTML.escape(playlist.title) %> +<% author = HTML.escape(playlist.author) %> + <% content_for "header" do %> -<title><%= playlist.title %> - Invidious</title> +<title><%= title %> - Invidious</title> <link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/playlist/<%= plid %>" /> <% end %> <div class="pure-g h-box"> <div class="pure-u-2-3"> - <h3><%= playlist.title %></h3> + <h3><%= title %></h3> <% if playlist.is_a? InvidiousPlaylist %> <b> <% if playlist.author == user.try &.email %> - <a href="/view_all_playlists"><%= playlist.author %></a> | + <a href="/feed/playlists"><%= author %></a> | <% else %> - <%= playlist.author %> | + <%= author %> | <% end %> <%= translate(locale, "`x` videos", "#{playlist.video_count}") %> | <%= translate(locale, "Updated `x` ago", recode_date(playlist.updated, locale)) %> | @@ -26,16 +29,21 @@ </b> <% else %> <b> - <a href="/channel/<%= playlist.ucid %>"><%= playlist.author %></a> | + <a href="/channel/<%= playlist.ucid %>"><%= author %></a> | <%= translate(locale, "`x` videos", "#{playlist.video_count}") %> | <%= translate(locale, "Updated `x` ago", recode_date(playlist.updated, locale)) %> </b> <% end %> + <% if !playlist.is_a? InvidiousPlaylist %> <div class="pure-u-2-3"> <a href="https://www.youtube.com/playlist?list=<%= playlist.id %>"> <%= translate(locale, "View playlist on YouTube") %> </a> + <span> | </span> + <a href="/redirect?referer=<%= env.get?("current_page") %>"> + <%= translate(locale, "Switch Invidious Instance") %> + </a> </div> <% end %> </div> @@ -59,7 +67,9 @@ </div> <div class="h-box"> - <p><%= playlist.description_html %></p> + <div id="descriptionWrapper"> + <p><%= playlist.description_html %></p> + </div> </div> <% if playlist.is_a?(InvidiousPlaylist) && playlist.author == user.try &.email %> @@ -86,11 +96,9 @@ <% end %> <div class="pure-g"> - <% videos.each_slice(4) do |slice| %> - <% slice.each do |item| %> - <%= rendered "components/item" %> - <% end %> - <% end %> +<% videos.each do |item| %> + <%= rendered "components/item" %> +<% end %> </div> <div class="pure-g h-box"> @@ -103,7 +111,7 @@ </div> <div class="pure-u-1 pure-u-lg-3-5"></div> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> - <% if videos.size == 100 %> + <% if page_count != 1 && page < page_count %> <a href="/playlist?list=<%= playlist.id %>&page=<%= page + 1 %>"> <%= translate(locale, "Next page") %> </a> diff --git a/src/invidious/views/playlists.ecr b/src/invidious/views/playlists.ecr index a77d106d..d9a17a9b 100644 --- a/src/invidious/views/playlists.ecr +++ b/src/invidious/views/playlists.ecr @@ -1,10 +1,13 @@ +<% ucid = channel.ucid %> +<% author = HTML.escape(channel.author) %> + <% content_for "header" do %> -<title><%= channel.author %> - Invidious</title> +<title><%= author %> - Invidious</title> <% end %> <% if channel.banner %> <div class="h-box"> - <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).full_path %>"> + <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>"> </div> <div class="h-box"> @@ -15,24 +18,24 @@ <div class="pure-g h-box"> <div class="pure-u-2-3"> <div class="channel-profile"> - <img src="/ggpht<%= URI.parse(channel.author_thumbnail).full_path %>"> - <span><%= channel.author %></span> + <img src="/ggpht<%= URI.parse(channel.author_thumbnail).request_target %>"> + <span><%= author %></span> </div> </div> <div class="pure-u-1-3" style="text-align:right"> - <h3> - <a href="/feed/channel/<%= channel.ucid %>"><i class="icon ion-logo-rss"></i></a> + <h3 style="text-align:right"> + <a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a> </h3> </div> </div> <div class="h-box"> - <p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content if !channel.description_html.empty? %></span></p> + <div id="descriptionWrapper"> + <p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content if !channel.description_html.empty? %></span></p> + </div> </div> <div class="h-box"> - <% ucid = channel.ucid %> - <% author = channel.author %> <% sub_count_text = number_to_short_text(channel.sub_count) %> <%= rendered "components/subscribe_widget" %> </div> @@ -40,10 +43,15 @@ <div class="pure-g h-box"> <div class="pure-g pure-u-1-3"> <div class="pure-u-1 pure-md-1-3"> - <a href="https://www.youtube.com/channel/<%= channel.ucid %>/playlists"><%= translate(locale, "View channel on YouTube") %></a> + <a href="https://www.youtube.com/channel/<%= ucid %>/playlists"><%= translate(locale, "View channel on YouTube") %></a> + </div> + + <div class="pure-u-1 pure-md-1-3"> + <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a> </div> + <div class="pure-u-1 pure-md-1-3"> - <a href="/channel/<%= channel.ucid %>"><%= translate(locale, "Videos") %></a> + <a href="/channel/<%= ucid %>"><%= translate(locale, "Videos") %></a> </div> <div class="pure-u-1 pure-md-1-3"> <% if !channel.auto_generated %> @@ -52,7 +60,7 @@ </div> <div class="pure-u-1 pure-md-1-3"> <% if channel.tabs.includes? "community" %> - <a href="/channel/<%= channel.ucid %>/community"><%= translate(locale, "Community") %></a> + <a href="/channel/<%= ucid %>/community"><%= translate(locale, "Community") %></a> <% end %> </div> </div> @@ -64,7 +72,7 @@ <% if sort_by == sort %> <b><%= translate(locale, sort) %></b> <% else %> - <a href="/channel/<%= channel.ucid %>/playlists?sort_by=<%= sort %>"> + <a href="/channel/<%= ucid %>/playlists?sort_by=<%= sort %>"> <%= translate(locale, sort) %> </a> <% end %> @@ -79,18 +87,16 @@ </div> <div class="pure-g"> - <% items.each_slice(4) do |slice| %> - <% slice.each do |item| %> - <%= rendered "components/item" %> - <% end %> - <% end %> +<% items.each do |item| %> + <%= rendered "components/item" %> +<% end %> </div> <div class="pure-g h-box"> <div class="pure-u-1 pure-u-md-4-5"></div> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> <% if continuation %> - <a href="/channel/<%= channel.ucid %>/playlists?continuation=<%= continuation %><% if sort_by != "last" %>&sort_by=<%= HTML.escape(sort_by) %><% end %>"> + <a href="/channel/<%= ucid %>/playlists?continuation=<%= continuation %><% if sort_by != "last" %>&sort_by=<%= HTML.escape(sort_by) %><% end %>"> <%= translate(locale, "Next page") %> </a> <% end %> diff --git a/src/invidious/views/preferences.ecr b/src/invidious/views/preferences.ecr index 1ef080be..be021c59 100644 --- a/src/invidious/views/preferences.ecr +++ b/src/invidious/views/preferences.ecr @@ -106,6 +106,16 @@ <input name="annotations" id="annotations" type="checkbox" <% if preferences.annotations %>checked<% end %>> </div> + <div class="pure-control-group"> + <label for="extend_desc"><%= translate(locale, "Automatically extend video description: ") %></label> + <input name="extend_desc" id="extend_desc" type="checkbox" <% if preferences.extend_desc %>checked<% end %>> + </div> + + <div class="pure-control-group"> + <label for="vr_mode"><%= translate(locale, "Interactive 360 degree videos") %></label> + <input name="vr_mode" id="vr_mode" type="checkbox" <% if preferences.vr_mode %>checked<% end %>> + </div> + <legend><%= translate(locale, "Visual preferences") %></legend> <div class="pure-control-group"> @@ -150,7 +160,7 @@ <label for="default_home"><%= translate(locale, "Default homepage: ") %></label> <select name="default_home" id="default_home"> <% feed_options.each do |option| %> - <option value="<%= option %>" <% if preferences.default_home == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option> + <option value="<%= option %>" <% if preferences.default_home == option %> selected <% end %>><%= translate(locale, option.blank? ? "Search" : option) %></option> <% end %> </select> </div> @@ -160,11 +170,24 @@ <% (feed_options.size - 1).times do |index| %> <select name="feed_menu[<%= index %>]" id="feed_menu[<%= index %>]"> <% feed_options.each do |option| %> - <option value="<%= option %>" <% if preferences.feed_menu[index]? == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option> + <option value="<%= option %>" <% if preferences.feed_menu[index]? == option %> selected <% end %>><%= translate(locale, option.blank? ? "Search" : option) %></option> <% end %> </select> <% end %> </div> + <% if env.get? "user" %> + <div class="pure-control-group"> + <label for="show_nick"><%= translate(locale, "Show nickname on top: ") %></label> + <input name="show_nick" id="show_nick" type="checkbox" <% if preferences.show_nick %>checked<% end %>> + </div> + <% end %> + + <legend><%= translate(locale, "Miscellaneous preferences") %></legend> + + <div class="pure-control-group"> + <label for="automatic_instance_redirect"><%= translate(locale, "Automaticatic instance redirection (fallback to redirect.invidious.io): ") %></label> + <input name="automatic_instance_redirect" id="automatic_instance_redirect" type="checkbox" <% if preferences.automatic_instance_redirect %>checked<% end %>> + </div> <% if env.get? "user" %> <legend><%= translate(locale, "Subscription preferences") %></legend> @@ -208,14 +231,14 @@ </div> <% # Web notifications are only supported over HTTPS %> - <% if Kemal.config.ssl || config.https_only %> + <% if Kemal.config.ssl || CONFIG.https_only %> <div class="pure-control-group"> <a href="#" data-onclick="notification_requestPermission"><%= translate(locale, "Enable web notifications") %></a> </div> <% end %> <% end %> - <% if env.get?("user") && config.admins.includes? env.get?("user").as(User).email %> + <% if env.get?("user") && CONFIG.admins.includes? env.get?("user").as(User).email %> <legend><%= translate(locale, "Administrator preferences") %></legend> <div class="pure-control-group"> @@ -240,28 +263,28 @@ <div class="pure-control-group"> <label for="popular_enabled"><%= translate(locale, "Popular enabled: ") %></label> - <input name="popular_enabled" id="popular_enabled" type="checkbox" <% if config.popular_enabled %>checked<% end %>> + <input name="popular_enabled" id="popular_enabled" type="checkbox" <% if CONFIG.popular_enabled %>checked<% end %>> </div> <div class="pure-control-group"> <label for="captcha_enabled"><%= translate(locale, "CAPTCHA enabled: ") %></label> - <input name="captcha_enabled" id="captcha_enabled" type="checkbox" <% if config.captcha_enabled %>checked<% end %>> + <input name="captcha_enabled" id="captcha_enabled" type="checkbox" <% if CONFIG.captcha_enabled %>checked<% end %>> </div> <div class="pure-control-group"> <label for="login_enabled"><%= translate(locale, "Login enabled: ") %></label> - <input name="login_enabled" id="login_enabled" type="checkbox" <% if config.login_enabled %>checked<% end %>> + <input name="login_enabled" id="login_enabled" type="checkbox" <% if CONFIG.login_enabled %>checked<% end %>> </div> <div class="pure-control-group"> <label for="registration_enabled"><%= translate(locale, "Registration enabled: ") %></label> - <input name="registration_enabled" id="registration_enabled" type="checkbox" <% if config.registration_enabled %>checked<% end %>> + <input name="registration_enabled" id="registration_enabled" type="checkbox" <% if CONFIG.registration_enabled %>checked<% end %>> </div> <div class="pure-control-group"> <label for="statistics_enabled"><%= translate(locale, "Report statistics: ") %></label> - <input name="statistics_enabled" id="statistics_enabled" type="checkbox" <% if config.statistics_enabled %>checked<% end %>> + <input name="statistics_enabled" id="statistics_enabled" type="checkbox" <% if CONFIG.statistics_enabled %>checked<% end %>> </div> <% end %> @@ -289,7 +312,7 @@ </div> <div class="pure-control-group"> - <a href="/view_all_playlists"><%= translate(locale, "View all playlists") %></a> + <a href="/feed/playlists"><%= translate(locale, "View all playlists") %></a> </div> <div class="pure-control-group"> diff --git a/src/invidious/views/search.ecr b/src/invidious/views/search.ecr index bc13b7ea..fd176e41 100644 --- a/src/invidious/views/search.ecr +++ b/src/invidious/views/search.ecr @@ -2,10 +2,112 @@ <title><%= search_query.not_nil!.size > 30 ? HTML.escape(query.not_nil![0,30].rstrip(".") + "...") : HTML.escape(query.not_nil!) %> - Invidious</title> <% end %> +<% search_query_encoded = env.get?("search").try { |x| URI.encode(x.as(String), space_to_plus: true) } %> + +<!-- Search redirection and filtering UI --> +<% if count == 0 %> + <h3 style="text-align: center"> + <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Broken? Try another Invidious Instance!") %></a> + </h3> +<% else %> + <details id="filters"> + <summary> + <h3 style="display:inline"> <%= translate(locale, "filter") %> </h3> + </summary> + <div id="filters" class="pure-g h-box"> + <div class="pure-u-1-3 pure-u-md-1-5"> + <b><%= translate(locale, "date") %></b> + <hr/> + <% ["hour", "today", "week", "month", "year"].each do |date| %> + <div class="pure-u-1 pure-md-1-5"> + <% if operator_hash.fetch("date", "all") == date %> + <b><%= translate(locale, date) %></b> + <% else %> + <a href="/search?q=<%= HTML.escape(query.not_nil!.gsub(/ ?date:[a-z]+/, "") + " date:" + date) %>&page=<%= page %>"> + <%= translate(locale, date) %> + </a> + <% end %> + </div> + <% end %> + </div> + <div class="pure-u-1-3 pure-u-md-1-5"> + <b><%= translate(locale, "content_type") %></b> + <hr/> + <% ["video", "channel", "playlist", "movie", "show"].each do |content_type| %> + <div class="pure-u-1 pure-md-1-5"> + <% if operator_hash.fetch("content_type", "all") == content_type %> + <b><%= translate(locale, content_type) %></b> + <% else %> + <a href="/search?q=<%= HTML.escape(query.not_nil!.gsub(/ ?content_type:[a-z]+/, "") + " content_type:" + content_type) %>&page=<%= page %>"> + <%= translate(locale, content_type) %> + </a> + <% end %> + </div> + <% end %> + </div> + <div class="pure-u-1-3 pure-u-md-1-5"> + <b><%= translate(locale, "duration") %></b> + <hr/> + <% ["short", "long"].each do |duration| %> + <div class="pure-u-1 pure-md-1-5"> + <% if operator_hash.fetch("duration", "all") == duration %> + <b><%= translate(locale, duration) %></b> + <% else %> + <a href="/search?q=<%= HTML.escape(query.not_nil!.gsub(/ ?duration:[a-z]+/, "") + " duration:" + duration) %>&page=<%= page %>"> + <%= translate(locale, duration) %> + </a> + <% end %> + </div> + <% end %> + </div> + <div class="pure-u-1-3 pure-u-md-1-5"> + <b><%= translate(locale, "features") %></b> + <hr/> + <% ["hd", "subtitles", "creative_commons", "3d", "live", "purchased", "4k", "360", "location", "hdr"].each do |feature| %> + <div class="pure-u-1 pure-md-1-5"> + <% if operator_hash.fetch("features", "all").includes?(feature) %> + <b><%= translate(locale, feature) %></b> + <% elsif operator_hash.has_key?("features") %> + <a href="/search?q=<%= HTML.escape(query.not_nil!.gsub(/features:/, "features:" + feature + ",")) %>&page=<%= page %>"> + <%= translate(locale, feature) %> + </a> + <% else %> + <a href="/search?q=<%= HTML.escape(query.not_nil! + " features:" + feature) %>&page=<%= page %>"> + <%= translate(locale, feature) %> + </a> + <% end %> + </div> + <% end %> + </div> + <div class="pure-u-1-3 pure-u-md-1-5"> + <b><%= translate(locale, "sort") %></b> + <hr/> + <% ["relevance", "rating", "date", "views"].each do |sort| %> + <div class="pure-u-1 pure-md-1-5"> + <% if operator_hash.fetch("sort", "relevance") == sort %> + <b><%= translate(locale, sort) %></b> + <% else %> + <a href="/search?q=<%= HTML.escape(query.not_nil!.gsub(/ ?sort:[a-z]+/, "") + " sort:" + sort) %>&page=<%= page %>"> + <%= translate(locale, sort) %> + </a> + <% end %> + </div> + <% end %> + </div> + </div> + </details> +<% end %> + +<% if count == 0 %> + <hr style="margin: 0;"/> +<% else %> + <hr/> +<% end %> + <div class="pure-g h-box v-box"> <div class="pure-u-1 pure-u-lg-1-5"> <% if page > 1 %> - <a href="/search?q=<%= HTML.escape(query.not_nil!) %>&page=<%= page - 1 %>"> + <a href="/search?q=<%= search_query_encoded %>&page=<%= page - 1 %>"> <%= translate(locale, "Previous page") %> </a> <% end %> @@ -13,7 +115,7 @@ <div class="pure-u-1 pure-u-lg-3-5"></div> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> <% if count >= 20 %> - <a href="/search?q=<%= HTML.escape(query.not_nil!) %>&page=<%= page + 1 %>"> + <a href="/search?q=<%= search_query_encoded %>&page=<%= page + 1 %>"> <%= translate(locale, "Next page") %> </a> <% end %> @@ -21,17 +123,15 @@ </div> <div class="pure-g"> - <% videos.each_slice(4) do |slice| %> - <% slice.each do |item| %> - <%= rendered "components/item" %> - <% end %> + <% videos.each do |item| %> + <%= rendered "components/item" %> <% end %> </div> <div class="pure-g h-box"> <div class="pure-u-1 pure-u-lg-1-5"> <% if page > 1 %> - <a href="/search?q=<%= HTML.escape(query.not_nil!) %>&page=<%= page - 1 %>"> + <a href="/search?q=<%= search_query_encoded %>&page=<%= page - 1 %>"> <%= translate(locale, "Previous page") %> </a> <% end %> @@ -39,7 +139,7 @@ <div class="pure-u-1 pure-u-lg-3-5"></div> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> <% if count >= 20 %> - <a href="/search?q=<%= HTML.escape(query.not_nil!) %>&page=<%= page + 1 %>"> + <a href="/search?q=<%= search_query_encoded %>&page=<%= page + 1 %>"> <%= translate(locale, "Next page") %> </a> <% end %> diff --git a/src/invidious/views/search_homepage.ecr b/src/invidious/views/search_homepage.ecr new file mode 100644 index 00000000..7d2dab83 --- /dev/null +++ b/src/invidious/views/search_homepage.ecr @@ -0,0 +1,24 @@ +<% content_for "header" do %> +<meta name="description" content="<%= translate(locale, "An alternative front-end to YouTube") %>"> +<title> + Invidious - <%= translate(locale, "search") %> +</title> +<link rel="stylesheet" href="/css/empty.css?v=<%= ASSET_COMMIT %>"> +<% end %> + +<%= rendered "components/feed_menu" %> + +<div class="pure-g h-box" id="search-widget"> + <div class="pure-u-1" id="logo"> + <h1 href="/" class="pure-menu-heading">Invidious</h1> + </div> + <div class="pure-u-1-4"></div> + <div class="pure-u-1 pure-u-md-12-24 searchbar"> + <form class="pure-form" action="/search" method="get"> + <fieldset> + <input autofocus type="search" style="width:100%" name="q" placeholder="<%= translate(locale, "search") %>" value="<%= env.get?("search").try {|x| HTML.escape(x.as(String)) } %>"> + </fieldset> + </form> + </div> + <div class="pure-u-1-4"></div> +</div> diff --git a/src/invidious/views/subscription_manager.ecr b/src/invidious/views/subscription_manager.ecr index 6cddcd6c..acf015f5 100644 --- a/src/invidious/views/subscription_manager.ecr +++ b/src/invidious/views/subscription_manager.ecr @@ -10,15 +10,15 @@ </a> </h3> </div> - <div class="pure-u-1-3" style="text-align:center"> - <h3> + <div class="pure-u-1-3"> + <h3 style="text-align:center"> <a href="/feed/history"> <%= translate(locale, "Watch history") %> </a> </h3> </div> - <div class="pure-u-1-3" style="text-align:right"> - <h3> + <div class="pure-u-1-3"> + <h3 style="text-align:right"> <a href="/data_control?referer=<%= URI.encode_www_form(referer) %>"> <%= translate(locale, "Import/export") %> </a> @@ -31,7 +31,7 @@ <div class="pure-g<% if channel.deleted %> deleted <% end %>"> <div class="pure-u-2-5"> <h3 style="padding-left:0.5em"> - <a href="/channel/<%= channel.id %>"><%= channel.author %></a> + <a href="/channel/<%= channel.id %>"><%= HTML.escape(channel.author) %></a> </h3> </div> <div class="pure-u-2-5"></div> diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index f6e5262d..7be95959 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -26,18 +26,21 @@ <span style="display:none" id="dark_mode_pref"><%= env.get("preferences").as(Preferences).dark_mode %></span> <div class="pure-g"> <div class="pure-u-1 pure-u-md-2-24"></div> - <div class="pure-u-1 pure-u-md-20-24"> + <div class="pure-u-1 pure-u-md-20-24" id="contents"> <div class="pure-g navbar h-box"> - <div class="pure-u-1 pure-u-md-4-24"> - <a href="/" class="index-link pure-menu-heading">Invidious</a> - </div> - <div class="pure-u-1 pure-u-md-12-24 searchbar"> - <form class="pure-form" action="/search" method="get"> - <fieldset> - <input type="search" style="width:100%" name="q" placeholder="<%= translate(locale, "search") %>" value="<%= env.get?("search").try {|x| HTML.escape(x.as(String)) } %>"> - </fieldset> - </form> - </div> + <% if navbar_search %> + <div class="pure-u-1 pure-u-md-4-24"> + <a href="/" class="index-link pure-menu-heading">Invidious</a> + </div> + <div class="pure-u-1 pure-u-md-12-24 searchbar"> + <form class="pure-form" action="/search" method="get"> + <fieldset> + <input type="search" style="width:100%" name="q" placeholder="<%= translate(locale, "search") %>" value="<%= env.get?("search").try {|x| HTML.escape(x.as(String)) } %>"> + </fieldset> + </form> + </div> + <% end %> + <div class="pure-u-1 pure-u-md-8-24 user-field"> <% if env.get? "user" %> <div class="pure-u-1-4"> @@ -64,6 +67,11 @@ <i class="icon ion-ios-cog"></i> </a> </div> + <% if env.get("preferences").as(Preferences).show_nick %> + <div class="pure-u-1-4"> + <span id="user_name"><%= env.get("user").as(User).email %></span> + </div> + <% end %> <div class="pure-u-1-4"> <form action="/signout?referer=<%= env.get?("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> @@ -87,7 +95,7 @@ <i class="icon ion-ios-cog"></i> </a> </div> - <% if config.login_enabled %> + <% if CONFIG.login_enabled %> <div class="pure-u-1-3"> <a href="/login?referer=<%= env.get?("current_page") %>" class="pure-menu-heading"> <%= translate(locale, "Log in") %> @@ -106,11 +114,11 @@ <%= content %> - <div class="footer"> + <footer> <div class="pure-g"> <div class="pure-u-1 pure-u-md-1-3"> <a href="https://github.com/iv-org/invidious"> - <%= translate(locale, "Released under the AGPLv3 by Omar Roth.") %> + <%= translate(locale, "Released under the AGPLv3 on Github.") %> </a> </div> <div class="pure-u-1 pure-u-md-1-3"> @@ -140,7 +148,7 @@ <%= translate(locale, "Current version: ") %> <%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %> @ <%= CURRENT_BRANCH %> </div> </div> - </div> + </footer> </div> <div class="pure-u-1 pure-u-md-2-24"></div> </div> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 786a88b6..68e7eb80 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -1,10 +1,15 @@ +<% ucid = video.ucid %> +<% title = HTML.escape(video.title) %> +<% author = HTML.escape(video.author) %> + + <% content_for "header" do %> <meta name="thumbnail" content="<%= thumbnail %>"> <meta name="description" content="<%= HTML.escape(video.short_description) %>"> <meta name="keywords" content="<%= video.keywords.join(",") %>"> <meta property="og:site_name" content="Invidious"> <meta property="og:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>"> -<meta property="og:title" content="<%= HTML.escape(video.title) %>"> +<meta property="og:title" content="<%= title %>"> <meta property="og:image" content="/vi/<%= video.id %>/maxres.jpg"> <meta property="og:description" content="<%= video.short_description %>"> <meta property="og:type" content="video.other"> @@ -16,14 +21,28 @@ <meta name="twitter:card" content="player"> <meta name="twitter:site" content="@omarroth1"> <meta name="twitter:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>"> -<meta name="twitter:title" content="<%= HTML.escape(video.title) %>"> +<meta name="twitter:title" content="<%= title %>"> <meta name="twitter:description" content="<%= video.short_description %>"> <meta name="twitter:image" content="<%= HOST_URL %>/vi/<%= video.id %>/maxres.jpg"> <meta name="twitter:player" content="<%= HOST_URL %>/embed/<%= video.id %>"> <meta name="twitter:player:width" content="1280"> <meta name="twitter:player:height" content="720"> +<link rel="alternate" href="https://www.youtube.com/watch?v=<%= video.id %>"> <%= rendered "components/player_sources" %> -<title><%= HTML.escape(video.title) %> - Invidious</title> +<title><%= title %> - Invidious</title> + +<!-- Description expansion also updates the 'Show more' button to 'Show less' so +we're going to need to do it here in order to allow for translations. + --> +<style> +#descexpansionbutton ~ label > a::after { + content: "<%= translate(locale, "Show more") %>" +} + +#descexpansionbutton:checked ~ label > a::after { + content: "<%= translate(locale, "Show less") %>" +} +</style> <% end %> <script id="video_data" type="application/json"> @@ -43,7 +62,9 @@ "show_replies_text" => HTML.escape(translate(locale, "Show replies")), "params" => params, "preferences" => preferences, - "premiere_timestamp" => video.premiere_timestamp.try &.to_unix + "premiere_timestamp" => video.premiere_timestamp.try &.to_unix, + "vr" => video.is_vr, + "projection_type" => video.projection_type }.to_pretty_json %> </script> @@ -54,7 +75,7 @@ <div class="h-box"> <h1> - <%= HTML.escape(video.title) %> + <%= title %> <% if params.listen %> <a title="<%=translate(locale, "Video mode")%>" href="/watch?<%= env.params.query %>&listen=0"> <i class="icon ion-ios-videocam"></i> @@ -80,6 +101,10 @@ <h3> <%= video.premiere_timestamp.try { |t| translate(locale, "Premieres in `x`", recode_date((t - Time.utc).ago, locale)) } %> </h3> + <% elsif video.live_now %> + <h3> + <%= video.premiere_timestamp.try { |t| translate(locale, "Started streaming `x` ago", recode_date((Time.utc - t).ago, locale)) } %> + </h3> <% end %> </div> @@ -90,6 +115,9 @@ <a href="https://www.youtube.com/watch?v=<%= video.id %>"><%= translate(locale, "Watch on YouTube") %></a> (<a href="https://www.youtube.com/embed/<%= video.id %>"><%= translate(locale, "Embed") %></a>) </span> + <p id="watch-on-another-invidious-instance"> + <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a> + </p> <p id="embed-link"> <a href="<%= embed_link %>"><%= translate(locale, "Embed Link") %></a> </p> @@ -112,8 +140,8 @@ <div class="pure-control-group"> <label for="playlist_id"><%= translate(locale, "Add to playlist: ") %></label> <select style="width:100%" name="playlist_id" id="playlist_id"> - <% playlists.each do |plid, title| %> - <option data-plid="<%= plid %>" value="<%= plid %>"><%= title %></option> + <% playlists.each do |plid, playlist_title| %> + <option data-plid="<%= plid %>" value="<%= plid %>"><%= HTML.escape(playlist_title) %></option> <% end %> </select> </div> @@ -156,8 +184,8 @@ </option> <% end %> <% captions.each do |caption| %> - <option value='{"id":"<%= video.id %>","label":"<%= caption.name.simpleText %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= caption.languageCode %>.vtt"}'> - <%= translate(locale, "Subtitles - `x` (.vtt)", caption.name.simpleText) %> + <option value='{"id":"<%= video.id %>","label":"<%= caption.name %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= caption.languageCode %>.vtt"}'> + <%= translate(locale, "Subtitles - `x` (.vtt)", caption.name) %> </option> <% end %> </select> @@ -203,14 +231,12 @@ <a href="/channel/<%= video.ucid %>" style="display:block;width:fit-content;width:-moz-fit-content"> <div class="channel-profile"> <% if !video.author_thumbnail.empty? %> - <img src="/ggpht<%= URI.parse(video.author_thumbnail).full_path %>"> + <img src="/ggpht<%= URI.parse(video.author_thumbnail).request_target %>"> <% end %> - <span id="channel-name"><%= video.author %></span> + <span id="channel-name"><%= author %></span> </div> </a> - <% ucid = video.ucid %> - <% author = video.author %> <% sub_count_text = video.sub_count_text %> <%= rendered "components/subscribe_widget" %> @@ -222,8 +248,20 @@ <% end %> </p> - <div> - <%= video.description_html %> + <div id="description-box"> <!-- Description --> + <% if video.description.size < 200 || params.extend_desc %> + <div id="descriptionWrapper"> + <%= video.description_html %> + </div> + <% else %> + <input id="descexpansionbutton" type="checkbox"/> + <div id="descriptionWrapper"> + <%= video.description_html %> + </div> + <label for="descexpansionbutton"> + <a></a> + </label> + <% end %> </div> <hr> |