diff options
Diffstat (limited to 'src')
46 files changed, 2198 insertions, 1616 deletions
diff --git a/src/ext/kemal_static_file_handler.cr b/src/ext/kemal_static_file_handler.cr index 6ef2d74c..eb068aeb 100644 --- a/src/ext/kemal_static_file_handler.cr +++ b/src/ext/kemal_static_file_handler.cr @@ -111,7 +111,7 @@ module Kemal if @fallthrough call_next(context) else - context.response.status_code = 405 + context.response.status = HTTP::Status::METHOD_NOT_ALLOWED context.response.headers.add("Allow", "GET, HEAD") end return @@ -124,7 +124,7 @@ module Kemal # File path cannot contains '\0' (NUL) because all filesystem I know # don't accept '\0' character as file name. if request_path.includes? '\0' - context.response.status_code = 400 + context.response.status = HTTP::Status::BAD_REQUEST return end @@ -143,13 +143,15 @@ module Kemal add_cache_headers(context.response.headers, last_modified) if cache_request?(context, last_modified) - context.response.status_code = 304 + context.response.status = HTTP::Status::NOT_MODIFIED return end send_file(context, file_path, file[:data], file[:filestat]) else - is_dir = Dir.exists? file_path + file_info = File.info?(file_path) + is_dir = file_info.try &.directory? || false + is_file = file_info.try &.file? || false if request_path != expanded_path redirect_to context, expanded_path @@ -157,19 +159,21 @@ module Kemal redirect_to context, expanded_path + '/' end - if Dir.exists?(file_path) + return call_next(context) if file_info.nil? + + if is_dir if config.is_a?(Hash) && config["dir_listing"] == true context.response.content_type = "text/html" directory_listing(context.response, request_path, file_path) else call_next(context) end - elsif File.exists?(file_path) - last_modified = modification_time(file_path) + elsif is_file + last_modified = file_info.modification_time add_cache_headers(context.response.headers, last_modified) if cache_request?(context, last_modified) - context.response.status_code = 304 + context.response.status = HTTP::Status::NOT_MODIFIED return end @@ -177,14 +181,12 @@ module Kemal data = Bytes.new(size) File.open(file_path, &.read(data)) - filestat = File.info(file_path) - - @cached_files[file_path] = {data: data, filestat: filestat} - send_file(context, file_path, data, filestat) + @cached_files[file_path] = {data: data, filestat: file_info} + send_file(context, file_path, data, file_info) else send_file(context, file_path) end - else + else # Not a normal file (FIFO/device/socket) call_next(context) end end diff --git a/src/invidious.cr b/src/invidious.cr index 070b4d18..2874cc71 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -37,6 +37,9 @@ require "./invidious/database/migrations/*" require "./invidious/helpers/*" require "./invidious/yt_backend/*" require "./invidious/frontend/*" +require "./invidious/videos/*" + +require "./invidious/jsonify/**" require "./invidious/*" require "./invidious/channels/*" @@ -172,311 +175,27 @@ end CONNECTION_CHANNEL = Channel({Bool, Channel(PQ::Notification)}).new(32) Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL, CONFIG.database_url) +Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new + Invidious::Jobs.start_all def popular_videos Invidious::Jobs::PullPopularVideosJob::POPULAR_VIDEOS.get end -before_all do |env| - preferences = Preferences.from_json("{}") - - begin - if prefs_cookie = env.request.cookies["PREFS"]? - preferences = Preferences.from_json(URI.decode_www_form(prefs_cookie.value)) - else - if language_header = env.request.headers["Accept-Language"]? - if language = ANG.language_negotiator.best(language_header, LOCALES.keys) - preferences.locale = language.header - end - end - end - rescue - preferences = Preferences.from_json("{}") - end - - env.set "preferences", preferences - env.response.headers["X-XSS-Protection"] = "1; mode=block" - env.response.headers["X-Content-Type-Options"] = "nosniff" - - # Allow media resources to be loaded from google servers - # TODO: check if *.youtube.com can be removed - if CONFIG.disabled?("local") || !preferences.local - extra_media_csp = " https://*.googlevideo.com:443 https://*.youtube.com:443" - else - extra_media_csp = "" - end - - # Only allow the pages at /embed/* to be embedded - if env.request.resource.starts_with?("/embed") - frame_ancestors = "'self' http: https:" - else - frame_ancestors = "'none'" - end - - # TODO: Remove style-src's 'unsafe-inline', requires to remove all - # inline styles (<style> [..] </style>, style=" [..] ") - env.response.headers["Content-Security-Policy"] = { - "default-src 'none'", - "script-src 'self'", - "style-src 'self' 'unsafe-inline'", - "img-src 'self' data:", - "font-src 'self' data:", - "connect-src 'self'", - "manifest-src 'self'", - "media-src 'self' blob:" + extra_media_csp, - "child-src 'self' blob:", - "frame-src 'self'", - "frame-ancestors " + frame_ancestors, - }.join("; ") - - env.response.headers["Referrer-Policy"] = "same-origin" - - # Ask the chrom*-based browsers to disable FLoC - # See: https://blog.runcloud.io/google-floc/ - env.response.headers["Permissions-Policy"] = "interest-cohort=()" - - if (Kemal.config.ssl || CONFIG.https_only) && CONFIG.hsts - env.response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains; preload" - end - - next if { - "/sb/", - "/vi/", - "/s_p/", - "/yts/", - "/ggpht/", - "/api/manifest/", - "/videoplayback", - "/latest_version", - "/download", - }.any? { |r| env.request.resource.starts_with? r } - - if env.request.cookies.has_key? "SID" - sid = env.request.cookies["SID"].value - - if sid.starts_with? "v1:" - raise "Cannot use token as SID" - end - - # Invidious users only have SID - if !env.request.cookies.has_key? "SSID" - if email = Invidious::Database::SessionIDs.select_email(sid) - user = Invidious::Database::Users.select!(email: email) - csrf_token = generate_response(sid, { - ":authorize_token", - ":playlist_ajax", - ":signout", - ":subscription_ajax", - ":token_ajax", - ":watch_ajax", - }, HMAC_KEY, 1.week) - - preferences = user.preferences - env.set "preferences", preferences - - env.set "sid", sid - env.set "csrf_token", csrf_token - env.set "user", user - end - else - headers = HTTP::Headers.new - headers["Cookie"] = env.request.headers["Cookie"] - - begin - user, sid = get_user(sid, headers, false) - csrf_token = generate_response(sid, { - ":authorize_token", - ":playlist_ajax", - ":signout", - ":subscription_ajax", - ":token_ajax", - ":watch_ajax", - }, HMAC_KEY, 1.week) - - preferences = user.preferences - env.set "preferences", preferences - - env.set "sid", sid - env.set "csrf_token", csrf_token - env.set "user", user - rescue ex - end - end - end - - dark_mode = convert_theme(env.params.query["dark_mode"]?) || preferences.dark_mode.to_s - thin_mode = env.params.query["thin_mode"]? || preferences.thin_mode.to_s - thin_mode = thin_mode == "true" - locale = env.params.query["hl"]? || preferences.locale - - preferences.dark_mode = dark_mode - preferences.thin_mode = thin_mode - preferences.locale = locale - env.set "preferences", preferences - - current_page = env.request.path - if env.request.query - query = HTTP::Params.parse(env.request.query.not_nil!) - - if query["referer"]? - query["referer"] = get_referer(env, "/") - end - - current_page += "?#{query}" - end +# Routing - env.set "current_page", URI.encode_www_form(current_page) +before_all do |env| + Invidious::Routes::BeforeAll.handle(env) end -{% unless flag?(:api_only) %} - Invidious::Routing.get "/", Invidious::Routes::Misc, :home - Invidious::Routing.get "/privacy", Invidious::Routes::Misc, :privacy - Invidious::Routing.get "/licenses", Invidious::Routes::Misc, :licenses - - Invidious::Routing.get "/channel/:ucid", Invidious::Routes::Channels, :home - Invidious::Routing.get "/channel/:ucid/home", Invidious::Routes::Channels, :home - Invidious::Routing.get "/channel/:ucid/videos", Invidious::Routes::Channels, :videos - Invidious::Routing.get "/channel/:ucid/playlists", Invidious::Routes::Channels, :playlists - Invidious::Routing.get "/channel/:ucid/community", Invidious::Routes::Channels, :community - Invidious::Routing.get "/channel/:ucid/about", Invidious::Routes::Channels, :about - Invidious::Routing.get "/channel/:ucid/live", Invidious::Routes::Channels, :live - Invidious::Routing.get "/user/:user/live", Invidious::Routes::Channels, :live - Invidious::Routing.get "/c/:user/live", Invidious::Routes::Channels, :live - - ["", "/videos", "/playlists", "/community", "/about"].each do |path| - # /c/LinusTechTips - Invidious::Routing.get "/c/:user#{path}", Invidious::Routes::Channels, :brand_redirect - # /user/linustechtips | Not always the same as /c/ - Invidious::Routing.get "/user/:user#{path}", Invidious::Routes::Channels, :brand_redirect - # /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow - Invidious::Routing.get "/attribution_link#{path}", Invidious::Routes::Channels, :brand_redirect - # /profile?user=linustechtips - Invidious::Routing.get "/profile/#{path}", Invidious::Routes::Channels, :profile - end - - Invidious::Routing.get "/watch", Invidious::Routes::Watch, :handle - Invidious::Routing.post "/watch_ajax", Invidious::Routes::Watch, :mark_watched - Invidious::Routing.get "/watch/:id", Invidious::Routes::Watch, :redirect - Invidious::Routing.get "/shorts/:id", Invidious::Routes::Watch, :redirect - Invidious::Routing.get "/clip/:clip", Invidious::Routes::Watch, :clip - Invidious::Routing.get "/w/:id", Invidious::Routes::Watch, :redirect - Invidious::Routing.get "/v/:id", Invidious::Routes::Watch, :redirect - Invidious::Routing.get "/e/:id", Invidious::Routes::Watch, :redirect - Invidious::Routing.get "/redirect", Invidious::Routes::Misc, :cross_instance_redirect - - Invidious::Routing.post "/download", Invidious::Routes::Watch, :download - - Invidious::Routing.get "/embed/", Invidious::Routes::Embed, :redirect - Invidious::Routing.get "/embed/:id", Invidious::Routes::Embed, :show - - Invidious::Routing.get "/create_playlist", Invidious::Routes::Playlists, :new - Invidious::Routing.post "/create_playlist", Invidious::Routes::Playlists, :create - Invidious::Routing.get "/subscribe_playlist", Invidious::Routes::Playlists, :subscribe - Invidious::Routing.get "/delete_playlist", Invidious::Routes::Playlists, :delete_page - Invidious::Routing.post "/delete_playlist", Invidious::Routes::Playlists, :delete - Invidious::Routing.get "/edit_playlist", Invidious::Routes::Playlists, :edit - Invidious::Routing.post "/edit_playlist", Invidious::Routes::Playlists, :update - Invidious::Routing.get "/add_playlist_items", Invidious::Routes::Playlists, :add_playlist_items_page - Invidious::Routing.post "/playlist_ajax", Invidious::Routes::Playlists, :playlist_ajax - Invidious::Routing.get "/playlist", Invidious::Routes::Playlists, :show - Invidious::Routing.get "/mix", Invidious::Routes::Playlists, :mix - Invidious::Routing.get "/watch_videos", Invidious::Routes::Playlists, :watch_videos - - Invidious::Routing.get "/opensearch.xml", Invidious::Routes::Search, :opensearch - Invidious::Routing.get "/results", Invidious::Routes::Search, :results - Invidious::Routing.get "/search", Invidious::Routes::Search, :search - Invidious::Routing.get "/hashtag/:hashtag", Invidious::Routes::Search, :hashtag - - # User routes - define_user_routes() - - # Feeds - Invidious::Routing.get "/view_all_playlists", Invidious::Routes::Feeds, :view_all_playlists_redirect - Invidious::Routing.get "/feed/playlists", Invidious::Routes::Feeds, :playlists - Invidious::Routing.get "/feed/popular", Invidious::Routes::Feeds, :popular - Invidious::Routing.get "/feed/trending", Invidious::Routes::Feeds, :trending - Invidious::Routing.get "/feed/subscriptions", Invidious::Routes::Feeds, :subscriptions - Invidious::Routing.get "/feed/history", Invidious::Routes::Feeds, :history - - # RSS Feeds - Invidious::Routing.get "/feed/channel/:ucid", Invidious::Routes::Feeds, :rss_channel - Invidious::Routing.get "/feed/private", Invidious::Routes::Feeds, :rss_private - Invidious::Routing.get "/feed/playlist/:plid", Invidious::Routes::Feeds, :rss_playlist - Invidious::Routing.get "/feeds/videos.xml", Invidious::Routes::Feeds, :rss_videos - - # Support push notifications via PubSubHubbub - Invidious::Routing.get "/feed/webhook/:token", Invidious::Routes::Feeds, :push_notifications_get - Invidious::Routing.post "/feed/webhook/:token", Invidious::Routes::Feeds, :push_notifications_post - - Invidious::Routing.get "/modify_notifications", Invidious::Routes::Notifications, :modify - - Invidious::Routing.post "/subscription_ajax", Invidious::Routes::Subscriptions, :toggle_subscription - Invidious::Routing.get "/subscription_manager", Invidious::Routes::Subscriptions, :subscription_manager -{% end %} - -Invidious::Routing.get "/ggpht/*", Invidious::Routes::Images, :ggpht -Invidious::Routing.options "/sb/:authority/:id/:storyboard/:index", Invidious::Routes::Images, :options_storyboard -Invidious::Routing.get "/sb/:authority/:id/:storyboard/:index", Invidious::Routes::Images, :get_storyboard -Invidious::Routing.get "/s_p/:id/:name", Invidious::Routes::Images, :s_p_image -Invidious::Routing.get "/yts/img/:name", Invidious::Routes::Images, :yts_image -Invidious::Routing.get "/vi/:id/:name", Invidious::Routes::Images, :thumbnails - -# API routes (macro) -define_v1_api_routes() - -# Video playback (macros) -define_api_manifest_routes() -define_video_playback_routes() +Invidious::Routing.register_all error 404 do |env| - if md = env.request.path.match(/^\/(?<id>([a-zA-Z0-9_-]{11})|(\w+))$/) - item = md["id"] - - # Check if item is branding URL e.g. https://youtube.com/gaming - response = YT_POOL.client &.get("/#{item}") - - if response.status_code == 301 - response = YT_POOL.client &.get(URI.parse(response.headers["Location"]).request_target) - end - - if response.body.empty? - env.response.headers["Location"] = "/" - halt env, status_code: 302 - end - - html = XML.parse_html(response.body) - ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1] - - if ucid - env.response.headers["Location"] = "/channel/#{ucid}" - halt env, status_code: 302 - end - - params = [] of String - env.params.query.each do |k, v| - params << "#{k}=#{v}" - end - params = params.join("&") - - url = "/watch?v=#{item}" - if !params.empty? - url += "&#{params}" - end - - # Check if item is video ID - if item.match(/^[a-zA-Z0-9_-]{11}$/) && YT_POOL.client &.head("/watch?v=#{item}").status_code != 404 - env.response.headers["Location"] = url - halt env, status_code: 302 - end - end - - env.response.headers["Location"] = "/" - halt env, status_code: 302 + Invidious::Routes::ErrorRoutes.error_404(env) end error 500 do |env, ex| - locale = env.get("preferences").as(Preferences).locale error_template(500, ex) end @@ -484,6 +203,8 @@ static_headers do |response| response.headers.add("Cache-Control", "max-age=2629800") end +# Init Kemal + public_folder "assets" Kemal.config.powered_by_header = false diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index f60ee7af..4c442959 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -130,8 +130,9 @@ def get_about_info(ucid, locale) : AboutChannel tabs = tabs_json.reject { |node| node["tabRenderer"]?.nil? }.map(&.["tabRenderer"]["title"].as_s.downcase) end - sub_count = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s? - .try { |text| short_text_to_number(text.split(" ")[0]) } || 0 + sub_count = initdata + .dig?("header", "c4TabbedHeaderRenderer", "subscriberCountText", "simpleText").try &.as_s? + .try { |text| short_text_to_number(text.split(" ")[0]).to_i32 } || 0 AboutChannel.new( ucid: ucid, diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index e0459cc3..e3d3d9ee 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -29,7 +29,7 @@ struct ChannelVideo json.field "title", self.title json.field "videoId", self.id json.field "videoThumbnails" do - generate_thumbnails(json, self.id) + Invidious::JSONify::APIv1.thumbnails(json, self.id) end json.field "lengthSeconds", self.length_seconds diff --git a/src/invidious/channels/community.cr b/src/invidious/channels/community.cr index 2a2c74aa..8e300288 100644 --- a/src/invidious/channels/community.cr +++ b/src/invidious/channels/community.cr @@ -138,7 +138,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) json.field "title", video_title json.field "videoId", video_id json.field "videoThumbnails" do - generate_thumbnails(json, video_id) + Invidious::JSONify::APIv1.thumbnails(json, video_id) end json.field "lengthSeconds", decode_length_seconds(attachment["lengthText"]["simpleText"].as_s) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 48453bb7..b495e597 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -1,53 +1,48 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:base64" => { - "2:string" => "videos", - "6:varint" => 2_i64, - "7:varint" => 1_i64, - "12:varint" => 1_i64, - "13:string" => "", - "23:varint" => 0_i64, - }, + object_inner_2 = { + "2:0:embedded" => { + "1:0:varint" => 0_i64, }, + "5:varint" => 50_i64, + "6:varint" => 1_i64, + "7:varint" => (page * 30).to_i64, + "9:varint" => 1_i64, + "10:varint" => 0_i64, } - if !v2 - if auto_generated - seed = Time.unix(1525757349) - until seed >= Time.utc - seed += 1.month - end - timestamp = seed - (page - 1).months - - object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x36_i64 - object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{timestamp.to_unix}" - else - object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0_i64 - object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{page}" - end - else - object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0_i64 + object_inner_2_encoded = object_inner_2 + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } - object["80226972:embedded"]["3:base64"].as(Hash)["61:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json({ - "1:string" => Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json({ - "1:varint" => 30_i64 * (page - 1), - }))), - }))) - end + object_inner_1 = { + "110:embedded" => { + "3:embedded" => { + "15:embedded" => { + "1:embedded" => { + "1:string" => object_inner_2_encoded, + "2:string" => "00000000-0000-0000-0000-000000000000", + }, + "3:varint" => 1_i64, + }, + }, + }, + } - case sort_by - when "newest" - when "popular" - object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 0x01_i64 - when "oldest" - object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 0x02_i64 - else nil # Ignore - end + object_inner_1_encoded = object_inner_1 + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } - object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"]))) - object["80226972:embedded"].delete("3:base64") + object = { + "80226972:embedded" => { + "2:string" => ucid, + "3:string" => object_inner_1_encoded, + "35:string" => "browse-feed#{ucid}videos102", + }, + } continuation = object.try { |i| Protodec::Any.cast_json(i) } .try { |i| Protodec::Any.from_json(i) } @@ -67,10 +62,11 @@ end def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") videos = [] of SearchVideo - 2.times do |i| - initial_data = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by) - videos.concat extract_videos(initial_data, author, ucid) - end + # 2.times do |i| + # initial_data = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by) + initial_data = get_channel_videos_response(ucid, 1, auto_generated: auto_generated, sort_by: sort_by) + videos = extract_videos(initial_data, author, ucid) + # end return videos.size, videos end diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index 5112ad3d..d691ca36 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -201,15 +201,6 @@ def fetch_youtube_comments(id, cursor, format, locale, thin_mode, region, sort_b end if node_replies && !response["commentRepliesContinuation"]? - if node_replies["moreText"]? - reply_count = (node_replies["moreText"]["simpleText"]? || node_replies["moreText"]["runs"]?.try &.[0]?.try &.["text"]?) - .try &.as_s.gsub(/\D/, "").to_i? || 1 - elsif node_replies["viewReplies"]? - reply_count = node_replies["viewReplies"]["buttonRenderer"]["text"]?.try &.["runs"][1]?.try &.["text"]?.try &.as_s.to_i? || 1 - else - reply_count = 1 - end - if node_replies["continuations"]? continuation = node_replies["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s elsif node_replies["contents"]? @@ -219,7 +210,7 @@ def fetch_youtube_comments(id, cursor, format, locale, thin_mode, region, sort_b json.field "replies" do json.object do - json.field "replyCount", reply_count + json.field "replyCount", node_comment["replyCount"]? || 1 json.field "continuation", continuation end end diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 786b65df..c9bf43a4 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -78,6 +78,10 @@ class Config property decrypt_polling : Bool = false # Used for crawling channels: threads should check all videos uploaded by a channel property full_refresh : Bool = false + + # Jobs config structure. See jobs.cr and jobs/base_job.cr + property jobs = Invidious::Jobs::JobsConfig.new + # Used to tell Invidious it is behind a proxy, so links to resources should be https:// property https_only : Bool? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions @@ -131,6 +135,9 @@ class Config # API URL for Anti-Captcha property captcha_api_url : String = "https://api.anti-captcha.com" + # Playlist length limit + property playlist_length_limit : Int32 = 500 + def disabled?(option) case disabled = CONFIG.disable_proxy when Bool diff --git a/src/invidious/database/nonces.cr b/src/invidious/database/nonces.cr index 469fcbd8..b87c81ec 100644 --- a/src/invidious/database/nonces.cr +++ b/src/invidious/database/nonces.cr @@ -4,7 +4,7 @@ module Invidious::Database::Nonces extend self # ------------------- - # Insert + # Insert / Delete # ------------------- def insert(nonce : String, expire : Time) @@ -17,6 +17,15 @@ module Invidious::Database::Nonces PG_DB.exec(request, nonce, expire) end + def delete_expired + request = <<-SQL + DELETE FROM nonces * + WHERE expire < now() + SQL + + PG_DB.exec(request) + end + # ------------------- # Update # ------------------- diff --git a/src/invidious/database/videos.cr b/src/invidious/database/videos.cr index e1fa01c3..695f5b33 100644 --- a/src/invidious/database/videos.cr +++ b/src/invidious/database/videos.cr @@ -22,6 +22,15 @@ module Invidious::Database::Videos PG_DB.exec(request, id) end + def delete_expired + request = <<-SQL + DELETE FROM videos * + WHERE updated < (now() - interval '6 hours') + SQL + + PG_DB.exec(request) + end + def update(video : Video) request = <<-SQL UPDATE videos diff --git a/src/invidious/exceptions.cr b/src/invidious/exceptions.cr index 05be73a6..425c08da 100644 --- a/src/invidious/exceptions.cr +++ b/src/invidious/exceptions.cr @@ -30,3 +30,6 @@ end # Exception threw when an element is not found. class NotFoundException < InfoException end + +class VideoNotAvailableException < Exception +end diff --git a/src/invidious/frontend/watch_page.cr b/src/invidious/frontend/watch_page.cr index 80b67641..a9b00860 100644 --- a/src/invidious/frontend/watch_page.cr +++ b/src/invidious/frontend/watch_page.cr @@ -7,7 +7,7 @@ module Invidious::Frontend::WatchPage getter full_videos : Array(Hash(String, JSON::Any)) getter video_streams : Array(Hash(String, JSON::Any)) getter audio_streams : Array(Hash(String, JSON::Any)) - getter captions : Array(Caption) + getter captions : Array(Invidious::Videos::Caption) def initialize( @full_videos, @@ -50,7 +50,7 @@ module Invidious::Frontend::WatchPage video_assets.full_videos.each do |option| mimetype = option["mimeType"].as_s.split(";")[0] - height = itag_to_metadata?(option["itag"]).try &.["height"]? + height = Invidious::Videos::Formats.itag_to_metadata?(option["itag"]).try &.["height"]? value = {"itag": option["itag"], "ext": mimetype.split("/")[1]}.to_json diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index fd86594c..a9ed1f64 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -1,8 +1,7 @@ -# "bn_BD" => load_locale("bn_BD"), # Bengali (Bangladesh) [Incomplete] -# "eu" => load_locale("eu"), # Basque [Incomplete] -# "sk" => load_locale("sk"), # Slovak [Incomplete] LOCALES_LIST = { "ar" => "العربية", # Arabic + "bn" => "বাংলা", # Bengali + "ca" => "Català", # Catalan "cs" => "Čeština", # Czech "da" => "Dansk", # Danish "de" => "Deutsch", # German @@ -11,6 +10,7 @@ LOCALES_LIST = { "eo" => "Esperanto", # Esperanto "es" => "Español", # Spanish "et" => "Eesti keel", # Estonian + "eu" => "Euskara", # Basque "fa" => "فارسی", # Persian "fi" => "Suomi", # Finnish "fr" => "Français", # French @@ -32,6 +32,8 @@ LOCALES_LIST = { "pt-PT" => "Português de Portugal", # Portuguese (Portugal) "ro" => "Română", # Romanian "ru" => "Русский", # Russian + "si" => "සිංහල", # Sinhala + "sk" => "Slovenčina", # Slovak "sl" => "Slovenščina", # Slovenian "sq" => "Shqip", # Albanian "sr" => "Srpski (latinica)", # Serbian (Latin) diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index 3918bd13..c52e2a0d 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -76,7 +76,7 @@ struct SearchVideo json.field "authorUrl", "/channel/#{self.ucid}" json.field "videoThumbnails" do - generate_thumbnails(json, self.id) + Invidious::JSONify::APIv1.thumbnails(json, self.id) end json.field "description", html_to_content(self.description_html) @@ -155,7 +155,7 @@ struct SearchPlaylist json.field "lengthSeconds", video.length_seconds json.field "videoThumbnails" do - generate_thumbnails(json, video.id) + Invidious::JSONify::APIv1.thumbnails(json, video.id) end end end diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 8ae5034a..ed0cca38 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -161,21 +161,19 @@ def number_with_separator(number) number.to_s.reverse.gsub(/(\d{3})(?=\d)/, "\\1,").reverse end -def short_text_to_number(short_text : String) : Int32 - case short_text - when .ends_with? "M" - number = short_text.rstrip(" mM").to_f - number *= 1000000 - when .ends_with? "K" - number = short_text.rstrip(" kK").to_f - number *= 1000 - else - number = short_text.rstrip(" ") +def short_text_to_number(short_text : String) : Int64 + matches = /(?<number>\d+(\.\d+)?)\s?(?<suffix>[mMkKbB])?/.match(short_text) + number = matches.try &.["number"].to_f || 0.0 + + case matches.try &.["suffix"].downcase + when "k" then number *= 1_000 + when "m" then number *= 1_000_000 + when "b" then number *= 1_000_000_000 end - number = number.to_i - - return number + return number.to_i64 +rescue ex + return 0_i64 end def number_to_short_text(number) diff --git a/src/invidious/jobs.cr b/src/invidious/jobs.cr index ec0cad64..524a3624 100644 --- a/src/invidious/jobs.cr +++ b/src/invidious/jobs.cr @@ -1,12 +1,39 @@ module Invidious::Jobs JOBS = [] of BaseJob + # Automatically generate a structure that wraps the various + # jobs' configs, so that the follwing YAML config can be used: + # + # jobs: + # job_name: + # enabled: true + # some_property: "value" + # + macro finished + struct JobsConfig + include YAML::Serializable + + {% for sc in BaseJob.subclasses %} + # Voodoo macro to transform `Some::Module::CustomJob` to `custom` + {% class_name = sc.id.split("::").last.id.gsub(/Job$/, "").underscore %} + + getter {{ class_name }} = {{ sc.name }}::Config.new + {% end %} + + def initialize + end + end + end + def self.register(job : BaseJob) JOBS << job end def self.start_all JOBS.each do |job| + # Don't run the main rountine if the job is disabled by config + next if job.disabled? + spawn { job.begin } end end diff --git a/src/invidious/jobs/base_job.cr b/src/invidious/jobs/base_job.cr index 47e75864..f90f0bfe 100644 --- a/src/invidious/jobs/base_job.cr +++ b/src/invidious/jobs/base_job.cr @@ -1,3 +1,33 @@ abstract class Invidious::Jobs::BaseJob abstract def begin + + # When this base job class is inherited, make sure to define + # a basic "Config" structure, that contains the "enable" property, + # and to create the associated instance property. + # + macro inherited + macro finished + # This config structure can be expanded as required. + struct Config + include YAML::Serializable + + property enable = true + + def initialize + end + end + + property cfg = Config.new + + # Return true if job is enabled by config + protected def enabled? : Bool + return (@cfg.enable == true) + end + + # Return true if job is disabled by config + protected def disabled? : Bool + return (@cfg.enable == false) + end + end + end end diff --git a/src/invidious/jobs/clear_expired_items_job.cr b/src/invidious/jobs/clear_expired_items_job.cr new file mode 100644 index 00000000..17191aac --- /dev/null +++ b/src/invidious/jobs/clear_expired_items_job.cr @@ -0,0 +1,27 @@ +class Invidious::Jobs::ClearExpiredItemsJob < Invidious::Jobs::BaseJob + # Remove items (videos, nonces, etc..) whose cache is outdated every hour. + # Removes the need for a cron job. + def begin + loop do + failed = false + + LOGGER.info("jobs: running ClearExpiredItems job") + + begin + Invidious::Database::Videos.delete_expired + Invidious::Database::Nonces.delete_expired + rescue DB::Error + failed = true + end + + # Retry earlier than scheduled on DB error + if failed + LOGGER.info("jobs: ClearExpiredItems failed. Retrying in 10 minutes.") + sleep 10.minutes + else + LOGGER.info("jobs: ClearExpiredItems done.") + sleep 1.hour + end + end + end +end diff --git a/src/invidious/jsonify/api_v1/common.cr b/src/invidious/jsonify/api_v1/common.cr new file mode 100644 index 00000000..64b06465 --- /dev/null +++ b/src/invidious/jsonify/api_v1/common.cr @@ -0,0 +1,18 @@ +require "json" + +module Invidious::JSONify::APIv1 + extend self + + def thumbnails(json : JSON::Builder, id : String) + json.array do + build_thumbnails(id).each do |thumbnail| + json.object do + json.field "quality", thumbnail[:name] + json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg" + json.field "width", thumbnail[:width] + json.field "height", thumbnail[:height] + end + end + end + end +end diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr new file mode 100644 index 00000000..642789aa --- /dev/null +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -0,0 +1,251 @@ +require "json" + +module Invidious::JSONify::APIv1 + extend self + + def video(video : Video, json : JSON::Builder, *, locale : String?) + json.object do + json.field "type", video.video_type + + json.field "title", video.title + json.field "videoId", video.id + + json.field "error", video.info["reason"] if video.info["reason"]? + + json.field "videoThumbnails" do + self.thumbnails(json, video.id) + end + json.field "storyboards" do + self.storyboards(json, video.id, video.storyboards) + end + + json.field "description", video.description + json.field "descriptionHtml", video.description_html + json.field "published", video.published.to_unix + json.field "publishedText", translate(locale, "`x` ago", recode_date(video.published, locale)) + json.field "keywords", video.keywords + + json.field "viewCount", video.views + json.field "likeCount", video.likes + json.field "dislikeCount", 0_i64 + + json.field "paid", video.paid + json.field "premium", video.premium + json.field "isFamilyFriendly", video.is_family_friendly + json.field "allowedRegions", video.allowed_regions + json.field "genre", video.genre + json.field "genreUrl", video.genre_url + + json.field "author", video.author + json.field "authorId", video.ucid + json.field "authorUrl", "/channel/#{video.ucid}" + + json.field "authorThumbnails" do + json.array do + qualities = {32, 48, 76, 100, 176, 512} + + qualities.each do |quality| + json.object do + json.field "url", video.author_thumbnail.gsub(/=s\d+/, "=s#{quality}") + json.field "width", quality + json.field "height", quality + end + end + end + end + + json.field "subCountText", video.sub_count_text + + json.field "lengthSeconds", video.length_seconds + json.field "allowRatings", video.allow_ratings + json.field "rating", 0_i64 + json.field "isListed", video.is_listed + json.field "liveNow", video.live_now + json.field "isUpcoming", video.is_upcoming + + if video.premiere_timestamp + json.field "premiereTimestamp", video.premiere_timestamp.try &.to_unix + end + + if hlsvp = video.hls_manifest_url + hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL) + json.field "hlsUrl", hlsvp + end + + json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{video.id}" + + json.field "adaptiveFormats" do + json.array do + video.adaptive_fmts.each do |fmt| + json.object do + # Only available on regular videos, not livestreams/OTF streams + if init_range = fmt["initRange"]? + json.field "init", "#{init_range["start"]}-#{init_range["end"]}" + end + if index_range = fmt["indexRange"]? + json.field "index", "#{index_range["start"]}-#{index_range["end"]}" + end + + # Not available on MPEG-4 Timed Text (`text/mp4`) streams (livestreams only) + json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]? + + json.field "url", fmt["url"] + json.field "itag", fmt["itag"].as_i.to_s + json.field "type", fmt["mimeType"] + json.field "clen", fmt["contentLength"]? || "-1" + + # Last modified is a unix timestamp with µS, with the dot omitted. + # E.g: 1638056732(.)141582 + # + # On livestreams, it's not present, so always fall back to the + # current unix timestamp (up to mS precision) for compatibility. + last_modified = fmt["lastModified"]? + last_modified ||= "#{Time.utc.to_unix_ms.to_s}000" + json.field "lmt", last_modified + + json.field "projectionType", fmt["projectionType"] + + if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) + fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 + json.field "fps", fps + json.field "container", fmt_info["ext"] + json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] + + if fmt_info["height"]? + json.field "resolution", "#{fmt_info["height"]}p" + + quality_label = "#{fmt_info["height"]}p" + if fps > 30 + quality_label += "60" + end + json.field "qualityLabel", quality_label + + if fmt_info["width"]? + json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" + end + end + end + + # Livestream chunk infos + json.field "targetDurationSec", fmt["targetDurationSec"].as_i if fmt.has_key?("targetDurationSec") + json.field "maxDvrDurationSec", fmt["maxDvrDurationSec"].as_i if fmt.has_key?("maxDvrDurationSec") + + # Audio-related data + json.field "audioQuality", fmt["audioQuality"] if fmt.has_key?("audioQuality") + json.field "audioSampleRate", fmt["audioSampleRate"].as_s.to_i if fmt.has_key?("audioSampleRate") + json.field "audioChannels", fmt["audioChannels"] if fmt.has_key?("audioChannels") + + # Extra misc stuff + json.field "colorInfo", fmt["colorInfo"] if fmt.has_key?("colorInfo") + json.field "captionTrack", fmt["captionTrack"] if fmt.has_key?("captionTrack") + end + end + end + end + + json.field "formatStreams" do + json.array do + video.fmt_stream.each do |fmt| + json.object do + json.field "url", fmt["url"] + json.field "itag", fmt["itag"].as_i.to_s + json.field "type", fmt["mimeType"] + json.field "quality", fmt["quality"] + + fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) + if fmt_info + fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 + json.field "fps", fps + json.field "container", fmt_info["ext"] + json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] + + if fmt_info["height"]? + json.field "resolution", "#{fmt_info["height"]}p" + + quality_label = "#{fmt_info["height"]}p" + if fps > 30 + quality_label += "60" + end + json.field "qualityLabel", quality_label + + if fmt_info["width"]? + json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" + end + end + end + end + end + end + end + + json.field "captions" do + json.array do + video.captions.each do |caption| + json.object do + json.field "label", caption.name + json.field "language_code", caption.language_code + json.field "url", "/api/v1/captions/#{video.id}?label=#{URI.encode_www_form(caption.name)}" + end + end + end + end + + json.field "recommendedVideos" do + json.array do + video.related_videos.each do |rv| + if rv["id"]? + json.object do + json.field "videoId", rv["id"] + json.field "title", rv["title"] + json.field "videoThumbnails" do + self.thumbnails(json, rv["id"]) + end + + json.field "author", rv["author"] + json.field "authorUrl", "/channel/#{rv["ucid"]?}" + json.field "authorId", rv["ucid"]? + if rv["author_thumbnail"]? + json.field "authorThumbnails" do + json.array do + qualities = {32, 48, 76, 100, 176, 512} + + qualities.each do |quality| + json.object do + json.field "url", rv["author_thumbnail"].gsub(/s\d+-/, "s#{quality}-") + json.field "width", quality + json.field "height", quality + end + end + end + end + end + + json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i + json.field "viewCountText", rv["short_view_count"]? + json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 + end + end + end + end + end + end + end + + def storyboards(json, id, storyboards) + json.array do + storyboards.each do |storyboard| + json.object do + json.field "url", "/api/v1/storyboards/#{id}?width=#{storyboard[:width]}&height=#{storyboard[:height]}" + json.field "templateUrl", storyboard[:url] + json.field "width", storyboard[:width] + json.field "height", storyboard[:height] + json.field "count", storyboard[:count] + json.field "interval", storyboard[:interval] + json.field "storyboardWidth", storyboard[:storyboard_width] + json.field "storyboardHeight", storyboard[:storyboard_height] + json.field "storyboardCount", storyboard[:storyboard_count] + end + end + end + end +end diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index c4eb7507..57f1f53e 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -56,7 +56,7 @@ struct PlaylistVideo json.field "authorUrl", "/channel/#{self.ucid}" json.field "videoThumbnails" do - generate_thumbnails(json, self.id) + Invidious::JSONify::APIv1.thumbnails(json, self.id) end if index diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index bfb8a377..ae65f10d 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -14,8 +14,6 @@ module Invidious::Routes::API::Manifest begin video = get_video(id, region: region) - rescue ex : VideoRedirect - return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex : NotFoundException haltf env, status_code: 404 rescue ex diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index 1f5ad8ef..421355bb 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -226,8 +226,8 @@ module Invidious::Routes::API::V1::Authenticated return error_json(403, "Invalid user") end - if playlist.index.size >= 500 - return error_json(400, "Playlist cannot have more than 500 videos") + if playlist.index.size >= CONFIG.playlist_length_limit + return error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos") end video_id = env.params.json["videoId"].try &.as(String) diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index 844fedb8..43d360e6 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -124,7 +124,7 @@ module Invidious::Routes::API::V1::Misc json.field "videoThumbnails" do json.array do - generate_thumbnails(json, video.id) + Invidious::JSONify::APIv1.thumbnails(json, video.id) end end diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 1b7b4fa7..a6b2eb4e 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -9,9 +9,6 @@ module Invidious::Routes::API::V1::Videos begin video = get_video(id, region: region) - rescue ex : VideoRedirect - env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) - return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) rescue ex : NotFoundException return error_json(404, ex) rescue ex @@ -41,9 +38,6 @@ module Invidious::Routes::API::V1::Videos begin video = get_video(id, region: region) - rescue ex : VideoRedirect - env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) - return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) rescue ex : NotFoundException haltf env, 404 rescue ex @@ -168,9 +162,6 @@ module Invidious::Routes::API::V1::Videos begin video = get_video(id, region: region) - rescue ex : VideoRedirect - env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) - return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) rescue ex : NotFoundException haltf env, 404 rescue ex @@ -185,7 +176,7 @@ module Invidious::Routes::API::V1::Videos response = JSON.build do |json| json.object do json.field "storyboards" do - generate_storyboards(json, id, storyboards) + Invidious::JSONify::APIv1.storyboards(json, id, storyboards) end end end diff --git a/src/invidious/routes/before_all.cr b/src/invidious/routes/before_all.cr new file mode 100644 index 00000000..8e2a253f --- /dev/null +++ b/src/invidious/routes/before_all.cr @@ -0,0 +1,152 @@ +module Invidious::Routes::BeforeAll + def self.handle(env) + preferences = Preferences.from_json("{}") + + begin + if prefs_cookie = env.request.cookies["PREFS"]? + preferences = Preferences.from_json(URI.decode_www_form(prefs_cookie.value)) + else + if language_header = env.request.headers["Accept-Language"]? + if language = ANG.language_negotiator.best(language_header, LOCALES.keys) + preferences.locale = language.header + end + end + end + rescue + preferences = Preferences.from_json("{}") + end + + env.set "preferences", preferences + env.response.headers["X-XSS-Protection"] = "1; mode=block" + env.response.headers["X-Content-Type-Options"] = "nosniff" + + # Allow media resources to be loaded from google servers + # TODO: check if *.youtube.com can be removed + if CONFIG.disabled?("local") || !preferences.local + extra_media_csp = " https://*.googlevideo.com:443 https://*.youtube.com:443" + else + extra_media_csp = "" + end + + # Only allow the pages at /embed/* to be embedded + if env.request.resource.starts_with?("/embed") + frame_ancestors = "'self' http: https:" + else + frame_ancestors = "'none'" + end + + # TODO: Remove style-src's 'unsafe-inline', requires to remove all + # inline styles (<style> [..] </style>, style=" [..] ") + env.response.headers["Content-Security-Policy"] = { + "default-src 'none'", + "script-src 'self'", + "style-src 'self' 'unsafe-inline'", + "img-src 'self' data:", + "font-src 'self' data:", + "connect-src 'self'", + "manifest-src 'self'", + "media-src 'self' blob:" + extra_media_csp, + "child-src 'self' blob:", + "frame-src 'self'", + "frame-ancestors " + frame_ancestors, + }.join("; ") + + env.response.headers["Referrer-Policy"] = "same-origin" + + # Ask the chrom*-based browsers to disable FLoC + # See: https://blog.runcloud.io/google-floc/ + env.response.headers["Permissions-Policy"] = "interest-cohort=()" + + if (Kemal.config.ssl || CONFIG.https_only) && CONFIG.hsts + env.response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains; preload" + end + + return if { + "/sb/", + "/vi/", + "/s_p/", + "/yts/", + "/ggpht/", + "/api/manifest/", + "/videoplayback", + "/latest_version", + "/download", + }.any? { |r| env.request.resource.starts_with? r } + + if env.request.cookies.has_key? "SID" + sid = env.request.cookies["SID"].value + + if sid.starts_with? "v1:" + raise "Cannot use token as SID" + end + + # Invidious users only have SID + if !env.request.cookies.has_key? "SSID" + if email = Invidious::Database::SessionIDs.select_email(sid) + user = Invidious::Database::Users.select!(email: email) + csrf_token = generate_response(sid, { + ":authorize_token", + ":playlist_ajax", + ":signout", + ":subscription_ajax", + ":token_ajax", + ":watch_ajax", + }, HMAC_KEY, 1.week) + + preferences = user.preferences + env.set "preferences", preferences + + env.set "sid", sid + env.set "csrf_token", csrf_token + env.set "user", user + end + else + headers = HTTP::Headers.new + headers["Cookie"] = env.request.headers["Cookie"] + + begin + user, sid = get_user(sid, headers, false) + csrf_token = generate_response(sid, { + ":authorize_token", + ":playlist_ajax", + ":signout", + ":subscription_ajax", + ":token_ajax", + ":watch_ajax", + }, HMAC_KEY, 1.week) + + preferences = user.preferences + env.set "preferences", preferences + + env.set "sid", sid + env.set "csrf_token", csrf_token + env.set "user", user + rescue ex + end + end + end + + dark_mode = convert_theme(env.params.query["dark_mode"]?) || preferences.dark_mode.to_s + thin_mode = env.params.query["thin_mode"]? || preferences.thin_mode.to_s + thin_mode = thin_mode == "true" + locale = env.params.query["hl"]? || preferences.locale + + preferences.dark_mode = dark_mode + preferences.thin_mode = thin_mode + preferences.locale = locale + env.set "preferences", preferences + + current_page = env.request.path + if env.request.query + query = HTTP::Params.parse(env.request.query.not_nil!) + + if query["referer"]? + query["referer"] = get_referer(env, "/") + end + + current_page += "?#{query}" + end + + env.set "current_page", URI.encode_www_form(current_page) + end +end diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 84da9993..289d87c9 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -2,11 +2,16 @@ module Invidious::Routes::Embed def self.redirect(env) + locale = env.get("preferences").as(Preferences).locale if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") begin playlist = get_playlist(plid) offset = env.params.query["index"]?.try &.to_i? || 0 videos = get_playlist_videos(playlist, offset: offset) + if videos.empty? + url = "/playlist?list=#{plid}" + raise NotFoundException.new(translate(locale, "error_video_not_in_playlist", url)) + end rescue ex : NotFoundException return error_template(404, ex) rescue ex @@ -26,6 +31,7 @@ module Invidious::Routes::Embed end def self.show(env) + locale = env.get("preferences").as(Preferences).locale id = env.params.url["id"] plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") @@ -62,6 +68,10 @@ module Invidious::Routes::Embed playlist = get_playlist(plid) offset = env.params.query["index"]?.try &.to_i? || 0 videos = get_playlist_videos(playlist, offset: offset) + if videos.empty? + url = "/playlist?list=#{plid}" + raise NotFoundException.new(translate(locale, "error_video_not_in_playlist", url)) + end rescue ex : NotFoundException return error_template(404, ex) rescue ex @@ -121,8 +131,6 @@ module Invidious::Routes::Embed begin video = get_video(id, region: params.region) - rescue ex : VideoRedirect - return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex : NotFoundException return error_template(404, ex) rescue ex diff --git a/src/invidious/routes/errors.cr b/src/invidious/routes/errors.cr new file mode 100644 index 00000000..b138b562 --- /dev/null +++ b/src/invidious/routes/errors.cr @@ -0,0 +1,47 @@ +module Invidious::Routes::ErrorRoutes + def self.error_404(env) + if md = env.request.path.match(/^\/(?<id>([a-zA-Z0-9_-]{11})|(\w+))$/) + item = md["id"] + + # Check if item is branding URL e.g. https://youtube.com/gaming + response = YT_POOL.client &.get("/#{item}") + + if response.status_code == 301 + response = YT_POOL.client &.get(URI.parse(response.headers["Location"]).request_target) + end + + if response.body.empty? + env.response.headers["Location"] = "/" + haltf env, status_code: 302 + end + + html = XML.parse_html(response.body) + ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1] + + if ucid + env.response.headers["Location"] = "/channel/#{ucid}" + haltf env, status_code: 302 + end + + params = [] of String + env.params.query.each do |k, v| + params << "#{k}=#{v}" + end + params = params.join("&") + + url = "/watch?v=#{item}" + if !params.empty? + url += "&#{params}" + end + + # Check if item is video ID + if item.match(/^[a-zA-Z0-9_-]{11}$/) && YT_POOL.client &.head("/watch?v=#{item}").status_code != 404 + env.response.headers["Location"] = url + haltf env, status_code: 302 + end + end + + env.response.headers["Location"] = "/" + haltf env, status_code: 302 + end +end diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index fe7e4e1c..0d242ee6 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -330,11 +330,11 @@ module Invidious::Routes::Playlists when "action_edit_playlist" # TODO: Playlist stub when "action_add_video" - if playlist.index.size >= 500 + if playlist.index.size >= CONFIG.playlist_length_limit if redirect - return error_template(400, "Playlist cannot have more than 500 videos") + return error_template(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos") else - return error_json(400, "Playlist cannot have more than 500 videos") + return error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos") end end diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index fe1d8e54..5f481557 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -61,8 +61,6 @@ module Invidious::Routes::Watch begin video = get_video(id, region: params.region) - rescue ex : VideoRedirect - return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex : NotFoundException LOGGER.error("get_video not found: #{id} : #{ex.message}") return error_template(404, ex) diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index bd72c577..f409f13c 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -1,130 +1,273 @@ module Invidious::Routing - {% for http_method in {"get", "post", "delete", "options", "patch", "put", "head"} %} + extend self + + {% for http_method in {"get", "post", "delete", "options", "patch", "put"} %} macro {{http_method.id}}(path, controller, method = :handle) - {{http_method.id}} \{{ path }} do |env| + unless Kemal::Utils.path_starts_with_slash?(\{{path}}) + raise Kemal::Exceptions::InvalidPathStartException.new({{http_method}}, \{{path}}) + end + + Kemal::RouteHandler::INSTANCE.add_route({{http_method.upcase}}, \{{path}}) do |env| \{{ controller }}.\{{ method.id }}(env) end end {% end %} -end -macro define_user_routes - # User login/out - Invidious::Routing.get "/login", Invidious::Routes::Login, :login_page - Invidious::Routing.post "/login", Invidious::Routes::Login, :login - Invidious::Routing.post "/signout", Invidious::Routes::Login, :signout - Invidious::Routing.get "/Captcha", Invidious::Routes::Login, :captcha - - # User preferences - Invidious::Routing.get "/preferences", Invidious::Routes::PreferencesRoute, :show - Invidious::Routing.post "/preferences", Invidious::Routes::PreferencesRoute, :update - Invidious::Routing.get "/toggle_theme", Invidious::Routes::PreferencesRoute, :toggle_theme - Invidious::Routing.get "/data_control", Invidious::Routes::PreferencesRoute, :data_control - Invidious::Routing.post "/data_control", Invidious::Routes::PreferencesRoute, :update_data_control - - # User account management - Invidious::Routing.get "/change_password", Invidious::Routes::Account, :get_change_password - Invidious::Routing.post "/change_password", Invidious::Routes::Account, :post_change_password - Invidious::Routing.get "/delete_account", Invidious::Routes::Account, :get_delete - Invidious::Routing.post "/delete_account", Invidious::Routes::Account, :post_delete - Invidious::Routing.get "/clear_watch_history", Invidious::Routes::Account, :get_clear_history - Invidious::Routing.post "/clear_watch_history", Invidious::Routes::Account, :post_clear_history - Invidious::Routing.get "/authorize_token", Invidious::Routes::Account, :get_authorize_token - Invidious::Routing.post "/authorize_token", Invidious::Routes::Account, :post_authorize_token - Invidious::Routing.get "/token_manager", Invidious::Routes::Account, :token_manager - Invidious::Routing.post "/token_ajax", Invidious::Routes::Account, :token_ajax -end + def register_all + {% unless flag?(:api_only) %} + get "/", Routes::Misc, :home + get "/privacy", Routes::Misc, :privacy + get "/licenses", Routes::Misc, :licenses + get "/redirect", Routes::Misc, :cross_instance_redirect -macro define_v1_api_routes - {{namespace = Invidious::Routes::API::V1}} - # Videos - Invidious::Routing.get "/api/v1/videos/:id", {{namespace}}::Videos, :videos - Invidious::Routing.get "/api/v1/storyboards/:id", {{namespace}}::Videos, :storyboards - Invidious::Routing.get "/api/v1/captions/:id", {{namespace}}::Videos, :captions - Invidious::Routing.get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations - Invidious::Routing.get "/api/v1/comments/:id", {{namespace}}::Videos, :comments - - # Feeds - Invidious::Routing.get "/api/v1/trending", {{namespace}}::Feeds, :trending - Invidious::Routing.get "/api/v1/popular", {{namespace}}::Feeds, :popular - - # Channels - Invidious::Routing.get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home - {% for route in {"videos", "latest", "playlists", "community", "search"} %} - Invidious::Routing.get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}} - Invidious::Routing.get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}} - {% end %} + self.register_channel_routes + self.register_watch_routes - # 301 redirects to new /api/v1/channels/community/:ucid and /:ucid/community - Invidious::Routing.get "/api/v1/channels/comments/:ucid", {{namespace}}::Channels, :channel_comments_redirect - Invidious::Routing.get "/api/v1/channels/:ucid/comments", {{namespace}}::Channels, :channel_comments_redirect + self.register_iv_playlist_routes + self.register_yt_playlist_routes + self.register_search_routes - # Search - Invidious::Routing.get "/api/v1/search", {{namespace}}::Search, :search - Invidious::Routing.get "/api/v1/search/suggestions", {{namespace}}::Search, :search_suggestions + self.register_user_routes + self.register_feed_routes - # Authenticated + # Support push notifications via PubSubHubbub + get "/feed/webhook/:token", Routes::Feeds, :push_notifications_get + post "/feed/webhook/:token", Routes::Feeds, :push_notifications_post - # The notification APIs cannot be extracted yet! They require the *local* notifications constant defined in invidious.cr - # - # Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications - # Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + get "/modify_notifications", Routes::Notifications, :modify + {% end %} - Invidious::Routing.get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences - Invidious::Routing.post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences + self.register_image_routes + self.register_api_v1_routes + self.register_api_manifest_routes + self.register_video_playback_routes + end - Invidious::Routing.get "/api/v1/auth/feed", {{namespace}}::Authenticated, :feed + # ------------------- + # Invidious routes + # ------------------- - Invidious::Routing.get "/api/v1/auth/subscriptions", {{namespace}}::Authenticated, :get_subscriptions - Invidious::Routing.post "/api/v1/auth/subscriptions/:ucid", {{namespace}}::Authenticated, :subscribe_channel - Invidious::Routing.delete "/api/v1/auth/subscriptions/:ucid", {{namespace}}::Authenticated, :unsubscribe_channel + def register_user_routes + # User login/out + get "/login", Routes::Login, :login_page + post "/login", Routes::Login, :login + post "/signout", Routes::Login, :signout + get "/Captcha", Routes::Login, :captcha + # User preferences + get "/preferences", Routes::PreferencesRoute, :show + post "/preferences", Routes::PreferencesRoute, :update + get "/toggle_theme", Routes::PreferencesRoute, :toggle_theme + get "/data_control", Routes::PreferencesRoute, :data_control + post "/data_control", Routes::PreferencesRoute, :update_data_control - Invidious::Routing.get "/api/v1/auth/playlists", {{namespace}}::Authenticated, :list_playlists - Invidious::Routing.post "/api/v1/auth/playlists", {{namespace}}::Authenticated, :create_playlist - Invidious::Routing.patch "/api/v1/auth/playlists/:plid",{{namespace}}:: Authenticated, :update_playlist_attribute - Invidious::Routing.delete "/api/v1/auth/playlists/:plid", {{namespace}}::Authenticated, :delete_playlist + # User account management + get "/change_password", Routes::Account, :get_change_password + post "/change_password", Routes::Account, :post_change_password + get "/delete_account", Routes::Account, :get_delete + post "/delete_account", Routes::Account, :post_delete + get "/clear_watch_history", Routes::Account, :get_clear_history + post "/clear_watch_history", Routes::Account, :post_clear_history + get "/authorize_token", Routes::Account, :get_authorize_token + post "/authorize_token", Routes::Account, :post_authorize_token + get "/token_manager", Routes::Account, :token_manager + post "/token_ajax", Routes::Account, :token_ajax + post "/subscription_ajax", Routes::Subscriptions, :toggle_subscription + get "/subscription_manager", Routes::Subscriptions, :subscription_manager + end + def register_iv_playlist_routes + get "/create_playlist", Routes::Playlists, :new + post "/create_playlist", Routes::Playlists, :create + get "/subscribe_playlist", Routes::Playlists, :subscribe + get "/delete_playlist", Routes::Playlists, :delete_page + post "/delete_playlist", Routes::Playlists, :delete + get "/edit_playlist", Routes::Playlists, :edit + post "/edit_playlist", Routes::Playlists, :update + get "/add_playlist_items", Routes::Playlists, :add_playlist_items_page + post "/playlist_ajax", Routes::Playlists, :playlist_ajax + end - Invidious::Routing.post "/api/v1/auth/playlists/:plid/videos", {{namespace}}::Authenticated, :insert_video_into_playlist - Invidious::Routing.delete "/api/v1/auth/playlists/:plid/videos/:index", {{namespace}}::Authenticated, :delete_video_in_playlist + def register_feed_routes + # Feeds + get "/view_all_playlists", Routes::Feeds, :view_all_playlists_redirect + get "/feed/playlists", Routes::Feeds, :playlists + get "/feed/popular", Routes::Feeds, :popular + get "/feed/trending", Routes::Feeds, :trending + get "/feed/subscriptions", Routes::Feeds, :subscriptions + get "/feed/history", Routes::Feeds, :history - Invidious::Routing.get "/api/v1/auth/tokens", {{namespace}}::Authenticated, :get_tokens - Invidious::Routing.post "/api/v1/auth/tokens/register", {{namespace}}::Authenticated, :register_token - Invidious::Routing.post "/api/v1/auth/tokens/unregister", {{namespace}}::Authenticated, :unregister_token + # RSS Feeds + get "/feed/channel/:ucid", Routes::Feeds, :rss_channel + get "/feed/private", Routes::Feeds, :rss_private + get "/feed/playlist/:plid", Routes::Feeds, :rss_playlist + get "/feeds/videos.xml", Routes::Feeds, :rss_videos + end - Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications - Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + # ------------------- + # Youtube routes + # ------------------- - # Misc - Invidious::Routing.get "/api/v1/stats", {{namespace}}::Misc, :stats - Invidious::Routing.get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist - Invidious::Routing.get "/api/v1/auth/playlists/:plid", {{namespace}}::Misc, :get_playlist - Invidious::Routing.get "/api/v1/mixes/:rdid", {{namespace}}::Misc, :mixes -end + def register_channel_routes + get "/channel/:ucid", Routes::Channels, :home + get "/channel/:ucid/home", Routes::Channels, :home + get "/channel/:ucid/videos", Routes::Channels, :videos + get "/channel/:ucid/playlists", Routes::Channels, :playlists + get "/channel/:ucid/community", Routes::Channels, :community + get "/channel/:ucid/about", Routes::Channels, :about + get "/channel/:ucid/live", Routes::Channels, :live + get "/user/:user/live", Routes::Channels, :live + get "/c/:user/live", Routes::Channels, :live -macro define_api_manifest_routes - Invidious::Routing.get "/api/manifest/dash/id/:id", Invidious::Routes::API::Manifest, :get_dash_video_id + ["", "/videos", "/playlists", "/community", "/about"].each do |path| + # /c/LinusTechTips + get "/c/:user#{path}", Routes::Channels, :brand_redirect + # /user/linustechtips | Not always the same as /c/ + get "/user/:user#{path}", Routes::Channels, :brand_redirect + # /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow + get "/attribution_link#{path}", Routes::Channels, :brand_redirect + # /profile?user=linustechtips + get "/profile/#{path}", Routes::Channels, :profile + end + end - Invidious::Routing.get "/api/manifest/dash/id/videoplayback", Invidious::Routes::API::Manifest, :get_dash_video_playback - Invidious::Routing.get "/api/manifest/dash/id/videoplayback/*", Invidious::Routes::API::Manifest, :get_dash_video_playback_greedy + def register_watch_routes + get "/watch", Routes::Watch, :handle + post "/watch_ajax", Routes::Watch, :mark_watched + get "/watch/:id", Routes::Watch, :redirect + get "/shorts/:id", Routes::Watch, :redirect + get "/clip/:clip", Routes::Watch, :clip + get "/w/:id", Routes::Watch, :redirect + get "/v/:id", Routes::Watch, :redirect + get "/e/:id", Routes::Watch, :redirect - Invidious::Routing.options "/api/manifest/dash/id/videoplayback", Invidious::Routes::API::Manifest, :options_dash_video_playback - Invidious::Routing.options "/api/manifest/dash/id/videoplayback/*", Invidious::Routes::API::Manifest, :options_dash_video_playback + post "/download", Routes::Watch, :download - Invidious::Routing.get "/api/manifest/hls_playlist/*", Invidious::Routes::API::Manifest, :get_hls_playlist - Invidious::Routing.get "/api/manifest/hls_variant/*", Invidious::Routes::API::Manifest, :get_hls_variant -end + get "/embed/", Routes::Embed, :redirect + get "/embed/:id", Routes::Embed, :show + end + + def register_yt_playlist_routes + get "/playlist", Routes::Playlists, :show + get "/mix", Routes::Playlists, :mix + get "/watch_videos", Routes::Playlists, :watch_videos + end + + def register_search_routes + get "/opensearch.xml", Routes::Search, :opensearch + get "/results", Routes::Search, :results + get "/search", Routes::Search, :search + get "/hashtag/:hashtag", Routes::Search, :hashtag + end + + # ------------------- + # Media proxy routes + # ------------------- + + def register_api_manifest_routes + get "/api/manifest/dash/id/:id", Routes::API::Manifest, :get_dash_video_id + + get "/api/manifest/dash/id/videoplayback", Routes::API::Manifest, :get_dash_video_playback + get "/api/manifest/dash/id/videoplayback/*", Routes::API::Manifest, :get_dash_video_playback_greedy + + options "/api/manifest/dash/id/videoplayback", Routes::API::Manifest, :options_dash_video_playback + options "/api/manifest/dash/id/videoplayback/*", Routes::API::Manifest, :options_dash_video_playback + + get "/api/manifest/hls_playlist/*", Routes::API::Manifest, :get_hls_playlist + get "/api/manifest/hls_variant/*", Routes::API::Manifest, :get_hls_variant + end + + def register_video_playback_routes + get "/videoplayback", Routes::VideoPlayback, :get_video_playback + get "/videoplayback/*", Routes::VideoPlayback, :get_video_playback_greedy + + options "/videoplayback", Routes::VideoPlayback, :options_video_playback + options "/videoplayback/*", Routes::VideoPlayback, :options_video_playback + + get "/latest_version", Routes::VideoPlayback, :latest_version + end + + def register_image_routes + get "/ggpht/*", Routes::Images, :ggpht + options "/sb/:authority/:id/:storyboard/:index", Routes::Images, :options_storyboard + get "/sb/:authority/:id/:storyboard/:index", Routes::Images, :get_storyboard + get "/s_p/:id/:name", Routes::Images, :s_p_image + get "/yts/img/:name", Routes::Images, :yts_image + get "/vi/:id/:name", Routes::Images, :thumbnails + end + + # ------------------- + # API routes + # ------------------- + + def register_api_v1_routes + {% begin %} + {{namespace = Routes::API::V1}} + + # Videos + get "/api/v1/videos/:id", {{namespace}}::Videos, :videos + get "/api/v1/storyboards/:id", {{namespace}}::Videos, :storyboards + get "/api/v1/captions/:id", {{namespace}}::Videos, :captions + get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations + get "/api/v1/comments/:id", {{namespace}}::Videos, :comments + + # Feeds + get "/api/v1/trending", {{namespace}}::Feeds, :trending + get "/api/v1/popular", {{namespace}}::Feeds, :popular + + # Channels + get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home + {% for route in {"videos", "latest", "playlists", "community", "search"} %} + get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}} + get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}} + {% end %} + + # 301 redirects to new /api/v1/channels/community/:ucid and /:ucid/community + get "/api/v1/channels/comments/:ucid", {{namespace}}::Channels, :channel_comments_redirect + get "/api/v1/channels/:ucid/comments", {{namespace}}::Channels, :channel_comments_redirect + + # Search + get "/api/v1/search", {{namespace}}::Search, :search + get "/api/v1/search/suggestions", {{namespace}}::Search, :search_suggestions + + # Authenticated + + # The notification APIs cannot be extracted yet! They require the *local* notifications constant defined in invidious.cr + # + # Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + # Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + + get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences + post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences + + get "/api/v1/auth/feed", {{namespace}}::Authenticated, :feed + + get "/api/v1/auth/subscriptions", {{namespace}}::Authenticated, :get_subscriptions + post "/api/v1/auth/subscriptions/:ucid", {{namespace}}::Authenticated, :subscribe_channel + delete "/api/v1/auth/subscriptions/:ucid", {{namespace}}::Authenticated, :unsubscribe_channel + + get "/api/v1/auth/playlists", {{namespace}}::Authenticated, :list_playlists + post "/api/v1/auth/playlists", {{namespace}}::Authenticated, :create_playlist + patch "/api/v1/auth/playlists/:plid",{{namespace}}:: Authenticated, :update_playlist_attribute + delete "/api/v1/auth/playlists/:plid", {{namespace}}::Authenticated, :delete_playlist + post "/api/v1/auth/playlists/:plid/videos", {{namespace}}::Authenticated, :insert_video_into_playlist + delete "/api/v1/auth/playlists/:plid/videos/:index", {{namespace}}::Authenticated, :delete_video_in_playlist -macro define_video_playback_routes - Invidious::Routing.get "/videoplayback", Invidious::Routes::VideoPlayback, :get_video_playback - Invidious::Routing.get "/videoplayback/*", Invidious::Routes::VideoPlayback, :get_video_playback_greedy + get "/api/v1/auth/tokens", {{namespace}}::Authenticated, :get_tokens + post "/api/v1/auth/tokens/register", {{namespace}}::Authenticated, :register_token + post "/api/v1/auth/tokens/unregister", {{namespace}}::Authenticated, :unregister_token - Invidious::Routing.options "/videoplayback", Invidious::Routes::VideoPlayback, :options_video_playback - Invidious::Routing.options "/videoplayback/*", Invidious::Routes::VideoPlayback, :options_video_playback + get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications - Invidious::Routing.get "/latest_version", Invidious::Routes::VideoPlayback, :latest_version + # Misc + get "/api/v1/stats", {{namespace}}::Misc, :stats + get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist + get "/api/v1/auth/playlists/:plid", {{namespace}}::Misc, :get_playlist + get "/api/v1/mixes/:rdid", {{namespace}}::Misc, :mixes + {% end %} + end end diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index f8b9e4e4..20ae0d47 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -71,7 +71,9 @@ struct Invidious::User Invidious::Database::Playlists.update_description(playlist.id, description) videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| - raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500 + if idx > CONFIG.playlist_length_limit + raise InfoException.new("Playlist cannot have more than #{CONFIG.playlist_length_limit} videos") + end video_id = video_id.try &.as_s? next if !video_id diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index e9526c18..d626c7d1 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -1,280 +1,22 @@ -CAPTION_LANGUAGES = { - "", - "English", - "English (auto-generated)", - "English (United Kingdom)", - "English (United States)", - "Afrikaans", - "Albanian", - "Amharic", - "Arabic", - "Armenian", - "Azerbaijani", - "Bangla", - "Basque", - "Belarusian", - "Bosnian", - "Bulgarian", - "Burmese", - "Cantonese (Hong Kong)", - "Catalan", - "Cebuano", - "Chinese", - "Chinese (China)", - "Chinese (Hong Kong)", - "Chinese (Simplified)", - "Chinese (Taiwan)", - "Chinese (Traditional)", - "Corsican", - "Croatian", - "Czech", - "Danish", - "Dutch", - "Dutch (auto-generated)", - "Esperanto", - "Estonian", - "Filipino", - "Finnish", - "French", - "French (auto-generated)", - "Galician", - "Georgian", - "German", - "German (auto-generated)", - "Greek", - "Gujarati", - "Haitian Creole", - "Hausa", - "Hawaiian", - "Hebrew", - "Hindi", - "Hmong", - "Hungarian", - "Icelandic", - "Igbo", - "Indonesian", - "Indonesian (auto-generated)", - "Interlingue", - "Irish", - "Italian", - "Italian (auto-generated)", - "Japanese", - "Japanese (auto-generated)", - "Javanese", - "Kannada", - "Kazakh", - "Khmer", - "Korean", - "Korean (auto-generated)", - "Kurdish", - "Kyrgyz", - "Lao", - "Latin", - "Latvian", - "Lithuanian", - "Luxembourgish", - "Macedonian", - "Malagasy", - "Malay", - "Malayalam", - "Maltese", - "Maori", - "Marathi", - "Mongolian", - "Nepali", - "Norwegian Bokmål", - "Nyanja", - "Pashto", - "Persian", - "Polish", - "Portuguese", - "Portuguese (auto-generated)", - "Portuguese (Brazil)", - "Punjabi", - "Romanian", - "Russian", - "Russian (auto-generated)", - "Samoan", - "Scottish Gaelic", - "Serbian", - "Shona", - "Sindhi", - "Sinhala", - "Slovak", - "Slovenian", - "Somali", - "Southern Sotho", - "Spanish", - "Spanish (auto-generated)", - "Spanish (Latin America)", - "Spanish (Mexico)", - "Spanish (Spain)", - "Sundanese", - "Swahili", - "Swedish", - "Tajik", - "Tamil", - "Telugu", - "Thai", - "Turkish", - "Turkish (auto-generated)", - "Ukrainian", - "Urdu", - "Uzbek", - "Vietnamese", - "Vietnamese (auto-generated)", - "Welsh", - "Western Frisian", - "Xhosa", - "Yiddish", - "Yoruba", - "Zulu", -} - -REGIONS = {"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW"} - -# See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476 -VIDEO_FORMATS = { - "5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, - "6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, - "13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"}, - "17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"}, - "18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"}, - "22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, - "34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - "35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - - "36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"}, - "37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, - "38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, - "43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, - "44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, - "45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, - "46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, - "59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - "78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - - # 3D videos - "82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - "83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - "84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, - "85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, - "100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, - "101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, - "102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, - - # Apple HTTP Live Streaming - "91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, - "92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, - "93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - "94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - "95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, - "96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, - "132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, - "151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"}, - - # DASH mp4 video - "133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"}, - "134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"}, - "135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, - "136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"}, - "137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"}, - "138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559) - "160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"}, - "212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, - "264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"}, - "298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, - "299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, - "266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"}, - - # Dash mp4 audio - "139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"}, - "140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"}, - "141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"}, - "256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, - "258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, - "325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"}, - "328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"}, - - # Dash webm - "167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, - "168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, - "169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, - "170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, - "218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, - "219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, - "278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"}, - "242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"}, - "243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"}, - "244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, - "245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, - "246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, - "247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"}, - "248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"}, - "271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"}, - # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug) - "272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, - "302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, - "315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - - # Dash webm audio - "171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128}, - "172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256}, - - # Dash webm audio with opus inside - "249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50}, - "250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70}, - "251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160}, - - # av01 video only formats sometimes served with "unknown" codecs - "394" => {"ext" => "mp4", "height" => 144, "vcodec" => "av01.0.05M.08"}, - "395" => {"ext" => "mp4", "height" => 240, "vcodec" => "av01.0.05M.08"}, - "396" => {"ext" => "mp4", "height" => 360, "vcodec" => "av01.0.05M.08"}, - "397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"}, -} - -struct VideoPreferences - include JSON::Serializable - - property annotations : Bool - property autoplay : Bool - property comments : Array(String) - property continue : Bool - property continue_autoplay : Bool - property controls : Bool - property listen : Bool - property local : Bool - property preferred_captions : Array(String) - property player_style : String - property quality : String - property quality_dash : String - property raw : Bool - property region : String? - property related_videos : Bool - property speed : Float32 | Float64 - property video_end : Float64 | Int32 - property video_loop : Bool - property extend_desc : Bool - property video_start : Float64 | Int32 - property volume : Int32 - property vr_mode : Bool - property save_player_pos : Bool +enum VideoType + Video + Livestream + Scheduled end struct Video include DB::Serializable + # Version of the JSON structure + # It prevents us from loading an incompatible version from cache + # (either newer or older, if instances with different versions run + # concurrently, e.g during a version upgrade rollout). + # + # NOTE: don't forget to bump this number if any change is made to + # the `params` structure in videos/parser.cr!!! + # + SCHEMA_VERSION = 2 + property id : String @[DB::Field(converter: Video::JSONConverter)] @@ -282,7 +24,7 @@ struct Video property updated : Time @[DB::Field(ignore: true)] - property captions : Array(Caption)? + @captions = [] of Invidious::Videos::Caption @[DB::Field(ignore: true)] property adaptive_fmts : Array(Hash(String, JSON::Any))? @@ -299,289 +41,45 @@ struct Video end end - def to_json(locale : String?, json : JSON::Builder) - json.object do - json.field "type", "video" - - json.field "title", self.title - json.field "videoId", self.id - - json.field "error", info["reason"] if info["reason"]? - - json.field "videoThumbnails" do - generate_thumbnails(json, self.id) - end - json.field "storyboards" do - generate_storyboards(json, self.id, self.storyboards) - end - - json.field "description", self.description - json.field "descriptionHtml", self.description_html - json.field "published", self.published.to_unix - json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) - json.field "keywords", self.keywords - - json.field "viewCount", self.views - json.field "likeCount", self.likes - json.field "dislikeCount", 0_i64 - - json.field "paid", self.paid - json.field "premium", self.premium - json.field "isFamilyFriendly", self.is_family_friendly - json.field "allowedRegions", self.allowed_regions - json.field "genre", self.genre - json.field "genreUrl", self.genre_url - - json.field "author", self.author - json.field "authorId", self.ucid - json.field "authorUrl", "/channel/#{self.ucid}" - - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - - qualities.each do |quality| - json.object do - json.field "url", self.author_thumbnail.gsub(/=s\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", quality - end - end - end - end - - json.field "subCountText", self.sub_count_text - - json.field "lengthSeconds", self.length_seconds - json.field "allowRatings", self.allow_ratings - json.field "rating", 0_i64 - json.field "isListed", self.is_listed - json.field "liveNow", self.live_now - json.field "isUpcoming", self.is_upcoming - - if self.premiere_timestamp - json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix - end - - if hlsvp = self.hls_manifest_url - hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL) - json.field "hlsUrl", hlsvp - end - - json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{id}" - - json.field "adaptiveFormats" do - json.array do - self.adaptive_fmts.each do |fmt| - json.object do - # Only available on regular videos, not livestreams/OTF streams - if init_range = fmt["initRange"]? - json.field "init", "#{init_range["start"]}-#{init_range["end"]}" - end - if index_range = fmt["indexRange"]? - json.field "index", "#{index_range["start"]}-#{index_range["end"]}" - end - - # Not available on MPEG-4 Timed Text (`text/mp4`) streams (livestreams only) - json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]? - - json.field "url", fmt["url"] - json.field "itag", fmt["itag"].as_i.to_s - json.field "type", fmt["mimeType"] - json.field "clen", fmt["contentLength"]? || "-1" - json.field "lmt", fmt["lastModified"] - json.field "projectionType", fmt["projectionType"] - - if fmt_info = itag_to_metadata?(fmt["itag"]) - fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 - json.field "fps", fps - json.field "container", fmt_info["ext"] - json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] - - if fmt_info["height"]? - json.field "resolution", "#{fmt_info["height"]}p" - - quality_label = "#{fmt_info["height"]}p" - if fps > 30 - quality_label += "60" - end - json.field "qualityLabel", quality_label - - if fmt_info["width"]? - json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" - end - end - end - - # Livestream chunk infos - json.field "targetDurationSec", fmt["targetDurationSec"].as_i if fmt.has_key?("targetDurationSec") - json.field "maxDvrDurationSec", fmt["maxDvrDurationSec"].as_i if fmt.has_key?("maxDvrDurationSec") - - # Audio-related data - json.field "audioQuality", fmt["audioQuality"] if fmt.has_key?("audioQuality") - json.field "audioSampleRate", fmt["audioSampleRate"].as_s.to_i if fmt.has_key?("audioSampleRate") - json.field "audioChannels", fmt["audioChannels"] if fmt.has_key?("audioChannels") - - # Extra misc stuff - json.field "colorInfo", fmt["colorInfo"] if fmt.has_key?("colorInfo") - json.field "captionTrack", fmt["captionTrack"] if fmt.has_key?("captionTrack") - end - end - end - end - - json.field "formatStreams" do - json.array do - self.fmt_stream.each do |fmt| - json.object do - json.field "url", fmt["url"] - json.field "itag", fmt["itag"].as_i.to_s - json.field "type", fmt["mimeType"] - json.field "quality", fmt["quality"] - - fmt_info = itag_to_metadata?(fmt["itag"]) - if fmt_info - fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 - json.field "fps", fps - json.field "container", fmt_info["ext"] - json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] - - if fmt_info["height"]? - json.field "resolution", "#{fmt_info["height"]}p" - - quality_label = "#{fmt_info["height"]}p" - if fps > 30 - quality_label += "60" - end - json.field "qualityLabel", quality_label - - if fmt_info["width"]? - json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" - end - end - end - end - end - end - end - - json.field "captions" do - json.array do - self.captions.each do |caption| - json.object do - json.field "label", caption.name - json.field "language_code", caption.language_code - json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}" - end - end - end - end + # Methods for API v1 JSON - json.field "recommendedVideos" do - json.array do - self.related_videos.each do |rv| - if rv["id"]? - json.object do - json.field "videoId", rv["id"] - json.field "title", rv["title"] - json.field "videoThumbnails" do - generate_thumbnails(json, rv["id"]) - end - - json.field "author", rv["author"] - json.field "authorUrl", "/channel/#{rv["ucid"]?}" - json.field "authorId", rv["ucid"]? - if rv["author_thumbnail"]? - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - - qualities.each do |quality| - json.object do - json.field "url", rv["author_thumbnail"].gsub(/s\d+-/, "s#{quality}-") - json.field "width", quality - json.field "height", quality - end - end - end - end - end - - json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i - json.field "viewCountText", rv["short_view_count"]? - json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 - end - end - end - end - end - end + def to_json(locale : String?, json : JSON::Builder) + Invidious::JSONify::APIv1.video(self, json, locale: locale) end # TODO: remove the locale and follow the crystal convention def to_json(locale : String?, _json : Nil) - JSON.build { |json| to_json(locale, json) } + JSON.build do |json| + Invidious::JSONify::APIv1.video(self, json, locale: locale) + end end def to_json(json : JSON::Builder | Nil = nil) to_json(nil, json) end - def title - info["videoDetails"]["title"]?.try &.as_s || "" - end + # Misc methods - def ucid - info["videoDetails"]["channelId"]?.try &.as_s || "" + def video_type : VideoType + video_type = info["videoType"]?.try &.as_s || "video" + return VideoType.parse?(video_type) || VideoType::Video end - def author - info["videoDetails"]["author"]?.try &.as_s || "" - end - - def length_seconds : Int32 - info.dig?("microformat", "playerMicroformatRenderer", "lengthSeconds").try &.as_s.to_i || - info["videoDetails"]["lengthSeconds"]?.try &.as_s.to_i || 0 - end - - def views : Int64 - info["videoDetails"]["viewCount"]?.try &.as_s.to_i64 || 0_i64 - end - - def likes : Int64 - info["likes"]?.try &.as_i64 || 0_i64 - end - - def dislikes : Int64 - info["dislikes"]?.try &.as_i64 || 0_i64 + def schema_version : Int + return info["version"]?.try &.as_i || 1 end def published : Time - info - .dig?("microformat", "playerMicroformatRenderer", "publishDate") + return info["published"]? .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc end def published=(other : Time) - info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d")) - end - - def allow_ratings - r = info["videoDetails"]["allowRatings"]?.try &.as_bool - r.nil? ? false : r + info["published"] = JSON::Any.new(other.to_s("%Y-%m-%d")) end def live_now - info["microformat"]?.try &.["playerMicroformatRenderer"]? - .try &.["liveBroadcastDetails"]?.try &.["isLiveNow"]?.try &.as_bool || false - end - - def is_listed - info["videoDetails"]["isCrawlable"]?.try &.as_bool || false - end - - def is_upcoming - info["videoDetails"]["isUpcoming"]?.try &.as_bool || false + return (self.video_type == VideoType::Livestream) end def premiere_timestamp : Time? @@ -590,31 +88,11 @@ struct Video .try { |t| Time.parse_rfc3339(t.as_s) } end - def keywords - info["videoDetails"]["keywords"]?.try &.as_a.map &.as_s || [] of String - end - def related_videos info["relatedVideos"]?.try &.as_a.map { |h| h.as_h.transform_values &.as_s } || [] of Hash(String, String) end - def allowed_regions - info - .dig?("microformat", "playerMicroformatRenderer", "availableCountries") - .try &.as_a.map &.as_s || [] of String - end - - def author_thumbnail : String - info["authorThumbnail"]?.try &.as_s || "" - end - - def author_verified : Bool - info["authorVerified"]?.try &.as_bool || false - end - - def sub_count_text : String - info["subCountText"]?.try &.as_s || "-" - end + # Methods for parsing streaming data def fmt_stream return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream @@ -665,6 +143,8 @@ struct Video adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("audio") end + # Misc. methods + def storyboards storyboards = info.dig?("storyboards", "playerStoryboardSpecRenderer", "spec") .try &.as_s.split("|") @@ -728,51 +208,19 @@ struct Video end def paid - reason = info.dig?("playabilityStatus", "reason").try &.as_s || "" - return reason.includes? "requires payment" + return (self.reason || "").includes? "requires payment" end def premium keywords.includes? "YouTube Red" end - def captions : Array(Caption) - return @captions.as(Array(Caption)) if @captions - captions = info["captions"]?.try &.["playerCaptionsTracklistRenderer"]?.try &.["captionTracks"]?.try &.as_a.map do |caption| - name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"] - language_code = caption["languageCode"].to_s - base_url = caption["baseUrl"].to_s - - caption = Caption.new(name.to_s, language_code, base_url) - caption.name = caption.name.split(" - ")[0] - caption + def captions : Array(Invidious::Videos::Caption) + if @captions.empty? && @info.has_key?("captions") + @captions = Invidious::Videos::Caption.from_yt_json(info["captions"]) end - captions ||= [] of Caption - @captions = captions - return @captions.as(Array(Caption)) - end - - def description - description = info - .dig?("microformat", "playerMicroformatRenderer", "description", "simpleText") - .try &.as_s || "" - end - - # TODO - def description=(value : String) - @description = value - end - def description_html - info["descriptionHtml"]?.try &.as_s || "<p></p>" - end - - def description_html=(value : String) - info["descriptionHtml"] = JSON::Any.new(value) - end - - def short_description - info["shortDescription"]?.try &.as_s? || "" + return @captions end def hls_manifest_url : String? @@ -783,25 +231,12 @@ struct Video info.dig?("streamingData", "dashManifestUrl").try &.as_s end - def genre : String - info["genre"]?.try &.as_s || "" - end - def genre_url : String? info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil end - def license : String? - info["license"]?.try &.as_s - end - - def is_family_friendly : Bool - info.dig?("microformat", "playerMicroformatRenderer", "isFamilySafe").try &.as_bool || false - end - def is_vr : Bool? - projection_type = info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s - return {"EQUIRECTANGULAR", "MESH"}.includes? projection_type + return {"EQUIRECTANGULAR", "MESH"}.includes? self.projection_type end def projection_type : String? @@ -811,284 +246,91 @@ struct Video def reason : String? info["reason"]?.try &.as_s end -end - -struct Caption - property name - property language_code - property base_url - - getter name : String - getter language_code : String - getter base_url : String - - setter name - - def initialize(@name, @language_code, @base_url) - end -end - -class VideoRedirect < Exception - property video_id : String - - def initialize(@video_id) - end -end - -# Use to parse both "compactVideoRenderer" and "endScreenVideoRenderer". -# The former is preferred as it has more videos in it. The second has -# the same 11 first entries as the compact rendered. -# -# TODO: "compactRadioRenderer" (Mix) and -# TODO: Use a proper struct/class instead of a hacky JSON object -def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? - return nil if !related["videoId"]? - - # The compact renderer has video length in seconds, where the end - # screen rendered has a full text version ("42:40") - length = related["lengthInSeconds"]?.try &.as_i.to_s - length ||= related.dig?("lengthText", "simpleText").try do |box| - decode_length_seconds(box.as_s).to_s - end - - # Both have "short", so the "long" option shouldn't be required - channel_info = (related["shortBylineText"]? || related["longBylineText"]?) - .try &.dig?("runs", 0) - author = channel_info.try &.dig?("text") - author_verified = has_verified_badge?(related["ownerBadges"]?).to_s + # Macros defining getters/setters for various types of data - ucid = channel_info.try { |ci| HelperExtractors.get_browse_id(ci) } - - # "4,088,033 views", only available on compact renderer - # and when video is not a livestream - view_count = related.dig?("viewCountText", "simpleText") - .try &.as_s.gsub(/\D/, "") - - short_view_count = related.try do |r| - HelperExtractors.get_short_view_count(r).to_s - end - - LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container") - - # TODO: when refactoring video types, make a struct for related videos - # or reuse an existing type, if that fits. - return { - "id" => related["videoId"], - "title" => related["title"]["simpleText"], - "author" => author || JSON::Any.new(""), - "ucid" => JSON::Any.new(ucid || ""), - "length_seconds" => JSON::Any.new(length || "0"), - "view_count" => JSON::Any.new(view_count || "0"), - "short_view_count" => JSON::Any.new(short_view_count || "0"), - "author_verified" => JSON::Any.new(author_verified), - } -end - -def extract_video_info(video_id : String, proxy_region : String? = nil, context_screen : String? = nil) - # Init client config for the API - client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region) - if context_screen == "embed" - client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed - end - - # Fetch data from the player endpoint - # 8AEB param for fetching YouTube stories - player_response = YoutubeAPI.player(video_id: video_id, params: "8AEB", client_config: client_config) - - playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s - - if playability_status != "OK" - subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason") - reason = subreason.try &.[]?("simpleText").try &.as_s - reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("") - reason ||= player_response.dig("playabilityStatus", "reason").as_s + private macro getset_string(name) + # Return {{name.stringify}} from `info` + def {{name.id.underscore}} : String + return info[{{name.stringify}}]?.try &.as_s || "" + end - # Stop here if video is not a scheduled livestream - if playability_status != "LIVE_STREAM_OFFLINE" - return { - "reason" => JSON::Any.new(reason), - } + # Update {{name.stringify}} into `info` + def {{name.id.underscore}}=(value : String) + info[{{name.stringify}}] = JSON::Any.new(value) end - else - reason = nil - end - # Don't fetch the next endpoint if the video is unavailable. - if {"OK", "LIVE_STREAM_OFFLINE"}.any?(playability_status) - next_response = YoutubeAPI.next({"videoId": video_id, "params": ""}) - player_response = player_response.merge(next_response) + {% if flag?(:debug_macros) %} {{debug}} {% end %} end - params = parse_video_info(video_id, player_response) - params["reason"] = JSON::Any.new(reason) if reason - - # Fetch the video streams using an Android client in order to get the decrypted URLs and - # maybe fix throttling issues (#2194).See for the explanation about the decrypted URLs: - # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 - if reason.nil? - if context_screen == "embed" - client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed - else - client_config.client_type = YoutubeAPI::ClientType::Android + private macro getset_string_array(name) + # Return {{name.stringify}} from `info` + def {{name.id.underscore}} : Array(String) + return info[{{name.stringify}}]?.try &.as_a.map &.as_s || [] of String end - # 8AEB param for fetching YouTube stories - android_player = YoutubeAPI.player(video_id: video_id, params: "8AEB", client_config: client_config) - - # Sometime, the video is available from the web client, but not on Android, so check - # that here, and fallback to the streaming data from the web client if needed. - # See: https://github.com/iv-org/invidious/issues/2549 - if android_player["playabilityStatus"]["status"] == "OK" - params["streamingData"] = android_player["streamingData"]? || JSON::Any.new("") - else - params["streamingData"] = player_response["streamingData"]? || JSON::Any.new("") + + # Update {{name.stringify}} into `info` + def {{name.id.underscore}}=(value : Array(String)) + info[{{name.stringify}}] = JSON::Any.new(value) end - end - # TODO: clean that up - {"captions", "microformat", "playabilityStatus", "storyboards", "videoDetails"}.each do |f| - params[f] = player_response[f] if player_response[f]? + {% if flag?(:debug_macros) %} {{debug}} {% end %} end - return params -end - -def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any) - # Top level elements - - main_results = player_response.dig?("contents", "twoColumnWatchNextResults") - - raise BrokenTubeException.new("twoColumnWatchNextResults") if !main_results - - primary_results = main_results.dig?("results", "results", "contents") - - raise BrokenTubeException.new("results") if !primary_results - - video_primary_renderer = primary_results - .as_a.find(&.["videoPrimaryInfoRenderer"]?) - .try &.["videoPrimaryInfoRenderer"] - - video_secondary_renderer = primary_results - .as_a.find(&.["videoSecondaryInfoRenderer"]?) - .try &.["videoSecondaryInfoRenderer"] - - raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer - raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer - - # Related videos - - LOGGER.debug("extract_video_info: parsing related videos...") + {% for op, type in {i32: Int32, i64: Int64} %} + private macro getset_{{op}}(name) + def \{{name.id.underscore}} : {{type}} + return info[\{{name.stringify}}]?.try &.as_i64.to_{{op}} || 0_{{op}} + end - related = [] of JSON::Any + def \{{name.id.underscore}}=(value : Int) + info[\{{name.stringify}}] = JSON::Any.new(value.to_i64) + end - # Parse "compactVideoRenderer" items (under secondary results) - secondary_results = main_results - .dig?("secondaryResults", "secondaryResults", "results") - secondary_results.try &.as_a.each do |element| - if item = element["compactVideoRenderer"]? - related_video = parse_related_video(item) - related << JSON::Any.new(related_video) if related_video + \{% if flag?(:debug_macros) %} \{{debug}} \{% end %} end - end + {% end %} - # If nothing was found previously, fall back to end screen renderer - if related.empty? - # Container for "endScreenVideoRenderer" items - player_overlays = player_response.dig?( - "playerOverlays", "playerOverlayRenderer", - "endScreen", "watchNextEndScreenRenderer", "results" - ) - - player_overlays.try &.as_a.each do |element| - if item = element["endScreenVideoRenderer"]? - related_video = parse_related_video(item) - related << JSON::Any.new(related_video) if related_video - end + private macro getset_bool(name) + # Return {{name.stringify}} from `info` + def {{name.id.underscore}} : Bool + return info[{{name.stringify}}]?.try &.as_bool || false end - end - - # Likes - toplevel_buttons = video_primary_renderer - .try &.dig?("videoActions", "menuRenderer", "topLevelButtons") - - if toplevel_buttons - likes_button = toplevel_buttons.as_a - .find(&.dig("toggleButtonRenderer", "defaultIcon", "iconType").as_s.== "LIKE") - .try &.["toggleButtonRenderer"] - - if likes_button - likes_txt = (likes_button["defaultText"]? || likes_button["toggledText"]?) - .try &.dig?("accessibility", "accessibilityData", "label") - likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt - - LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"") - LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes + # Update {{name.stringify}} into `info` + def {{name.id.underscore}}=(value : Bool) + info[{{name.stringify}}] = JSON::Any.new(value) end - end - - # Description - - short_description = player_response.dig?("videoDetails", "shortDescription") - - description_html = video_secondary_renderer.try &.dig?("description", "runs") - .try &.as_a.try { |t| content_to_comment_html(t, video_id) } - - # Video metadata - - metadata = video_secondary_renderer - .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows") - .try &.as_a - - genre = player_response.dig?("microformat", "playerMicroformatRenderer", "category") - genre_ucid = nil - license = nil - - metadata.try &.each do |row| - metadata_title = row.dig?("metadataRowRenderer", "title", "simpleText").try &.as_s - contents = row.dig?("metadataRowRenderer", "contents", 0) - if metadata_title == "Category" - contents = contents.try &.dig?("runs", 0) - - genre = contents.try &.["text"]? - genre_ucid = contents.try &.dig?("navigationEndpoint", "browseEndpoint", "browseId") - elsif metadata_title == "License" - license = contents.try &.dig?("runs", 0, "text") - elsif metadata_title == "Licensed to YouTube by" - license = contents.try &.["simpleText"]? - end + {% if flag?(:debug_macros) %} {{debug}} {% end %} end - # Author infos + # Method definitions, using the macros above - if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer") - author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url") - author_verified = has_verified_badge?(author_info["badges"]?) + getset_string author + getset_string authorThumbnail + getset_string description + getset_string descriptionHtml + getset_string genre + getset_string genreUcid + getset_string license + getset_string shortDescription + getset_string subCountText + getset_string title + getset_string ucid - subs_text = author_info["subscriberCountText"]? - .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") } - .try &.as_s.split(" ", 2)[0] - end + getset_string_array allowedRegions + getset_string_array keywords - # Return data - - params = { - "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil), - "relatedVideos" => JSON::Any.new(related), - "likes" => JSON::Any.new(likes || 0_i64), - "dislikes" => JSON::Any.new(0_i64), - "descriptionHtml" => JSON::Any.new(description_html || "<p></p>"), - "genre" => JSON::Any.new(genre.try &.as_s || ""), - "genreUrl" => JSON::Any.new(nil), - "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""), - "license" => JSON::Any.new(license.try &.as_s || ""), - "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), - "authorVerified" => JSON::Any.new(author_verified), - "subCountText" => JSON::Any.new(subs_text || "-"), - } + getset_i32 lengthSeconds + getset_i64 likes + getset_i64 views - return params + getset_bool allowRatings + getset_bool authorVerified + getset_bool isFamilyFriendly + getset_bool isListed + getset_bool isUpcoming end def get_video(id, refresh = true, region = nil, force_refresh = false) @@ -1098,7 +340,8 @@ def get_video(id, refresh = true, region = nil, force_refresh = false) if (refresh && (Time.utc - video.updated > 10.minutes) || (video.premiere_timestamp.try &.< Time.utc)) || - force_refresh + force_refresh || + video.schema_version != Video::SCHEMA_VERSION # cache control begin video = fetch_video(id, region) Invidious::Database::Videos.update(video) @@ -1137,12 +380,6 @@ def fetch_video(id, region) end end - # Try to fetch video info using an embedded client - if info["reason"]? - embed_info = extract_video_info(video_id: id, context_screen: "embed") - info = embed_info if !embed_info["reason"]? - end - if reason = info["reason"]? if reason == "Video unavailable" raise NotFoundException.new(reason.as_s || "") @@ -1160,10 +397,6 @@ def fetch_video(id, region) return video end -def itag_to_metadata?(itag : JSON::Any) - return VIDEO_FORMATS[itag.to_s]? -end - def process_continuation(query, plid, id) continuation = nil if plid @@ -1178,135 +411,6 @@ def process_continuation(query, plid, id) continuation end -def process_video_params(query, preferences) - annotations = query["iv_load_policy"]?.try &.to_i? - autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } - comments = query["comments"]?.try &.split(",").map(&.downcase) - continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe } - continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } - listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe } - local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe } - player_style = query["player_style"]? - preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase) - quality = query["quality"]? - quality_dash = query["quality_dash"]? - region = query["region"]? - related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe } - speed = query["speed"]?.try &.rchop("x").to_f? - video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe } - extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe } - volume = query["volume"]?.try &.to_i? - vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe } - save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe } - - if preferences - # region ||= preferences.region - annotations ||= preferences.annotations.to_unsafe - autoplay ||= preferences.autoplay.to_unsafe - comments ||= preferences.comments - continue ||= preferences.continue.to_unsafe - continue_autoplay ||= preferences.continue_autoplay.to_unsafe - listen ||= preferences.listen.to_unsafe - local ||= preferences.local.to_unsafe - player_style ||= preferences.player_style - preferred_captions ||= preferences.captions - quality ||= preferences.quality - quality_dash ||= preferences.quality_dash - related_videos ||= preferences.related_videos.to_unsafe - speed ||= preferences.speed - video_loop ||= preferences.video_loop.to_unsafe - extend_desc ||= preferences.extend_desc.to_unsafe - volume ||= preferences.volume - vr_mode ||= preferences.vr_mode.to_unsafe - save_player_pos ||= preferences.save_player_pos.to_unsafe - end - - annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe - autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe - comments ||= CONFIG.default_user_preferences.comments - continue ||= CONFIG.default_user_preferences.continue.to_unsafe - continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe - listen ||= CONFIG.default_user_preferences.listen.to_unsafe - local ||= CONFIG.default_user_preferences.local.to_unsafe - player_style ||= CONFIG.default_user_preferences.player_style - preferred_captions ||= CONFIG.default_user_preferences.captions - quality ||= CONFIG.default_user_preferences.quality - quality_dash ||= CONFIG.default_user_preferences.quality_dash - related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe - speed ||= CONFIG.default_user_preferences.speed - video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe - extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe - volume ||= CONFIG.default_user_preferences.volume - vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe - save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe - - annotations = annotations == 1 - autoplay = autoplay == 1 - continue = continue == 1 - continue_autoplay = continue_autoplay == 1 - listen = listen == 1 - local = local == 1 - related_videos = related_videos == 1 - video_loop = video_loop == 1 - extend_desc = extend_desc == 1 - vr_mode = vr_mode == 1 - save_player_pos = save_player_pos == 1 - - if CONFIG.disabled?("dash") && quality == "dash" - quality = "high" - end - - if CONFIG.disabled?("local") && local - local = false - end - - if start = query["t"]? || query["time_continue"]? || query["start"]? - video_start = decode_time(start) - end - video_start ||= 0 - - if query["end"]? - video_end = decode_time(query["end"]) - end - video_end ||= -1 - - raw = query["raw"]?.try &.to_i? - raw ||= 0 - raw = raw == 1 - - controls = query["controls"]?.try &.to_i? - controls ||= 1 - controls = controls >= 1 - - params = VideoPreferences.new({ - annotations: annotations, - autoplay: autoplay, - comments: comments, - continue: continue, - continue_autoplay: continue_autoplay, - controls: controls, - listen: listen, - local: local, - player_style: player_style, - preferred_captions: preferred_captions, - quality: quality, - quality_dash: quality_dash, - raw: raw, - region: region, - related_videos: related_videos, - speed: speed, - video_end: video_end, - video_loop: video_loop, - extend_desc: extend_desc, - video_start: video_start, - volume: volume, - vr_mode: vr_mode, - save_player_pos: save_player_pos, - }) - - return params -end - def build_thumbnails(id) return { {host: HOST_URL, height: 720, width: 1280, name: "maxres", url: "maxres"}, @@ -1320,34 +424,3 @@ def build_thumbnails(id) {host: HOST_URL, height: 90, width: 120, name: "end", url: "3"}, } end - -def generate_thumbnails(json, id) - json.array do - build_thumbnails(id).each do |thumbnail| - json.object do - json.field "quality", thumbnail[:name] - json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg" - json.field "width", thumbnail[:width] - json.field "height", thumbnail[:height] - end - end - end -end - -def generate_storyboards(json, id, storyboards) - json.array do - storyboards.each do |storyboard| - json.object do - json.field "url", "/api/v1/storyboards/#{id}?width=#{storyboard[:width]}&height=#{storyboard[:height]}" - json.field "templateUrl", storyboard[:url] - json.field "width", storyboard[:width] - json.field "height", storyboard[:height] - json.field "count", storyboard[:count] - json.field "interval", storyboard[:interval] - json.field "storyboardWidth", storyboard[:storyboard_width] - json.field "storyboardHeight", storyboard[:storyboard_height] - json.field "storyboardCount", storyboard[:storyboard_count] - end - end - end -end diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr new file mode 100644 index 00000000..4642c1a7 --- /dev/null +++ b/src/invidious/videos/caption.cr @@ -0,0 +1,168 @@ +require "json" + +module Invidious::Videos + struct Caption + property name : String + property language_code : String + property base_url : String + + def initialize(@name, @language_code, @base_url) + end + + # Parse the JSON structure from Youtube + def self.from_yt_json(container : JSON::Any) : Array(Caption) + caption_tracks = container + .dig?("playerCaptionsTracklistRenderer", "captionTracks") + .try &.as_a + + captions_list = [] of Caption + return captions_list if caption_tracks.nil? + + caption_tracks.each do |caption| + name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"] + name = name.to_s.split(" - ")[0] + + language_code = caption["languageCode"].to_s + base_url = caption["baseUrl"].to_s + + captions_list << Caption.new(name, language_code, base_url) + end + + return captions_list + end + + # List of all caption languages available on Youtube. + LANGUAGES = { + "", + "English", + "English (auto-generated)", + "English (United Kingdom)", + "English (United States)", + "Afrikaans", + "Albanian", + "Amharic", + "Arabic", + "Armenian", + "Azerbaijani", + "Bangla", + "Basque", + "Belarusian", + "Bosnian", + "Bulgarian", + "Burmese", + "Cantonese (Hong Kong)", + "Catalan", + "Cebuano", + "Chinese", + "Chinese (China)", + "Chinese (Hong Kong)", + "Chinese (Simplified)", + "Chinese (Taiwan)", + "Chinese (Traditional)", + "Corsican", + "Croatian", + "Czech", + "Danish", + "Dutch", + "Dutch (auto-generated)", + "Esperanto", + "Estonian", + "Filipino", + "Finnish", + "French", + "French (auto-generated)", + "Galician", + "Georgian", + "German", + "German (auto-generated)", + "Greek", + "Gujarati", + "Haitian Creole", + "Hausa", + "Hawaiian", + "Hebrew", + "Hindi", + "Hmong", + "Hungarian", + "Icelandic", + "Igbo", + "Indonesian", + "Indonesian (auto-generated)", + "Interlingue", + "Irish", + "Italian", + "Italian (auto-generated)", + "Japanese", + "Japanese (auto-generated)", + "Javanese", + "Kannada", + "Kazakh", + "Khmer", + "Korean", + "Korean (auto-generated)", + "Kurdish", + "Kyrgyz", + "Lao", + "Latin", + "Latvian", + "Lithuanian", + "Luxembourgish", + "Macedonian", + "Malagasy", + "Malay", + "Malayalam", + "Maltese", + "Maori", + "Marathi", + "Mongolian", + "Nepali", + "Norwegian Bokmål", + "Nyanja", + "Pashto", + "Persian", + "Polish", + "Portuguese", + "Portuguese (auto-generated)", + "Portuguese (Brazil)", + "Punjabi", + "Romanian", + "Russian", + "Russian (auto-generated)", + "Samoan", + "Scottish Gaelic", + "Serbian", + "Shona", + "Sindhi", + "Sinhala", + "Slovak", + "Slovenian", + "Somali", + "Southern Sotho", + "Spanish", + "Spanish (auto-generated)", + "Spanish (Latin America)", + "Spanish (Mexico)", + "Spanish (Spain)", + "Sundanese", + "Swahili", + "Swedish", + "Tajik", + "Tamil", + "Telugu", + "Thai", + "Turkish", + "Turkish (auto-generated)", + "Ukrainian", + "Urdu", + "Uzbek", + "Vietnamese", + "Vietnamese (auto-generated)", + "Welsh", + "Western Frisian", + "Xhosa", + "Yiddish", + "Yoruba", + "Zulu", + } + end +end diff --git a/src/invidious/videos/formats.cr b/src/invidious/videos/formats.cr new file mode 100644 index 00000000..e98e7257 --- /dev/null +++ b/src/invidious/videos/formats.cr @@ -0,0 +1,116 @@ +module Invidious::Videos::Formats + def self.itag_to_metadata?(itag : JSON::Any) + return FORMATS[itag.to_s]? + end + + # See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476 + private FORMATS = { + "5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, + "6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, + "13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"}, + "17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"}, + "18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"}, + "22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, + "34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + + "36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"}, + "37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, + "38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, + "43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, + "44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, + "45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, + "46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, + "59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + + # 3D videos + "82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, + "85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, + "100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, + "101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, + "102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, + + # Apple HTTP Live Streaming + "91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, + "92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, + "93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, + "96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, + "132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, + "151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"}, + + # DASH mp4 video + "133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"}, + "134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"}, + "135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, + "136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"}, + "137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"}, + "138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559) + "160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"}, + "212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, + "264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"}, + "298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, + "299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, + "266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"}, + + # Dash mp4 audio + "139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"}, + "140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"}, + "141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"}, + "256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, + "258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, + "325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"}, + "328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"}, + + # Dash webm + "167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"}, + "242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"}, + "243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"}, + "244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, + "245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, + "246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, + "247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"}, + "248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"}, + "271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"}, + # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug) + "272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, + "302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, + "315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + + # Dash webm audio + "171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128}, + "172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256}, + + # Dash webm audio with opus inside + "249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50}, + "250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70}, + "251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160}, + + # av01 video only formats sometimes served with "unknown" codecs + "394" => {"ext" => "mp4", "height" => 144, "vcodec" => "av01.0.05M.08"}, + "395" => {"ext" => "mp4", "height" => 240, "vcodec" => "av01.0.05M.08"}, + "396" => {"ext" => "mp4", "height" => 360, "vcodec" => "av01.0.05M.08"}, + "397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"}, + } +end diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr new file mode 100644 index 00000000..5df49286 --- /dev/null +++ b/src/invidious/videos/parser.cr @@ -0,0 +1,371 @@ +require "json" + +# Use to parse both "compactVideoRenderer" and "endScreenVideoRenderer". +# The former is preferred as it has more videos in it. The second has +# the same 11 first entries as the compact rendered. +# +# TODO: "compactRadioRenderer" (Mix) and +# TODO: Use a proper struct/class instead of a hacky JSON object +def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? + return nil if !related["videoId"]? + + # The compact renderer has video length in seconds, where the end + # screen rendered has a full text version ("42:40") + length = related["lengthInSeconds"]?.try &.as_i.to_s + length ||= related.dig?("lengthText", "simpleText").try do |box| + decode_length_seconds(box.as_s).to_s + end + + # Both have "short", so the "long" option shouldn't be required + channel_info = (related["shortBylineText"]? || related["longBylineText"]?) + .try &.dig?("runs", 0) + + author = channel_info.try &.dig?("text") + author_verified = has_verified_badge?(related["ownerBadges"]?).to_s + + ucid = channel_info.try { |ci| HelperExtractors.get_browse_id(ci) } + + # "4,088,033 views", only available on compact renderer + # and when video is not a livestream + view_count = related.dig?("viewCountText", "simpleText") + .try &.as_s.gsub(/\D/, "") + + short_view_count = related.try do |r| + HelperExtractors.get_short_view_count(r).to_s + end + + LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container") + + # TODO: when refactoring video types, make a struct for related videos + # or reuse an existing type, if that fits. + return { + "id" => related["videoId"], + "title" => related["title"]["simpleText"], + "author" => author || JSON::Any.new(""), + "ucid" => JSON::Any.new(ucid || ""), + "length_seconds" => JSON::Any.new(length || "0"), + "view_count" => JSON::Any.new(view_count || "0"), + "short_view_count" => JSON::Any.new(short_view_count || "0"), + "author_verified" => JSON::Any.new(author_verified), + } +end + +def extract_video_info(video_id : String, proxy_region : String? = nil) + # Init client config for the API + client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region) + + # Fetch data from the player endpoint + # 8AEB param is used to fetch YouTube stories + player_response = YoutubeAPI.player(video_id: video_id, params: "8AEB", client_config: client_config) + + playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s + + if playability_status != "OK" + subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason") + reason = subreason.try &.[]?("simpleText").try &.as_s + reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("") + reason ||= player_response.dig("playabilityStatus", "reason").as_s + + # Stop here if video is not a scheduled livestream + if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) + return { + "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), + "reason" => JSON::Any.new(reason), + } + end + elsif video_id != player_response.dig("videoDetails", "videoId") + # YouTube may return a different video player response than expected. + # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 + raise VideoNotAvailableException.new("The video returned by YouTube isn't the requested one. (WEB client)") + else + reason = nil + end + + # Don't fetch the next endpoint if the video is unavailable. + if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) + next_response = YoutubeAPI.next({"videoId": video_id, "params": ""}) + player_response = player_response.merge(next_response) + end + + params = parse_video_info(video_id, player_response) + params["reason"] = JSON::Any.new(reason) if reason + + new_player_response = nil + + if reason.nil? + # Fetch the video streams using an Android client in order to get the + # decrypted URLs and maybe fix throttling issues (#2194). See the + # following issue for an explanation about decrypted URLs: + # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 + client_config.client_type = YoutubeAPI::ClientType::Android + new_player_response = try_fetch_streaming_data(video_id, client_config) + elsif !reason.includes?("your country") # Handled separately + # The Android embedded client could help here + client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed + new_player_response = try_fetch_streaming_data(video_id, client_config) + end + + # Last hope + if new_player_response.nil? + client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed + new_player_response = try_fetch_streaming_data(video_id, client_config) + end + + # Replace player response and reset reason + if !new_player_response.nil? + player_response = new_player_response + params.delete("reason") + end + + {"captions", "playabilityStatus", "playerConfig", "storyboards", "streamingData"}.each do |f| + params[f] = player_response[f] if player_response[f]? + end + + # Data structure version, for cache control + params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64) + + return params +end + +def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)? + LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.") + # 8AEB param is used to fetch YouTube stories + response = YoutubeAPI.player(video_id: id, params: "8AEB", client_config: client_config) + + playability_status = response["playabilityStatus"]["status"] + LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.") + + if id != response.dig("videoDetails", "videoId") + # YouTube may return a different video player response than expected. + # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 + raise VideoNotAvailableException.new( + "The video returned by YouTube isn't the requested one. (#{client_config.client_type} client)" + ) + elsif playability_status == "OK" + return response + else + return nil + end +end + +def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any) + # Top level elements + + main_results = player_response.dig?("contents", "twoColumnWatchNextResults") + + raise BrokenTubeException.new("twoColumnWatchNextResults") if !main_results + + # Primary results are not available on Music videos + # See: https://github.com/iv-org/invidious/pull/3238#issuecomment-1207193725 + if primary_results = main_results.dig?("results", "results", "contents") + video_primary_renderer = primary_results + .as_a.find(&.["videoPrimaryInfoRenderer"]?) + .try &.["videoPrimaryInfoRenderer"] + + video_secondary_renderer = primary_results + .as_a.find(&.["videoSecondaryInfoRenderer"]?) + .try &.["videoSecondaryInfoRenderer"] + + raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer + raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer + end + + video_details = player_response.dig?("videoDetails") + microformat = player_response.dig?("microformat", "playerMicroformatRenderer") + + raise BrokenTubeException.new("videoDetails") if !video_details + raise BrokenTubeException.new("microformat") if !microformat + + # Basic video infos + + title = video_details["title"]?.try &.as_s + + # We have to try to extract viewCount from videoPrimaryInfoRenderer first, + # then from videoDetails, as the latter is "0" for livestreams (we want + # to get the amount of viewers watching). + views_txt = video_primary_renderer + .try &.dig?("viewCount", "videoViewCountRenderer", "viewCount", "runs", 0, "text") + views_txt ||= video_details["viewCount"]? + views = views_txt.try &.as_s.gsub(/\D/, "").to_i64? + + length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"]) + .try &.as_s.to_i64 + + published = microformat["publishDate"]? + .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc + + premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp") + .try { |t| Time.parse_rfc3339(t.as_s) } + + live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow") + .try &.as_bool || false + + # Extra video infos + + allowed_regions = microformat["availableCountries"]? + .try &.as_a.map &.as_s || [] of String + + allow_ratings = video_details["allowRatings"]?.try &.as_bool + family_friendly = microformat["isFamilySafe"].try &.as_bool + is_listed = video_details["isCrawlable"]?.try &.as_bool + is_upcoming = video_details["isUpcoming"]?.try &.as_bool + + keywords = video_details["keywords"]? + .try &.as_a.map &.as_s || [] of String + + # Related videos + + LOGGER.debug("extract_video_info: parsing related videos...") + + related = [] of JSON::Any + + # Parse "compactVideoRenderer" items (under secondary results) + secondary_results = main_results + .dig?("secondaryResults", "secondaryResults", "results") + secondary_results.try &.as_a.each do |element| + if item = element["compactVideoRenderer"]? + related_video = parse_related_video(item) + related << JSON::Any.new(related_video) if related_video + end + end + + # If nothing was found previously, fall back to end screen renderer + if related.empty? + # Container for "endScreenVideoRenderer" items + player_overlays = player_response.dig?( + "playerOverlays", "playerOverlayRenderer", + "endScreen", "watchNextEndScreenRenderer", "results" + ) + + player_overlays.try &.as_a.each do |element| + if item = element["endScreenVideoRenderer"]? + related_video = parse_related_video(item) + related << JSON::Any.new(related_video) if related_video + end + end + end + + # Likes + + toplevel_buttons = video_primary_renderer + .try &.dig?("videoActions", "menuRenderer", "topLevelButtons") + + if toplevel_buttons + likes_button = toplevel_buttons.try &.as_a + .find(&.dig?("toggleButtonRenderer", "defaultIcon", "iconType").=== "LIKE") + .try &.["toggleButtonRenderer"] + + # New format as of september 2022 + likes_button ||= toplevel_buttons.try &.as_a + .find(&.["segmentedLikeDislikeButtonRenderer"]?) + .try &.dig?( + "segmentedLikeDislikeButtonRenderer", + "likeButton", "toggleButtonRenderer" + ) + + if likes_button + # Note: The like count from `toggledText` is off by one, as it would + # represent the new like count in the event where the user clicks on "like". + likes_txt = (likes_button["defaultText"]? || likes_button["toggledText"]?) + .try &.dig?("accessibility", "accessibilityData", "label") + likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt + + LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"") + LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes + end + end + + # Description + + description = microformat.dig?("description", "simpleText").try &.as_s || "" + short_description = player_response.dig?("videoDetails", "shortDescription") + + description_html = video_secondary_renderer.try &.dig?("description", "runs") + .try &.as_a.try { |t| content_to_comment_html(t, video_id) } + + # Video metadata + + metadata = video_secondary_renderer + .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows") + .try &.as_a + + genre = microformat["category"]? + genre_ucid = nil + license = nil + + metadata.try &.each do |row| + metadata_title = extract_text(row.dig?("metadataRowRenderer", "title")) + contents = row.dig?("metadataRowRenderer", "contents", 0) + + if metadata_title == "Category" + contents = contents.try &.dig?("runs", 0) + + genre = contents.try &.["text"]? + genre_ucid = contents.try &.dig?("navigationEndpoint", "browseEndpoint", "browseId") + elsif metadata_title == "License" + license = contents.try &.dig?("runs", 0, "text") + elsif metadata_title == "Licensed to YouTube by" + license = contents.try &.["simpleText"]? + end + end + + # Author infos + + author = video_details["author"]?.try &.as_s + ucid = video_details["channelId"]?.try &.as_s + + if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer") + author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url") + author_verified = has_verified_badge?(author_info["badges"]?) + + subs_text = author_info["subscriberCountText"]? + .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") } + .try &.as_s.split(" ", 2)[0] + end + + # Return data + + if live_now + video_type = VideoType::Livestream + elsif !premiere_timestamp.nil? + video_type = VideoType::Scheduled + published = premiere_timestamp || Time.utc + else + video_type = VideoType::Video + end + + params = { + "videoType" => JSON::Any.new(video_type.to_s), + # Basic video infos + "title" => JSON::Any.new(title || ""), + "views" => JSON::Any.new(views || 0_i64), + "likes" => JSON::Any.new(likes || 0_i64), + "lengthSeconds" => JSON::Any.new(length_txt || 0_i64), + "published" => JSON::Any.new(published.to_rfc3339), + # Extra video infos + "allowedRegions" => JSON::Any.new(allowed_regions.map { |v| JSON::Any.new(v) }), + "allowRatings" => JSON::Any.new(allow_ratings || false), + "isFamilyFriendly" => JSON::Any.new(family_friendly || false), + "isListed" => JSON::Any.new(is_listed || false), + "isUpcoming" => JSON::Any.new(is_upcoming || false), + "keywords" => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }), + # Related videos + "relatedVideos" => JSON::Any.new(related), + # Description + "description" => JSON::Any.new(description || ""), + "descriptionHtml" => JSON::Any.new(description_html || "<p></p>"), + "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil), + # Video metadata + "genre" => JSON::Any.new(genre.try &.as_s || ""), + "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""), + "license" => JSON::Any.new(license.try &.as_s || ""), + # Author infos + "author" => JSON::Any.new(author || ""), + "ucid" => JSON::Any.new(ucid || ""), + "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), + "authorVerified" => JSON::Any.new(author_verified || false), + "subCountText" => JSON::Any.new(subs_text || "-"), + } + + return params +end diff --git a/src/invidious/videos/regions.cr b/src/invidious/videos/regions.cr new file mode 100644 index 00000000..575f8c25 --- /dev/null +++ b/src/invidious/videos/regions.cr @@ -0,0 +1,27 @@ +# List of geographical regions that Youtube recognizes. +# This is used to determine if a video is either restricted to a list +# of allowed regions (= whitelisted) or if it can't be watched in +# a set of regions (= blacklisted). +REGIONS = { + "AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", + "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", + "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", + "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", + "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", + "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", + "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", + "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", + "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", + "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", + "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", + "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", + "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", + "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", + "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", + "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", + "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", + "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", + "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", + "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", + "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW", +} diff --git a/src/invidious/videos/video_preferences.cr b/src/invidious/videos/video_preferences.cr new file mode 100644 index 00000000..34cf7ff0 --- /dev/null +++ b/src/invidious/videos/video_preferences.cr @@ -0,0 +1,156 @@ +struct VideoPreferences + include JSON::Serializable + + property annotations : Bool + property autoplay : Bool + property comments : Array(String) + property continue : Bool + property continue_autoplay : Bool + property controls : Bool + property listen : Bool + property local : Bool + property preferred_captions : Array(String) + property player_style : String + property quality : String + property quality_dash : String + property raw : Bool + property region : String? + property related_videos : Bool + property speed : Float32 | Float64 + property video_end : Float64 | Int32 + property video_loop : Bool + property extend_desc : Bool + property video_start : Float64 | Int32 + property volume : Int32 + property vr_mode : Bool + property save_player_pos : Bool +end + +def process_video_params(query, preferences) + annotations = query["iv_load_policy"]?.try &.to_i? + autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } + comments = query["comments"]?.try &.split(",").map(&.downcase) + continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe } + continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } + listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe } + local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe } + player_style = query["player_style"]? + preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase) + quality = query["quality"]? + quality_dash = query["quality_dash"]? + region = query["region"]? + related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe } + speed = query["speed"]?.try &.rchop("x").to_f? + video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe } + extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe } + volume = query["volume"]?.try &.to_i? + vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe } + save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe } + + if preferences + # region ||= preferences.region + annotations ||= preferences.annotations.to_unsafe + autoplay ||= preferences.autoplay.to_unsafe + comments ||= preferences.comments + continue ||= preferences.continue.to_unsafe + continue_autoplay ||= preferences.continue_autoplay.to_unsafe + listen ||= preferences.listen.to_unsafe + local ||= preferences.local.to_unsafe + player_style ||= preferences.player_style + preferred_captions ||= preferences.captions + quality ||= preferences.quality + quality_dash ||= preferences.quality_dash + related_videos ||= preferences.related_videos.to_unsafe + speed ||= preferences.speed + video_loop ||= preferences.video_loop.to_unsafe + extend_desc ||= preferences.extend_desc.to_unsafe + volume ||= preferences.volume + vr_mode ||= preferences.vr_mode.to_unsafe + save_player_pos ||= preferences.save_player_pos.to_unsafe + end + + annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe + autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe + comments ||= CONFIG.default_user_preferences.comments + continue ||= CONFIG.default_user_preferences.continue.to_unsafe + continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe + listen ||= CONFIG.default_user_preferences.listen.to_unsafe + local ||= CONFIG.default_user_preferences.local.to_unsafe + player_style ||= CONFIG.default_user_preferences.player_style + preferred_captions ||= CONFIG.default_user_preferences.captions + quality ||= CONFIG.default_user_preferences.quality + quality_dash ||= CONFIG.default_user_preferences.quality_dash + related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe + speed ||= CONFIG.default_user_preferences.speed + video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe + extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe + volume ||= CONFIG.default_user_preferences.volume + vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe + save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe + + annotations = annotations == 1 + autoplay = autoplay == 1 + continue = continue == 1 + continue_autoplay = continue_autoplay == 1 + listen = listen == 1 + local = local == 1 + related_videos = related_videos == 1 + video_loop = video_loop == 1 + extend_desc = extend_desc == 1 + vr_mode = vr_mode == 1 + save_player_pos = save_player_pos == 1 + + if CONFIG.disabled?("dash") && quality == "dash" + quality = "high" + end + + if CONFIG.disabled?("local") && local + local = false + end + + if start = query["t"]? || query["time_continue"]? || query["start"]? + video_start = decode_time(start) + end + video_start ||= 0 + + if query["end"]? + video_end = decode_time(query["end"]) + end + video_end ||= -1 + + raw = query["raw"]?.try &.to_i? + raw ||= 0 + raw = raw == 1 + + controls = query["controls"]?.try &.to_i? + controls ||= 1 + controls = controls >= 1 + + params = VideoPreferences.new({ + annotations: annotations, + autoplay: autoplay, + comments: comments, + continue: continue, + continue_autoplay: continue_autoplay, + controls: controls, + listen: listen, + local: local, + player_style: player_style, + preferred_captions: preferred_captions, + quality: quality, + quality_dash: quality_dash, + raw: raw, + region: region, + related_videos: related_videos, + speed: speed, + video_end: video_end, + video_loop: video_loop, + extend_desc: extend_desc, + video_start: video_start, + volume: volume, + vr_mode: vr_mode, + save_player_pos: save_player_pos, + }) + + return params +end diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr index 92f81ee4..dea86abe 100644 --- a/src/invidious/views/channel.ecr +++ b/src/invidious/views/channel.ecr @@ -1,7 +1,20 @@ <% ucid = channel.ucid %> <% author = HTML.escape(channel.author) %> +<% channel_profile_pic = URI.parse(channel.author_thumbnail).request_target %> <% content_for "header" do %> +<meta name="description" content="<%= channel.description %>"> +<meta property="og:site_name" content="Invidious"> +<meta property="og:url" content="<%= HOST_URL %>/channel/<%= ucid %>"> +<meta property="og:title" content="<%= author %>"> +<meta property="og:image" content="/ggpht<%= channel_profile_pic %>"> +<meta property="og:description" content="<%= channel.description %>"> +<meta name="twitter:card" content="summary"> +<meta name="twitter:url" content="<%= HOST_URL %>/channel/<%= ucid %>"> +<meta name="twitter:title" content="<%= author %>"> +<meta name="twitter:description" content="<%= channel.description %>"> +<meta name="twitter:image" content="/ggpht<%= channel_profile_pic %>"> +<link rel="alternate" href="https://www.youtube.com/channel/<%= ucid %>"> <title><%= author %> - Invidious</title> <link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/channel/<%= ucid %>" /> <% end %> @@ -19,7 +32,7 @@ <div class="pure-g h-box"> <div class="pure-u-2-3"> <div class="channel-profile"> - <img src="/ggpht<%= URI.parse(channel.author_thumbnail).request_target %>"> + <img src="/ggpht<%= channel_profile_pic %>"> <span><%= author %></span><% if !channel.verified.nil? && channel.verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end %> </div> </div> diff --git a/src/invidious/views/licenses.ecr b/src/invidious/views/licenses.ecr index 25b24ed4..667cfa37 100644 --- a/src/invidious/views/licenses.ecr +++ b/src/invidious/views/licenses.ecr @@ -25,6 +25,20 @@ <tr> <td> + <a href="/js/handlers.js?v=<%= ASSET_COMMIT %>">handlers.js</a> + </td> + + <td> + <a href="https://www.gnu.org/licenses/agpl-3.0.html">AGPL-3.0</a> + </td> + + <td> + <a href="/js/handlers.js?v=<%= ASSET_COMMIT %>"><%= translate(locale, "source") %></a> + </td> + </tr> + + <tr> + <td> <a href="/js/community.js?v=<%= ASSET_COMMIT %>">community.js</a> </td> @@ -169,7 +183,7 @@ </td> <td> - <a href="https://choosealicense.com/licenses/mit/">MIT</a> + <a href="https://choosealicense.com/licenses/mit/">Expat</a> </td> <td> @@ -253,7 +267,7 @@ </td> <td> - <a href="https://choosealicense.com/licenses/mit">MIT</a> + <a href="https://choosealicense.com/licenses/mit">Expat</a> </td> <td> diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index caf5299f..98f72eba 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -67,7 +67,7 @@ </a> </div> <% if env.get("preferences").as(Preferences).show_nick %> - <div class="pure-u-1-4"> + <div class="pure-u-1-4" style="overflow: hidden; white-space: nowrap;"> <span id="user_name"><%= HTML.escape(env.get("user").as(Invidious::User).email) %></span> </div> <% end %> diff --git a/src/invidious/views/user/preferences.ecr b/src/invidious/views/user/preferences.ecr index dbb5e9db..d841982c 100644 --- a/src/invidious/views/user/preferences.ecr +++ b/src/invidious/views/user/preferences.ecr @@ -89,7 +89,7 @@ <label for="captions[0]"><%= translate(locale, "preferences_captions_label") %></label> <% preferences.captions.each_with_index do |caption, index| %> <select class="pure-u-1-6" name="captions[<%= index %>]" id="captions[<%= index %>]"> - <% CAPTION_LANGUAGES.each do |option| %> + <% Invidious::Videos::Caption::LANGUAGES.each do |option| %> <option value="<%= option %>" <% if preferences.captions[index] == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option> <% end %> </select> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 50c63d21..a6f2e524 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -7,7 +7,7 @@ <meta name="thumbnail" content="<%= thumbnail %>"> <meta name="description" content="<%= HTML.escape(video.short_description) %>"> <meta name="keywords" content="<%= video.keywords.join(",") %>"> -<meta property="og:site_name" content="Invidious"> +<meta property="og:site_name" content="<%= author %> | Invidious"> <meta property="og:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>"> <meta property="og:title" content="<%= title %>"> <meta property="og:image" content="/vi/<%= video.id %>/maxres.jpg"> @@ -19,7 +19,6 @@ <meta property="og:video:width" content="1280"> <meta property="og:video:height" content="720"> <meta name="twitter:card" content="player"> -<meta name="twitter:site" content="@omarroth1"> <meta name="twitter:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>"> <meta name="twitter:title" content="<%= title %>"> <meta name="twitter:description" content="<%= HTML.escape(video.short_description) %>"> @@ -270,7 +269,7 @@ we're going to need to do it here in order to allow for translations. <% video.related_videos.each do |rv| %> <% if rv["id"]? %> - <a href="/watch?v=<%= rv["id"] %>"> + <a href="/watch?v=<%= rv["id"] %>&listen=<%= params.listen %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> <img loading="lazy" class="thumbnail" src="/vi/<%= rv["id"] %>/mqdefault.jpg"> diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 3feb9233..46e5bf85 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -7,17 +7,16 @@ {% end %} def add_yt_headers(request) - request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" - request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" - request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" - request.headers["accept-language"] ||= "en-us,en;q=0.5" - return if request.resource.starts_with? "/sorry/index" - request.headers["x-youtube-client-name"] ||= "1" - request.headers["x-youtube-client-version"] ||= "2.20200609" + if request.headers["User-Agent"] == "Crystal" + request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36" + end + request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" + request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + request.headers["Accept-Language"] ||= "en-us,en;q=0.5" # Preserve original cookies and add new YT consent cookie for EU servers - request.headers["cookie"] = "#{request.headers["cookie"]?}; CONSENT=YES+" + request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=YES+" if !CONFIG.cookies.empty? - request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" + request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" end end diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index dc65cc52..edc722cf 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -17,6 +17,7 @@ private ITEM_PARSERS = { Parsers::PlaylistRendererParser, Parsers::CategoryRendererParser, Parsers::RichItemRendererParser, + Parsers::ReelItemRendererParser, } record AuthorFallback, name : String, id : String @@ -169,7 +170,7 @@ private module Parsers # Always simpleText # TODO change default value to nil subscriber_count = item_contents.dig?("subscriberCountText", "simpleText") - .try { |s| short_text_to_number(s.as_s.split(" ")[0]) } || 0 + .try { |s| short_text_to_number(s.as_s.split(" ")[0]).to_i32 } || 0 # Auto-generated channels doesn't have videoCountText # Taken from: https://github.com/iv-org/invidious/pull/2228#discussion_r717620922 @@ -369,7 +370,7 @@ private module Parsers end # Parses an InnerTube richItemRenderer into a SearchVideo. - # Returns nil when the given object isn't a shelfRenderer + # Returns nil when the given object isn't a RichItemRenderer # # A richItemRenderer seems to be a simple wrapper for a videoRenderer, used # by the result page for hashtags. It is located inside a continuationItems @@ -390,6 +391,90 @@ private module Parsers return {{@type.name}} end end + + # Parses an InnerTube reelItemRenderer into a SearchVideo. + # Returns nil when the given object isn't a reelItemRenderer + # + # reelItemRenderer items are used in the new (2022) channel layout, + # in the "shorts" tab. + # + module ReelItemRendererParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["reelItemRenderer"]? + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + video_id = item_contents["videoId"].as_s + + video_details_container = item_contents.dig( + "navigationEndpoint", "reelWatchEndpoint", + "overlay", "reelPlayerOverlayRenderer", + "reelPlayerHeaderSupportedRenderers", + "reelPlayerHeaderRenderer" + ) + + # Author infos + + author = video_details_container + .dig?("channelTitleText", "runs", 0, "text") + .try &.as_s || author_fallback.name + + ucid = video_details_container + .dig?("channelNavigationEndpoint", "browseEndpoint", "browseId") + .try &.as_s || author_fallback.id + + # Title & publication date + + title = video_details_container.dig?("reelTitleText") + .try { |t| extract_text(t) } || "" + + published = video_details_container + .dig?("timestampText", "simpleText") + .try { |t| decode_date(t.as_s) } || Time.utc + + # View count + + view_count_text = video_details_container.dig?("viewCountText", "simpleText") + view_count_text ||= video_details_container + .dig?("viewCountText", "accessibility", "accessibilityData", "label") + + view_count = view_count_text.try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64 + + # Duration + + a11y_data = item_contents + .dig?("accessibility", "accessibilityData", "label") + .try &.as_s || "" + + regex_match = /- (?<min>\d+ minutes? )?(?<sec>\d+ seconds?)+ -/.match(a11y_data) + + minutes = regex_match.try &.["min"].to_i(strict: false) || 0 + seconds = regex_match.try &.["sec"].to_i(strict: false) || 0 + + duration = (minutes*60 + seconds) + + SearchVideo.new({ + title: title, + id: video_id, + author: author, + ucid: ucid, + published: published, + views: view_count, + description_html: "", + length_seconds: duration, + live_now: false, + premium: false, + premiere_timestamp: Time.unix(0), + author_verified: false, + }) + end + + def self.parser_name + return {{@type.name}} + end + end end # The following are the extractors for extracting an array of items from @@ -436,21 +521,31 @@ private module Extractors content = extract_selected_tab(target["tabs"])["content"] if section_list_contents = content.dig?("sectionListRenderer", "contents") - section_list_contents.as_a.each do |renderer_container| - renderer_container_contents = renderer_container["itemSectionRenderer"]["contents"][0] - - # Category extraction - if items_container = renderer_container_contents["shelfRenderer"]? - raw_items << renderer_container_contents - next - elsif items_container = renderer_container_contents["gridRenderer"]? - else - items_container = renderer_container_contents - end + raw_items = unpack_section_list(section_list_contents) + elsif rich_grid_contents = content.dig?("richGridRenderer", "contents") + raw_items = rich_grid_contents.as_a + end - items_container["items"]?.try &.as_a.each do |item| - raw_items << item - end + return raw_items + end + + private def self.unpack_section_list(contents) + raw_items = [] of JSON::Any + + contents.as_a.each do |renderer_container| + renderer_container_contents = renderer_container["itemSectionRenderer"]["contents"][0] + + # Category extraction + if items_container = renderer_container_contents["shelfRenderer"]? + raw_items << renderer_container_contents + next + elsif items_container = renderer_container_contents["gridRenderer"]? + else + items_container = renderer_container_contents + end + + items_container["items"]?.try &.as_a.each do |item| + raw_items << item end end @@ -525,14 +620,11 @@ private module Extractors end private def self.extract(target) - raw_items = [] of JSON::Any - if content = target["gridContinuation"]? - raw_items = content["items"].as_a - elsif content = target["continuationItems"]? - raw_items = content.as_a - end + content = target["continuationItems"]? + content ||= target.dig?("gridContinuation", "items") + content ||= target.dig?("richGridContinuation", "contents") - return raw_items + return content.nil? ? [] of JSON::Any : content.as_a end def self.extractor_name diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 30d7613b..91a9332c 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -7,9 +7,17 @@ module YoutubeAPI private DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" - private ANDROID_APP_VERSION = "17.29.35" - private ANDROID_SDK_VERSION = 30_i64 - private IOS_APP_VERSION = "17.30.1" + private ANDROID_APP_VERSION = "17.33.42" + # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1308 + private ANDROID_USER_AGENT = "com.google.android.youtube/17.33.42 (Linux; U; Android 12; US) gzip" + private ANDROID_SDK_VERSION = 31_i64 + private ANDROID_VERSION = "12" + private IOS_APP_VERSION = "17.33.2" + # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1330 + private IOS_USER_AGENT = "com.google.ios.youtube/17.33.2 (iPhone14,5; U; CPU iOS 15_6 like Mac OS X;)" + # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1224 + private IOS_VERSION = "15.6.0.19G71" + private WINDOWS_VERSION = "10.0" # Enumerate used to select one of the clients supported by the API enum ClientType @@ -33,80 +41,130 @@ module YoutubeAPI # List of hard-coded values used by the different clients HARDCODED_CLIENTS = { ClientType::Web => { - name: "WEB", - version: "2.20220804.07.00", - api_key: DEFAULT_API_KEY, - screen: "WATCH_FULL_SCREEN", + name: "WEB", + name_proto: "1", + version: "2.20221118.01.00", + api_key: DEFAULT_API_KEY, + screen: "WATCH_FULL_SCREEN", + os_name: "Windows", + os_version: WINDOWS_VERSION, + platform: "DESKTOP", }, ClientType::WebEmbeddedPlayer => { - name: "WEB_EMBEDDED_PLAYER", # 56 - version: "1.20220803.01.00", - api_key: DEFAULT_API_KEY, - screen: "EMBED", + name: "WEB_EMBEDDED_PLAYER", + name_proto: "56", + version: "1.20220803.01.00", + api_key: DEFAULT_API_KEY, + screen: "EMBED", + os_name: "Windows", + os_version: WINDOWS_VERSION, + platform: "DESKTOP", }, ClientType::WebMobile => { - name: "MWEB", - version: "2.20220805.01.00", - api_key: DEFAULT_API_KEY, + name: "MWEB", + name_proto: "2", + version: "2.20220805.01.00", + api_key: DEFAULT_API_KEY, + os_name: "Android", + os_version: ANDROID_VERSION, + platform: "MOBILE", }, ClientType::WebScreenEmbed => { - name: "WEB", - version: "2.20220804.00.00", - api_key: DEFAULT_API_KEY, - screen: "EMBED", + name: "WEB", + name_proto: "1", + version: "2.20220804.00.00", + api_key: DEFAULT_API_KEY, + screen: "EMBED", + os_name: "Windows", + os_version: WINDOWS_VERSION, + platform: "DESKTOP", }, # Android ClientType::Android => { name: "ANDROID", + name_proto: "3", version: ANDROID_APP_VERSION, api_key: "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w", android_sdk_version: ANDROID_SDK_VERSION, + user_agent: ANDROID_USER_AGENT, + os_name: "Android", + os_version: ANDROID_VERSION, + platform: "MOBILE", }, ClientType::AndroidEmbeddedPlayer => { - name: "ANDROID_EMBEDDED_PLAYER", # 55 - version: ANDROID_APP_VERSION, - api_key: DEFAULT_API_KEY, + name: "ANDROID_EMBEDDED_PLAYER", + name_proto: "55", + version: ANDROID_APP_VERSION, + api_key: DEFAULT_API_KEY, }, ClientType::AndroidScreenEmbed => { - name: "ANDROID", # 3 + name: "ANDROID", + name_proto: "3", version: ANDROID_APP_VERSION, api_key: DEFAULT_API_KEY, screen: "EMBED", android_sdk_version: ANDROID_SDK_VERSION, + user_agent: ANDROID_USER_AGENT, + os_name: "Android", + os_version: ANDROID_VERSION, + platform: "MOBILE", }, # IOS ClientType::IOS => { - name: "IOS", # 5 - version: IOS_APP_VERSION, - api_key: "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc", + name: "IOS", + name_proto: "5", + version: IOS_APP_VERSION, + api_key: "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc", + user_agent: IOS_USER_AGENT, + device_make: "Apple", + device_model: "iPhone14,5", + os_name: "iPhone", + os_version: IOS_VERSION, + platform: "MOBILE", }, ClientType::IOSEmbedded => { - name: "IOS_MESSAGES_EXTENSION", # 66 - version: IOS_APP_VERSION, - api_key: DEFAULT_API_KEY, + name: "IOS_MESSAGES_EXTENSION", + name_proto: "66", + version: IOS_APP_VERSION, + api_key: DEFAULT_API_KEY, + user_agent: IOS_USER_AGENT, + device_make: "Apple", + device_model: "iPhone14,5", + os_name: "iPhone", + os_version: IOS_VERSION, + platform: "MOBILE", }, ClientType::IOSMusic => { - name: "IOS_MUSIC", # 26 - version: "4.32", - api_key: "AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s", + name: "IOS_MUSIC", + name_proto: "26", + version: "5.21", + api_key: "AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s", + user_agent: "com.google.ios.youtubemusic/5.21 (iPhone14,5; U; CPU iOS 15_6 like Mac OS X;)", + device_make: "Apple", + device_model: "iPhone14,5", + os_name: "iPhone", + os_version: IOS_VERSION, + platform: "MOBILE", }, # TV app ClientType::TvHtml5 => { - name: "TVHTML5", # 7 - version: "7.20220325", - api_key: DEFAULT_API_KEY, + name: "TVHTML5", + name_proto: "7", + version: "7.20220325", + api_key: DEFAULT_API_KEY, }, ClientType::TvHtml5ScreenEmbed => { - name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER", # 85 - version: "2.0", - api_key: DEFAULT_API_KEY, - screen: "EMBED", + name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER", + name_proto: "85", + version: "2.0", + api_key: DEFAULT_API_KEY, + screen: "EMBED", }, } @@ -160,6 +218,10 @@ module YoutubeAPI HARDCODED_CLIENTS[@client_type][:name] end + def name_proto : String + HARDCODED_CLIENTS[@client_type][:name_proto] + end + # :ditto: def version : String HARDCODED_CLIENTS[@client_type][:version] @@ -179,6 +241,30 @@ module YoutubeAPI HARDCODED_CLIENTS[@client_type][:android_sdk_version]? end + def user_agent : String? + HARDCODED_CLIENTS[@client_type][:user_agent]? + end + + def os_name : String? + HARDCODED_CLIENTS[@client_type][:os_name]? + end + + def device_make : String? + HARDCODED_CLIENTS[@client_type][:device_make]? + end + + def device_model : String? + HARDCODED_CLIENTS[@client_type][:device_model]? + end + + def os_version : String? + HARDCODED_CLIENTS[@client_type][:os_version]? + end + + def platform : String? + HARDCODED_CLIENTS[@client_type][:platform]? + end + # Convert to string, for logging purposes def to_s return { @@ -226,6 +312,26 @@ module YoutubeAPI client_context["client"]["androidSdkVersion"] = android_sdk_version end + if device_make = client_config.device_make + client_context["client"]["deviceMake"] = device_make + end + + if device_model = client_config.device_model + client_context["client"]["deviceModel"] = device_model + end + + if os_name = client_config.os_name + client_context["client"]["osName"] = os_name + end + + if os_version = client_config.os_version + client_context["client"]["osVersion"] = os_version + end + + if platform = client_config.platform + client_context["client"]["platform"] = platform + end + return client_context end @@ -361,8 +467,18 @@ module YoutubeAPI ) # JSON Request data, required by the API data = { - "videoId" => video_id, - "context" => self.make_context(client_config), + "contentCheckOk" => true, + "videoId" => video_id, + "context" => self.make_context(client_config), + "racyCheckOk" => true, + "user" => { + "lockedSafetyMode" => false, + }, + "playbackContext" => { + "contentPlaybackContext" => { + "html5Preference": "HTML5_PREF_WANTS", + }, + }, } # Append the additional parameters if those were provided @@ -460,10 +576,17 @@ module YoutubeAPI url = "#{endpoint}?key=#{client_config.api_key}&prettyPrint=false" headers = HTTP::Headers{ - "Content-Type" => "application/json; charset=UTF-8", - "Accept-Encoding" => "gzip, deflate", + "Content-Type" => "application/json; charset=UTF-8", + "Accept-Encoding" => "gzip, deflate", + "x-goog-api-format-version" => "2", + "x-youtube-client-name" => client_config.name_proto, + "x-youtube-client-version" => client_config.version, } + if user_agent = client_config.user_agent + headers["User-Agent"] = user_agent + end + # Logging LOGGER.debug("YoutubeAPI: Using endpoint: \"#{endpoint}\"") LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}") |