diff options
Diffstat (limited to 'src')
78 files changed, 2258 insertions, 1076 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index e0bd0101..566d4dc9 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -23,6 +23,7 @@ require "kilt" require "./ext/kemal_content_for.cr" require "./ext/kemal_static_file_handler.cr" +require "http_proxy" require "athena-negotiation" require "openssl/hmac" require "option_parser" @@ -92,6 +93,10 @@ SOFTWARE = { YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size) +# Image request pool + +GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size) + # CLI Kemal.config.extra_options do |parser| parser.banner = "Usage: invidious [arguments]" @@ -117,6 +122,9 @@ Kemal.config.extra_options do |parser| parser.on("-l LEVEL", "--log-level=LEVEL", "Log level, one of #{LogLevel.values} (default: #{CONFIG.log_level})") do |log_level| CONFIG.log_level = LogLevel.parse(log_level) end + parser.on("-k", "--colorize", "Colorize logs") do + CONFIG.colorize_logs = true + end parser.on("-v", "--version", "Print version") do puts SOFTWARE.to_pretty_json exit @@ -133,7 +141,7 @@ if CONFIG.output.upcase != "STDOUT" FileUtils.mkdir_p(File.dirname(CONFIG.output)) end OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mode: "a") -LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) +LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level, CONFIG.colorize_logs) # Check table integrity Invidious::Database.check_integrity(CONFIG) @@ -153,6 +161,15 @@ Invidious::Database.check_integrity(CONFIG) {% puts "\nDone checking player dependencies, now compiling Invidious...\n" %} {% end %} +# Misc + +DECRYPT_FUNCTION = + if sig_helper_address = CONFIG.signature_server.presence + IV::DecryptFunction.new(sig_helper_address) + else + nil + end + # Start jobs if CONFIG.channel_threads > 0 @@ -163,11 +180,6 @@ if CONFIG.feed_threads > 0 Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB) end -DECRYPT_FUNCTION = DecryptFunction.new(CONFIG.decrypt_polling) -if CONFIG.decrypt_polling - Invidious::Jobs.register Invidious::Jobs::UpdateDecryptFunctionJob.new -end - if CONFIG.statistics_enabled Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, SOFTWARE) end @@ -180,11 +192,14 @@ if CONFIG.popular_enabled Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB) end -CONNECTION_CHANNEL = ::Channel({Bool, ::Channel(PQ::Notification)}).new(32) -Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL, CONFIG.database_url) +NOTIFICATION_CHANNEL = ::Channel(VideoNotification).new(32) +CONNECTION_CHANNEL = ::Channel({Bool, ::Channel(PQ::Notification)}).new(32) +Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(NOTIFICATION_CHANNEL, CONNECTION_CHANNEL, CONFIG.database_url) Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new +Invidious::Jobs.register Invidious::Jobs::InstanceListRefreshJob.new + Invidious::Jobs.start_all def popular_videos diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index b5a27667..13909527 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -15,7 +15,8 @@ record AboutChannel, allowed_regions : Array(String), tabs : Array(String), tags : Array(String), - verified : Bool + verified : Bool, + is_age_gated : Bool def get_about_info(ucid, locale) : AboutChannel begin @@ -45,45 +46,102 @@ def get_about_info(ucid, locale) : AboutChannel end tags = [] of String + tab_names = [] of String + total_views = 0_i64 + joined = Time.unix(0) - if auto_generated - author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s - author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s - author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s - - # Raises a KeyError on failure. - banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? - banner = banners.try &.[-1]?.try &.["url"].as_s? + if age_gate_renderer = initdata.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs", 0, "tabRenderer", "content", "sectionListRenderer", "contents", 0, "channelAgeGateRenderer") + description_node = nil + author = age_gate_renderer["channelTitle"].as_s + ucid = initdata.dig("responseContext", "serviceTrackingParams", 0, "params", 0, "value").as_s + author_url = "https://www.youtube.com/channel/#{ucid}" + author_thumbnail = age_gate_renderer.dig("avatar", "thumbnails", 0, "url").as_s + banner = nil + is_family_friendly = false + is_age_gated = true + tab_names = ["videos", "shorts", "streams"] + auto_generated = false + else + if auto_generated + author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s + author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s + author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s + + # Raises a KeyError on failure. + banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? + banner = banners.try &.[-1]?.try &.["url"].as_s? + + description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] + # some channels have the description in a simpleText + # ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/ + description_node = description_base_node.dig?("simpleText") || description_base_node + + tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges") + .try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String + else + author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s + author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s + author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s + author_verified = has_verified_badge?(initdata.dig?("header", "c4TabbedHeaderRenderer", "badges")) - description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] - # some channels have the description in a simpleText - # ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/ - description_node = description_base_node.dig?("simpleText") || description_base_node + ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s - tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges") - .try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String - else - author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s - author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s - author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s - author_verified = has_verified_badge?(initdata.dig?("header", "c4TabbedHeaderRenderer", "badges")) + # Raises a KeyError on failure. + banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? + banners ||= initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "banner", "imageBannerViewModel", "image", "sources") + banner = banners.try &.[-1]?.try &.["url"].as_s? - ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s + # if banner.includes? "channels/c4/default_banner" + # banner = nil + # end - # Raises a KeyError on failure. - banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? - banner = banners.try &.[-1]?.try &.["url"].as_s? + description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]? + tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String + end - # if banner.includes? "channels/c4/default_banner" - # banner = nil - # end + is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool + if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]? + # Get the name of the tabs available on this channel + tab_names = tabs_json.as_a.compact_map do |entry| + name = entry.dig?("tabRenderer", "title").try &.as_s.downcase + + # This is a small fix to not add extra code on the HTML side + # I.e, the URL for the "live" tab is .../streams, so use "streams" + # everywhere for the sake of simplicity + (name == "live") ? "streams" : name + end + + # Get the currently active tab ("About") + about_tab = extract_selected_tab(tabs_json) + + # Try to find the about metadata section + channel_about_meta = about_tab.dig?( + "content", + "sectionListRenderer", "contents", 0, + "itemSectionRenderer", "contents", 0, + "channelAboutFullMetadataRenderer" + ) - description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]? - tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String + if !channel_about_meta.nil? + total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 + + # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. + joined = extract_text(channel_about_meta["joinedDateText"]?) + .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) + + # Normal Auto-generated channels + # https://support.google.com/youtube/answer/2579942 + # For auto-generated channels, channel_about_meta only has + # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] + auto_generated = ( + (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \ + extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" || + channel_about_meta.dig?("links", 0, "channelExternalLinkViewModel", "title", "content").try &.as_s == "Auto-generated by YouTube" + ) + end + end end - is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool - allowed_regions = initdata .dig?("microformat", "microformatDataRenderer", "availableCountries") .try &.as_a.map(&.as_s) || [] of String @@ -101,56 +159,18 @@ def get_about_info(ucid, locale) : AboutChannel end end - total_views = 0_i64 - joined = Time.unix(0) - - tab_names = [] of String - - if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]? - # Get the name of the tabs available on this channel - tab_names = tabs_json.as_a.compact_map do |entry| - name = entry.dig?("tabRenderer", "title").try &.as_s.downcase + sub_count = 0 - # This is a small fix to not add extra code on the HTML side - # I.e, the URL for the "live" tab is .../streams, so use "streams" - # everywhere for the sake of simplicity - (name == "live") ? "streams" : name - end - - # Get the currently active tab ("About") - about_tab = extract_selected_tab(tabs_json) - - # Try to find the about metadata section - channel_about_meta = about_tab.dig?( - "content", - "sectionListRenderer", "contents", 0, - "itemSectionRenderer", "contents", 0, - "channelAboutFullMetadataRenderer" - ) - - if !channel_about_meta.nil? - total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 - - # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. - joined = extract_text(channel_about_meta["joinedDateText"]?) - .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) - - # Normal Auto-generated channels - # https://support.google.com/youtube/answer/2579942 - # For auto-generated channels, channel_about_meta only has - # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] - auto_generated = ( - (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \ - extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" || - channel_about_meta.dig?("links", 0, "channelExternalLinkViewModel", "title", "content").try &.as_s == "Auto-generated by YouTube" - ) + if (metadata_rows = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows").try &.as_a) + metadata_rows.each do |row| + metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") } + if !metadata_part.nil? + sub_count = short_text_to_number(metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32 + end + break if sub_count != 0 end end - sub_count = initdata - .dig?("header", "c4TabbedHeaderRenderer", "subscriberCountText", "simpleText").try &.as_s? - .try { |text| short_text_to_number(text.split(" ")[0]).to_i32 } || 0 - AboutChannel.new( ucid: ucid, author: author, @@ -168,6 +188,7 @@ def get_about_info(ucid, locale) : AboutChannel tabs: tab_names, tags: tags, verified: author_verified || false, + is_age_gated: is_age_gated || false, ) end diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index be739673..65982325 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -223,7 +223,7 @@ def fetch_channel(ucid, pull_all_videos : Bool) length_seconds = channel_video.try &.length_seconds length_seconds ||= 0 - live_now = channel_video.try &.live_now + live_now = channel_video.try &.badges.live_now? live_now ||= false premiere_timestamp = channel_video.try &.premiere_timestamp @@ -232,7 +232,7 @@ def fetch_channel(ucid, pull_all_videos : Bool) id: video_id, title: title, published: published, - updated: Time.utc, + updated: updated, ucid: ucid, author: author, length_seconds: length_seconds, @@ -249,11 +249,7 @@ def fetch_channel(ucid, pull_all_videos : Bool) if was_insert LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions") - if CONFIG.enable_user_notifications - Invidious::Database::Users.add_notification(video) - else - Invidious::Database::Users.feed_needs_update(video) - end + NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video)) else LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated") end @@ -275,7 +271,7 @@ def fetch_channel(ucid, pull_all_videos : Bool) ucid: video.ucid, author: video.author, length_seconds: video.length_seconds, - live_now: video.live_now, + live_now: video.badges.live_now?, premiere_timestamp: video.premiere_timestamp, views: video.views, }) @@ -285,11 +281,7 @@ def fetch_channel(ucid, pull_all_videos : Bool) if Time.utc - video.published > 1.minute was_insert = Invidious::Database::ChannelVideos.insert(video) if was_insert - if CONFIG.enable_user_notifications - Invidious::Database::Users.add_notification(video) - else - Invidious::Database::Users.feed_needs_update(video) - end + NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video)) end end end diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 351790d7..96400f47 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -1,67 +1,3 @@ -def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - object_inner_2 = { - "2:0:embedded" => { - "1:0:varint" => 0_i64, - }, - "5:varint" => 50_i64, - "6:varint" => 1_i64, - "7:varint" => (page * 30).to_i64, - "9:varint" => 1_i64, - "10:varint" => 0_i64, - } - - object_inner_2_encoded = object_inner_2 - .try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - sort_by_numerical = - case sort_by - when "newest" then 1_i64 - when "popular" then 2_i64 - when "oldest" then 4_i64 - else 1_i64 # Fallback to "newest" - end - - object_inner_1 = { - "110:embedded" => { - "3:embedded" => { - "15:embedded" => { - "1:embedded" => { - "1:string" => object_inner_2_encoded, - }, - "2:embedded" => { - "1:string" => "00000000-0000-0000-0000-000000000000", - }, - "3:varint" => sort_by_numerical, - }, - }, - }, - } - - object_inner_1_encoded = object_inner_1 - .try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:string" => object_inner_1_encoded, - "35:string" => "browse-feed#{ucid}videos102", - }, - } - - continuation = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return continuation -end - module Invidious::Channel::Tabs extend self @@ -69,10 +5,6 @@ module Invidious::Channel::Tabs # Regular videos # ------------------- - def make_initial_video_ctoken(ucid, sort_by) : String - return produce_channel_videos_continuation(ucid, sort_by: sort_by) - end - # Wrapper for AboutChannel, as we still need to call get_videos with # an author name and ucid directly (e.g in RSS feeds). # TODO: figure out how to get rid of that @@ -94,7 +26,7 @@ module Invidious::Channel::Tabs end def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest") - continuation ||= make_initial_video_ctoken(ucid, sort_by) + continuation ||= make_initial_videos_ctoken(ucid, sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, author, ucid) @@ -123,14 +55,10 @@ module Invidious::Channel::Tabs # Shorts # ------------------- - def get_shorts(channel : AboutChannel, continuation : String? = nil) - if continuation.nil? - # EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts" - # TODO: try to extract the continuation tokens that allows other sorting options - initial_data = YoutubeAPI.browse(channel.ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D") - else - initial_data = YoutubeAPI.browse(continuation: continuation) - end + def get_shorts(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") + continuation ||= make_initial_shorts_ctoken(channel.ucid, sort_by) + initial_data = YoutubeAPI.browse(continuation: continuation) + return extract_items(initial_data, channel.author, channel.ucid) end @@ -138,21 +66,17 @@ module Invidious::Channel::Tabs # Livestreams # ------------------- - def get_livestreams(channel : AboutChannel, continuation : String? = nil) - if continuation.nil? - # EgdzdHJlYW1z8gYECgJ6AA%3D%3D is the protobuf object to load "streams" - initial_data = YoutubeAPI.browse(channel.ucid, params: "EgdzdHJlYW1z8gYECgJ6AA%3D%3D") - else - initial_data = YoutubeAPI.browse(continuation: continuation) - end + def get_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") + continuation ||= make_initial_livestreams_ctoken(channel.ucid, sort_by) + initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, channel.author, channel.ucid) end - def get_60_livestreams(channel : AboutChannel, continuation : String? = nil) + def get_60_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") if continuation.nil? - # Fetch the first "page" of streams - items, next_continuation = get_livestreams(channel) + # Fetch the first "page" of stream + items, next_continuation = get_livestreams(channel, sort_by: sort_by) else # Fetch a "page" of streams using the given continuation token items, next_continuation = get_livestreams(channel, continuation: continuation) @@ -167,4 +91,102 @@ module Invidious::Channel::Tabs return items, next_continuation end + + # ------------------- + # C-tokens + # ------------------- + + private def sort_options_videos_short(sort_by : String) + case sort_by + when "newest" then return 4_i64 + when "popular" then return 2_i64 + when "oldest" then return 5_i64 + else return 4_i64 # Fallback to "newest" + end + end + + # Generate the initial "continuation token" to get the first page of the + # "videos" tab. The following page requires the ctoken provided in that + # first page, and so on. + private def make_initial_videos_ctoken(ucid : String, sort_by = "newest") + object = { + "15:embedded" => { + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "4:varint" => sort_options_videos_short(sort_by), + }, + } + + return channel_ctoken_wrap(ucid, object) + end + + # Generate the initial "continuation token" to get the first page of the + # "shorts" tab. The following page requires the ctoken provided in that + # first page, and so on. + private def make_initial_shorts_ctoken(ucid : String, sort_by = "newest") + object = { + "10:embedded" => { + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "4:varint" => sort_options_videos_short(sort_by), + }, + } + + return channel_ctoken_wrap(ucid, object) + end + + # Generate the initial "continuation token" to get the first page of the + # "livestreams" tab. The following page requires the ctoken provided in that + # first page, and so on. + private def make_initial_livestreams_ctoken(ucid : String, sort_by = "newest") + sort_by_numerical = + case sort_by + when "newest" then 12_i64 + when "popular" then 14_i64 + when "oldest" then 13_i64 + else 12_i64 # Fallback to "newest" + end + + object = { + "14:embedded" => { + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "5:varint" => sort_by_numerical, + }, + } + + return channel_ctoken_wrap(ucid, object) + end + + # The protobuf structure common between videos/shorts/livestreams + private def channel_ctoken_wrap(ucid : String, object) + object_inner = { + "110:embedded" => { + "3:embedded" => object, + }, + } + + object_inner_encoded = object_inner + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + object = { + "80226972:embedded" => { + "2:string" => ucid, + "3:string" => object_inner_encoded, + }, + } + + continuation = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + return continuation + end end diff --git a/src/invidious/comments/content.cr b/src/invidious/comments/content.cr index beefd9ad..1f55bfe6 100644 --- a/src/invidious/comments/content.cr +++ b/src/invidious/comments/content.cr @@ -5,35 +5,35 @@ def text_to_parsed_content(text : String) : JSON::Any # In first case line is just a simple node before # check patterns inside line # { 'text': line } - currentNodes = [] of JSON::Any - initialNode = {"text" => line} - currentNodes << (JSON.parse(initialNode.to_json)) + current_nodes = [] of JSON::Any + initial_node = {"text" => line} + current_nodes << (JSON.parse(initial_node.to_json)) # For each match with url pattern, get last node and preserve # last node before create new node with url information # { 'text': match, 'navigationEndpoint': { 'urlEndpoint' : 'url': match } } - line.scan(/https?:\/\/[^ ]*/).each do |urlMatch| + line.scan(/https?:\/\/[^ ]*/).each do |url_match| # Retrieve last node and update node without match - lastNode = currentNodes[currentNodes.size - 1].as_h - splittedLastNode = lastNode["text"].as_s.split(urlMatch[0]) - lastNode["text"] = JSON.parse(splittedLastNode[0].to_json) - currentNodes[currentNodes.size - 1] = JSON.parse(lastNode.to_json) + last_node = current_nodes[-1].as_h + splitted_last_node = last_node["text"].as_s.split(url_match[0]) + last_node["text"] = JSON.parse(splitted_last_node[0].to_json) + current_nodes[-1] = JSON.parse(last_node.to_json) # Create new node with match and navigation infos - currentNode = {"text" => urlMatch[0], "navigationEndpoint" => {"urlEndpoint" => {"url" => urlMatch[0]}}} - currentNodes << (JSON.parse(currentNode.to_json)) + current_node = {"text" => url_match[0], "navigationEndpoint" => {"urlEndpoint" => {"url" => url_match[0]}}} + current_nodes << (JSON.parse(current_node.to_json)) # If text remain after match create new simple node with text after match - afterNode = {"text" => splittedLastNode.size > 1 ? splittedLastNode[1] : ""} - currentNodes << (JSON.parse(afterNode.to_json)) + after_node = {"text" => splitted_last_node.size > 1 ? splitted_last_node[1] : ""} + current_nodes << (JSON.parse(after_node.to_json)) end # After processing of matches inside line # Add \n at end of last node for preserve carriage return - lastNode = currentNodes[currentNodes.size - 1].as_h - lastNode["text"] = JSON.parse("#{currentNodes[currentNodes.size - 1]["text"]}\n".to_json) - currentNodes[currentNodes.size - 1] = JSON.parse(lastNode.to_json) + last_node = current_nodes[-1].as_h + last_node["text"] = JSON.parse("#{last_node["text"]}\n".to_json) + current_nodes[-1] = JSON.parse(last_node.to_json) # Finally add final nodes to nodes returned - currentNodes.each do |node| + current_nodes.each do |node| nodes << (node) end end @@ -53,8 +53,8 @@ def content_to_comment_html(content, video_id : String? = "") text = HTML.escape(run["text"].as_s) - if navigationEndpoint = run.dig?("navigationEndpoint") - text = parse_link_endpoint(navigationEndpoint, text, video_id) + if navigation_endpoint = run.dig?("navigationEndpoint") + text = parse_link_endpoint(navigation_endpoint, text, video_id) end text = "<b>#{text}</b>" if run["bold"]? diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 09c2168b..4b3bdafc 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -13,6 +13,7 @@ struct ConfigPreferences property annotations : Bool = false property annotations_subscribed : Bool = false + property preload : Bool = true property autoplay : Bool = false property captions : Array(String) = ["", "", ""] property comments : Array(String) = ["youtube", ""] @@ -54,6 +55,15 @@ struct ConfigPreferences end end +struct HTTPProxyConfig + include YAML::Serializable + + property user : String + property password : String + property host : String + property port : Int32 +end + class Config include YAML::Serializable @@ -68,14 +78,14 @@ class Config property output : String = "STDOUT" # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr property log_level : LogLevel = LogLevel::Info + # Enables colors in logs. Useful for debugging purposes + property colorize_logs : Bool = false # Database configuration with separate parameters (username, hostname, etc) property db : DBConfig? = nil # Database configuration using 12-Factor "Database URL" syntax @[YAML::Field(converter: Preferences::URIConverter)] property database_url : URI = URI.parse("") - # Use polling to keep decryption function up to date - property decrypt_polling : Bool = false # Used for crawling channels: threads should check all videos uploaded by a channel property full_refresh : Bool = false @@ -120,16 +130,27 @@ class Config # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729) @[YAML::Field(converter: Preferences::FamilyConverter)] property force_resolve : Socket::Family = Socket::Family::UNSPEC + + # External signature solver server socket (either a path to a UNIX domain socket or "<IP>:<Port>") + property signature_server : String? = nil + # Port to listen for connections (overridden by command line argument) property port : Int32 = 3000 # Host to bind (overridden by command line argument) property host_binding : String = "0.0.0.0" # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) property pool_size : Int32 = 100 + # HTTP Proxy configuration + property http_proxy : HTTPProxyConfig? = nil # Use Innertube's transcripts API instead of timedtext for closed captions property use_innertube_for_captions : Bool = false + # visitor data ID for Google session + property visitor_data : String? = nil + # poToken for passing bot attestation + property po_token : String? = nil + # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] property cookies : HTTP::Cookies = HTTP::Cookies.new @@ -163,6 +184,9 @@ class Config config = Config.from_yaml(config_yaml) # Update config from env vars (upcased and prefixed with "INVIDIOUS_") + # + # Also checks if any top-level config options are set to "CHANGE_ME!!" + # TODO: Support non-top-level config options such as the ones in DBConfig {% for ivar in Config.instance_vars %} {% env_id = "INVIDIOUS_#{ivar.id.upcase}" %} @@ -199,6 +223,12 @@ class Config exit(1) end end + + # Warn when any config attribute is set to "CHANGE_ME!!" + if config.{{ivar.id}} == "CHANGE_ME!!" + puts "Config: The value of '#{ {{ivar.stringify}} }' needs to be changed!!" + exit(1) + end {% end %} # HMAC_key is mandatory @@ -206,9 +236,6 @@ class Config if config.hmac_key.empty? puts "Config: 'hmac_key' is required/can't be empty" exit(1) - elsif config.hmac_key == "CHANGE_ME!!" - puts "Config: The value of 'hmac_key' needs to be changed!!" - exit(1) end # Build database_url from db.* if it's not set directly diff --git a/src/invidious/database/playlists.cr b/src/invidious/database/playlists.cr index c6754a1e..08aa719a 100644 --- a/src/invidious/database/playlists.cr +++ b/src/invidious/database/playlists.cr @@ -140,6 +140,7 @@ module Invidious::Database::Playlists request = <<-SQL SELECT id,title FROM playlists WHERE author = $1 AND id LIKE 'IV%' + ORDER BY title SQL PG_DB.query_all(request, email, as: {String, String}) diff --git a/src/invidious/database/users.cr b/src/invidious/database/users.cr index d54e6a76..4a3056ea 100644 --- a/src/invidious/database/users.cr +++ b/src/invidious/database/users.cr @@ -119,15 +119,15 @@ module Invidious::Database::Users # Update (notifs) # ------------------- - def add_notification(video : ChannelVideo) + def add_multiple_notifications(channel_id : String, video_ids : Array(String)) request = <<-SQL UPDATE users - SET notifications = array_append(notifications, $1), + SET notifications = array_cat(notifications, $1), feed_needs_update = true WHERE $2 = ANY(subscriptions) SQL - PG_DB.exec(request, video.id, video.ucid) + PG_DB.exec(request, video_ids, channel_id) end def remove_notification(user : User, vid : String) @@ -154,14 +154,14 @@ module Invidious::Database::Users # Update (misc) # ------------------- - def feed_needs_update(video : ChannelVideo) + def feed_needs_update(channel_id : String) request = <<-SQL UPDATE users SET feed_needs_update = true WHERE $1 = ANY(subscriptions) SQL - PG_DB.exec(request, video.ucid) + PG_DB.exec(request, channel_id) end def update_preferences(user : User) diff --git a/src/invidious/frontend/comments_youtube.cr b/src/invidious/frontend/comments_youtube.cr index aecac87f..a0e1d783 100644 --- a/src/invidious/frontend/comments_youtube.cr +++ b/src/invidious/frontend/comments_youtube.cr @@ -149,12 +149,12 @@ module Invidious::Frontend::Comments if comments["videoId"]? html << <<-END_HTML - <a href="https://www.youtube.com/watch?v=#{comments["videoId"]}&lc=#{child["commentId"]}" title="#{translate(locale, "YouTube comment permalink")}">[YT]</a> + <a rel="noreferrer noopener" href="https://www.youtube.com/watch?v=#{comments["videoId"]}&lc=#{child["commentId"]}" title="#{translate(locale, "YouTube comment permalink")}">[YT]</a> | END_HTML elsif comments["authorId"]? html << <<-END_HTML - <a href="https://www.youtube.com/channel/#{comments["authorId"]}/community?lb=#{child["commentId"]}" title="#{translate(locale, "YouTube comment permalink")}">[YT]</a> + <a rel="noreferrer noopener" href="https://www.youtube.com/channel/#{comments["authorId"]}/community?lb=#{child["commentId"]}" title="#{translate(locale, "YouTube comment permalink")}">[YT]</a> | END_HTML end diff --git a/src/invidious/frontend/misc.cr b/src/invidious/frontend/misc.cr index 43ba9f5c..7a6cf79d 100644 --- a/src/invidious/frontend/misc.cr +++ b/src/invidious/frontend/misc.cr @@ -6,9 +6,9 @@ module Invidious::Frontend::Misc if prefs.automatic_instance_redirect current_page = env.get?("current_page").as(String) - redirect_url = "/redirect?referer=#{current_page}" + return "/redirect?referer=#{current_page}" else - redirect_url = "https://redirect.invidious.io#{env.request.resource}" + return "https://redirect.invidious.io#{env.request.resource}" end end end diff --git a/src/invidious/frontend/pagination.cr b/src/invidious/frontend/pagination.cr index 3f931f4e..a29f5936 100644 --- a/src/invidious/frontend/pagination.cr +++ b/src/invidious/frontend/pagination.cr @@ -3,6 +3,24 @@ require "uri" module Invidious::Frontend::Pagination extend self + private def first_page(str : String::Builder, locale : String?, url : String) + str << %(<a href=") << url << %(" class="pure-button pure-button-secondary">) + + if locale_is_rtl?(locale) + # Inverted arrow ("first" points to the right) + str << translate(locale, "First page") + str << " " + str << %(<i class="icon ion-ios-arrow-forward"></i>) + else + # Regular arrow ("first" points to the left) + str << %(<i class="icon ion-ios-arrow-back"></i>) + str << " " + str << translate(locale, "First page") + end + + str << "</a>" + end + private def previous_page(str : String::Builder, locale : String?, url : String) # Link str << %(<a href=") << url << %(" class="pure-button pure-button-secondary">) @@ -72,18 +90,24 @@ module Invidious::Frontend::Pagination end end - def nav_ctoken(locale : String?, *, base_url : String | URI, ctoken : String?) + def nav_ctoken(locale : String?, *, base_url : String | URI, ctoken : String?, first_page : Bool, params : URI::Params) return String.build do |str| str << %(<div class="h-box">\n) str << %(<div class="page-nav-container flexible">\n) - str << %(<div class="page-prev-container flex-left"></div>\n) + str << %(<div class="page-prev-container flex-left">) + + if !first_page + self.first_page(str, locale, base_url.to_s) + end + + str << %(</div>\n) str << %(<div class="page-next-container flex-right">) if !ctoken.nil? - params_next = URI::Params{"continuation" => ctoken} - url_next = HttpServer::Utils.add_params_to_url(base_url, params_next) + params["continuation"] = ctoken + url_next = HttpServer::Utils.add_params_to_url(base_url, params) self.next_page(str, locale, url_next.to_s) end diff --git a/src/invidious/frontend/watch_page.cr b/src/invidious/frontend/watch_page.cr index c8cb7110..2e2f6ad0 100644 --- a/src/invidious/frontend/watch_page.cr +++ b/src/invidious/frontend/watch_page.cr @@ -13,7 +13,7 @@ module Invidious::Frontend::WatchPage @full_videos, @video_streams, @audio_streams, - @captions + @captions, ) end end diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr index bf56d826..fec3f62c 100644 --- a/src/invidious/helpers/crystal_class_overrides.cr +++ b/src/invidious/helpers/crystal_class_overrides.cr @@ -3,9 +3,9 @@ # IPv6 addresses. # class TCPSocket - def initialize(host : String, port, dns_timeout = nil, connect_timeout = nil, family = Socket::Family::UNSPEC) + def initialize(host, port, dns_timeout = nil, connect_timeout = nil, blocking = false, family = Socket::Family::UNSPEC) Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo| - super(addrinfo.family, addrinfo.type, addrinfo.protocol) + super(addrinfo.family, addrinfo.type, addrinfo.protocol, blocking) connect(addrinfo, timeout: connect_timeout) do |error| close error @@ -26,7 +26,7 @@ class HTTP::Client end hostname = @host.starts_with?('[') && @host.ends_with?(']') ? @host[1..-2] : @host - io = TCPSocket.new hostname, @port, @dns_timeout, @connect_timeout, @family + io = TCPSocket.new hostname, @port, @dns_timeout, @connect_timeout, family: @family io.read_timeout = @read_timeout if @read_timeout io.write_timeout = @write_timeout if @write_timeout io.sync = false @@ -35,7 +35,7 @@ class HTTP::Client if tls = @tls tcp_socket = io begin - io = OpenSSL::SSL::Socket::Client.new(tcp_socket, context: tls, sync_close: true, hostname: @host) + io = OpenSSL::SSL::Socket::Client.new(tcp_socket, context: tls, sync_close: true, hostname: @host.rchop('.')) rescue exc # don't leak the TCP socket when the SSL connection failed tcp_socket.close diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr index 21b789bc..900cb0c6 100644 --- a/src/invidious/helpers/errors.cr +++ b/src/invidious/helpers/errors.cr @@ -43,6 +43,8 @@ def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exce # URLs for the error message below url_faq = "https://github.com/iv-org/documentation/blob/master/docs/faq.md" url_search_issues = "https://github.com/iv-org/invidious/issues" + url_search_issues += "?q=is:issue+is:open+" + url_search_issues += URI.encode_www_form("[Bug] #{issue_title}") url_switch = "https://redirect.invidious.io" + env.request.resource @@ -128,7 +130,7 @@ def error_json_helper( env : HTTP::Server::Context, status_code : Int32, exception : Exception, - additional_fields : Hash(String, Object) | Nil = nil + additional_fields : Hash(String, Object) | Nil = nil, ) if exception.is_a?(InfoException) return error_json_helper(env, status_code, exception.message || "", additional_fields) @@ -150,7 +152,7 @@ def error_json_helper( env : HTTP::Server::Context, status_code : Int32, message : String, - additional_fields : Hash(String, Object) | Nil = nil + additional_fields : Hash(String, Object) | Nil = nil, ) env.response.content_type = "application/json" env.response.status_code = status_code @@ -190,7 +192,7 @@ def error_redirect_helper(env : HTTP::Server::Context) <a href="/redirect?referer=#{env.get("current_page")}">#{switch_instance}</a> </li> <li> - <a href="https://youtube.com#{env.request.resource}">#{go_to_youtube}</a> + <a rel="noreferrer noopener" href="https://youtube.com#{env.request.resource}">#{go_to_youtube}</a> </li> </ul> END_HTML diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index 174f620d..13ea9fe9 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -27,6 +27,7 @@ class Kemal::RouteHandler # Processes the route if it's a match. Otherwise renders 404. private def process_request(context) raise Kemal::Exceptions::RouteNotFound.new(context) unless context.route_found? + return if context.response.closed? content = context.route.handler.call(context) if !Kemal.config.error_handlers.empty? && Kemal.config.error_handlers.has_key?(context.response.status_code) && exclude_match?(context) @@ -97,7 +98,7 @@ class AuthHandler < Kemal::Handler if token = env.request.headers["Authorization"]? token = JSON.parse(URI.decode_www_form(token.lchop("Bearer "))) session = URI.decode_www_form(token["session"].as_s) - scopes, expire, signature = validate_request(token, session, env.request, HMAC_KEY, nil) + scopes, _, _ = validate_request(token, session, env.request, HMAC_KEY, nil) if email = Invidious::Database::SessionIDs.select_email(session) user = Invidious::Database::Users.select!(email: email) diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index 23a1aafc..1ba3ea61 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -1,8 +1,22 @@ +# Languages requiring a better level of translation (at least 20%) +# to be added to the list below: +# +# "af" => "", # Afrikaans +# "az" => "", # Azerbaijani +# "be" => "", # Belarusian +# "bn_BD" => "", # Bengali (Bangladesh) +# "ia" => "", # Interlingua +# "or" => "", # Odia +# "tk" => "", # Turkmen +# "tok => "", # Toki Pona +# LOCALES_LIST = { "ar" => "العربية", # Arabic + "bg" => "български", # Bulgarian "bn" => "বাংলা", # Bengali "ca" => "Català", # Catalan "cs" => "Čeština", # Czech + "cy" => "Cymraeg", # Welsh "da" => "Dansk", # Danish "de" => "Deutsch", # German "el" => "Ελληνικά", # Greek @@ -23,6 +37,7 @@ LOCALES_LIST = { "it" => "Italiano", # Italian "ja" => "日本語", # Japanese "ko" => "한국어", # Korean + "lmo" => "Lombard", # Lombard "lt" => "Lietuvių", # Lithuanian "nb-NO" => "Norsk bokmål", # Norwegian Bokmål "nl" => "Nederlands", # Dutch diff --git a/src/invidious/helpers/i18next.cr b/src/invidious/helpers/i18next.cr index 9f4077e1..684e6d14 100644 --- a/src/invidious/helpers/i18next.cr +++ b/src/invidious/helpers/i18next.cr @@ -95,7 +95,6 @@ module I18next::Plurals "hr" => PluralForms::Special_Hungarian_Serbian, "it" => PluralForms::Special_Spanish_Italian, "pt" => PluralForms::Special_French_Portuguese, - "pt" => PluralForms::Special_French_Portuguese, "sr" => PluralForms::Special_Hungarian_Serbian, } @@ -189,7 +188,7 @@ module I18next::Plurals # Emulate the `rule.numbers.size == 2 && rule.numbers[0] == 1` check # from original i18next code - private def is_simple_plural(form : PluralForms) : Bool + private def simple_plural?(form : PluralForms) : Bool case form when .single_gt_one? then return true when .single_not_one? then return true @@ -211,7 +210,7 @@ module I18next::Plurals idx = SuffixIndex.get_index(plural_form, count) # Simple plurals are handled differently in all versions (but v4) - if @simplify_plural_suffix && is_simple_plural(plural_form) + if @simplify_plural_suffix && simple_plural?(plural_form) return (idx == 1) ? "_plural" : "" end @@ -262,9 +261,9 @@ module I18next::Plurals when .special_hebrew? then return special_hebrew(count) when .special_odia? then return special_odia(count) # Mixed v3/v4 forms - when .special_spanish_italian? then return special_cldr_Spanish_Italian(count) - when .special_french_portuguese? then return special_cldr_French_Portuguese(count) - when .special_hungarian_serbian? then return special_cldr_Hungarian_Serbian(count) + when .special_spanish_italian? then return special_cldr_spanish_italian(count) + when .special_french_portuguese? then return special_cldr_french_portuguese(count) + when .special_hungarian_serbian? then return special_cldr_hungarian_serbian(count) else # default, if nothing matched above return 0_u8 @@ -535,7 +534,7 @@ module I18next::Plurals # # This rule is mostly compliant to CLDR v42 # - def self.special_cldr_Spanish_Italian(count : Int) : UInt8 + def self.special_cldr_spanish_italian(count : Int) : UInt8 return 0_u8 if (count == 1) # one return 1_u8 if (count != 0 && count % 1_000_000 == 0) # many return 2_u8 # other @@ -545,7 +544,7 @@ module I18next::Plurals # # This rule is mostly compliant to CLDR v42 # - def self.special_cldr_French_Portuguese(count : Int) : UInt8 + def self.special_cldr_french_portuguese(count : Int) : UInt8 return 0_u8 if (count == 0 || count == 1) # one return 1_u8 if (count % 1_000_000 == 0) # many return 2_u8 # other @@ -555,7 +554,7 @@ module I18next::Plurals # # This rule is mostly compliant to CLDR v42 # - def self.special_cldr_Hungarian_Serbian(count : Int) : UInt8 + def self.special_cldr_hungarian_serbian(count : Int) : UInt8 n_mod_10 = count % 10 n_mod_100 = count % 100 diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr index e2e50905..03349595 100644 --- a/src/invidious/helpers/logger.cr +++ b/src/invidious/helpers/logger.cr @@ -1,3 +1,5 @@ +require "colorize" + enum LogLevel All = 0 Trace = 1 @@ -10,7 +12,9 @@ enum LogLevel end class Invidious::LogHandler < Kemal::BaseLogHandler - def initialize(@io : IO = STDOUT, @level = LogLevel::Debug) + def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, use_color : Bool = true) + Colorize.enabled = use_color + Colorize.on_tty_only! end def call(context : HTTP::Server::Context) @@ -34,28 +38,27 @@ class Invidious::LogHandler < Kemal::BaseLogHandler context end - def puts(message : String) - @io << message << '\n' - @io.flush - end - def write(message : String) @io << message @io.flush end - def set_log_level(level : String) - @level = LogLevel.parse(level) - end - - def set_log_level(level : LogLevel) - @level = level + def color(level) + case level + when LogLevel::Trace then :cyan + when LogLevel::Debug then :green + when LogLevel::Info then :white + when LogLevel::Warn then :yellow + when LogLevel::Error then :red + when LogLevel::Fatal then :magenta + else :default + end end {% for level in %w(trace debug info warn error fatal) %} def {{level.id}}(message : String) if LogLevel::{{level.id.capitalize}} >= @level - puts("#{Time.utc} [{{level.id}}] #{message}") + puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}}))) end end {% end %} diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index 31a3cf44..f8e8f187 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -1,3 +1,16 @@ +@[Flags] +enum VideoBadges + LiveNow + Premium + ThreeD + FourK + New + EightK + VR180 + VR360 + ClosedCaptions +end + struct SearchVideo include DB::Serializable @@ -9,10 +22,10 @@ struct SearchVideo property views : Int64 property description_html : String property length_seconds : Int32 - property live_now : Bool - property premium : Bool property premiere_timestamp : Time? property author_verified : Bool + property author_thumbnail : String? + property badges : VideoBadges def to_xml(auto_generated, query_params, xml : XML::Builder) query_params["v"] = self.id @@ -76,6 +89,24 @@ struct SearchVideo json.field "authorUrl", "/channel/#{self.ucid}" json.field "authorVerified", self.author_verified + author_thumbnail = self.author_thumbnail + + if author_thumbnail + json.field "authorThumbnails" do + json.array do + qualities = {32, 48, 76, 100, 176, 512} + + qualities.each do |quality| + json.object do + json.field "url", author_thumbnail.gsub(/=s\d+/, "=s#{quality}") + json.field "width", quality + json.field "height", quality + end + end + end + end + end + json.field "videoThumbnails" do Invidious::JSONify::APIv1.thumbnails(json, self.id) end @@ -88,13 +119,20 @@ struct SearchVideo json.field "published", self.published.to_unix json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) json.field "lengthSeconds", self.length_seconds - json.field "liveNow", self.live_now - json.field "premium", self.premium - json.field "isUpcoming", self.is_upcoming + json.field "liveNow", self.badges.live_now? + json.field "premium", self.badges.premium? + json.field "isUpcoming", self.upcoming? if self.premiere_timestamp json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix end + json.field "isNew", self.badges.new? + json.field "is4k", self.badges.four_k? + json.field "is8k", self.badges.eight_k? + json.field "isVr180", self.badges.vr180? + json.field "isVr360", self.badges.vr360? + json.field "is3d", self.badges.three_d? + json.field "hasCaptions", self.badges.closed_captions? end end @@ -109,7 +147,7 @@ struct SearchVideo to_json(nil, json) end - def is_upcoming + def upcoming? premiere_timestamp ? true : false end end @@ -204,7 +242,7 @@ struct SearchChannel qualities.each do |quality| json.object do - json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}") + json.field "url", self.author_thumbnail.gsub(/=s\d+/, "=s#{quality}") json.field "width", quality json.field "height", quality end diff --git a/src/invidious/helpers/sig_helper.cr b/src/invidious/helpers/sig_helper.cr new file mode 100644 index 00000000..6d198a42 --- /dev/null +++ b/src/invidious/helpers/sig_helper.cr @@ -0,0 +1,349 @@ +require "uri" +require "socket" +require "socket/tcp_socket" +require "socket/unix_socket" + +{% if flag?(:advanced_debug) %} + require "io/hexdump" +{% end %} + +private alias NetworkEndian = IO::ByteFormat::NetworkEndian + +module Invidious::SigHelper + enum UpdateStatus + Updated + UpdateNotRequired + Error + end + + # ------------------- + # Payload types + # ------------------- + + abstract struct Payload + end + + struct StringPayload < Payload + getter string : String + + def initialize(str : String) + raise Exception.new("SigHelper: String can't be empty") if str.empty? + @string = str + end + + def self.from_bytes(slice : Bytes) + size = IO::ByteFormat::NetworkEndian.decode(UInt16, slice) + if size == 0 # Error code + raise Exception.new("SigHelper: Server encountered an error") + end + + if (slice.bytesize - 2) != size + raise Exception.new("SigHelper: String size mismatch") + end + + if str = String.new(slice[2..]) + return self.new(str) + else + raise Exception.new("SigHelper: Can't read string from socket") + end + end + + def to_io(io) + # `.to_u16` raises if there is an overflow during the conversion + io.write_bytes(@string.bytesize.to_u16, NetworkEndian) + io.write(@string.to_slice) + end + end + + private enum Opcode + FORCE_UPDATE = 0 + DECRYPT_N_SIGNATURE = 1 + DECRYPT_SIGNATURE = 2 + GET_SIGNATURE_TIMESTAMP = 3 + GET_PLAYER_STATUS = 4 + PLAYER_UPDATE_TIMESTAMP = 5 + end + + private record Request, + opcode : Opcode, + payload : Payload? + + # ---------------------- + # High-level functions + # ---------------------- + + class Client + @mux : Multiplexor + + def initialize(uri_or_path) + @mux = Multiplexor.new(uri_or_path) + end + + # Forces the server to re-fetch the YouTube player, and extract the necessary + # components from it (nsig function code, sig function code, signature timestamp). + def force_update : UpdateStatus + request = Request.new(Opcode::FORCE_UPDATE, nil) + + value = send_request(request) do |bytes| + IO::ByteFormat::NetworkEndian.decode(UInt16, bytes) + end + + case value + when 0x0000 then return UpdateStatus::Error + when 0xFFFF then return UpdateStatus::UpdateNotRequired + when 0xF44F then return UpdateStatus::Updated + else + code = value.nil? ? "nil" : value.to_s(base: 16) + raise Exception.new("SigHelper: Invalid status code received #{code}") + end + end + + # Decrypt a provided n signature using the server's current nsig function + # code, and return the result (or an error). + def decrypt_n_param(n : String) : String? + request = Request.new(Opcode::DECRYPT_N_SIGNATURE, StringPayload.new(n)) + + n_dec = self.send_request(request) do |bytes| + StringPayload.from_bytes(bytes).string + end + + return n_dec + end + + # Decrypt a provided s signature using the server's current sig function + # code, and return the result (or an error). + def decrypt_sig(sig : String) : String? + request = Request.new(Opcode::DECRYPT_SIGNATURE, StringPayload.new(sig)) + + sig_dec = self.send_request(request) do |bytes| + StringPayload.from_bytes(bytes).string + end + + return sig_dec + end + + # Return the signature timestamp from the server's current player + def get_signature_timestamp : UInt64? + request = Request.new(Opcode::GET_SIGNATURE_TIMESTAMP, nil) + + return self.send_request(request) do |bytes| + IO::ByteFormat::NetworkEndian.decode(UInt64, bytes) + end + end + + # Return the current player's version + def get_player : UInt32? + request = Request.new(Opcode::GET_PLAYER_STATUS, nil) + + return self.send_request(request) do |bytes| + has_player = (bytes[0] == 0xFF) + player_version = IO::ByteFormat::NetworkEndian.decode(UInt32, bytes[1..4]) + has_player ? player_version : nil + end + end + + # Return when the player was last updated + def get_player_timestamp : UInt64? + request = Request.new(Opcode::PLAYER_UPDATE_TIMESTAMP, nil) + + return self.send_request(request) do |bytes| + IO::ByteFormat::NetworkEndian.decode(UInt64, bytes) + end + end + + private def send_request(request : Request, &) + channel = @mux.send(request) + slice = channel.receive + return yield slice + rescue ex + LOGGER.debug("SigHelper: Error when sending a request") + LOGGER.trace(ex.inspect_with_backtrace) + return nil + end + end + + # --------------------- + # Low level functions + # --------------------- + + class Multiplexor + alias TransactionID = UInt32 + record Transaction, channel = ::Channel(Bytes).new + + @prng = Random.new + @mutex = Mutex.new + @queue = {} of TransactionID => Transaction + + @conn : Connection + @uri_or_path : String + + def initialize(@uri_or_path) + @conn = Connection.new(uri_or_path) + listen + end + + def listen : Nil + raise "Socket is closed" if @conn.closed? + + LOGGER.debug("SigHelper: Multiplexor listening") + + spawn do + loop do + begin + receive_data + rescue ex + LOGGER.info("SigHelper: Connection to helper died with '#{ex.message}' trying to reconnect...") + # We close the socket because for some reason is not closed. + @conn.close + loop do + begin + @conn = Connection.new(@uri_or_path) + LOGGER.info("SigHelper: Reconnected to SigHelper!") + rescue ex + LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}'. Retrying") + sleep 500.milliseconds + next + end + break if !@conn.closed? + end + end + Fiber.yield + end + end + end + + def send(request : Request) + transaction = Transaction.new + transaction_id = @prng.rand(TransactionID) + + # Add transaction to queue + @mutex.synchronize do + # On a 32-bits random integer, this should never happen. Though, just in case, ... + if @queue[transaction_id]? + raise Exception.new("SigHelper: Duplicate transaction ID! You got a shiny pokemon!") + end + + @queue[transaction_id] = transaction + end + + write_packet(transaction_id, request) + + return transaction.channel + end + + def receive_data + transaction_id, slice = read_packet + + @mutex.synchronize do + if transaction = @queue.delete(transaction_id) + # Remove transaction from queue and send data to the channel + transaction.channel.send(slice) + LOGGER.trace("SigHelper: Transaction unqueued and data sent to channel") + else + raise Exception.new("SigHelper: Received transaction was not in queue") + end + end + end + + # Read a single packet from the socket + private def read_packet : {TransactionID, Bytes} + # Header + transaction_id = @conn.read_bytes(UInt32, NetworkEndian) + length = @conn.read_bytes(UInt32, NetworkEndian) + + LOGGER.trace("SigHelper: Recv transaction 0x#{transaction_id.to_s(base: 16)} / length #{length}") + + if length > 67_000 + raise Exception.new("SigHelper: Packet longer than expected (#{length})") + end + + # Payload + slice = Bytes.new(length) + @conn.read(slice) if length > 0 + + LOGGER.trace("SigHelper: payload = #{slice}") + LOGGER.trace("SigHelper: Recv transaction 0x#{transaction_id.to_s(base: 16)} - Done") + + return transaction_id, slice + end + + # Write a single packet to the socket + private def write_packet(transaction_id : TransactionID, request : Request) + LOGGER.trace("SigHelper: Send transaction 0x#{transaction_id.to_s(base: 16)} / opcode #{request.opcode}") + + io = IO::Memory.new(1024) + io.write_bytes(request.opcode.to_u8, NetworkEndian) + io.write_bytes(transaction_id, NetworkEndian) + + if payload = request.payload + payload.to_io(io) + end + + @conn.send(io) + @conn.flush + + LOGGER.trace("SigHelper: Send transaction 0x#{transaction_id.to_s(base: 16)} - Done") + end + end + + class Connection + @socket : UNIXSocket | TCPSocket + + {% if flag?(:advanced_debug) %} + @io : IO::Hexdump + {% end %} + + def initialize(host_or_path : String) + case host_or_path + when .starts_with?('/') + # Make sure that the file exists + if File.exists?(host_or_path) + @socket = UNIXSocket.new(host_or_path) + else + raise Exception.new("SigHelper: '#{host_or_path}' no such file") + end + when .starts_with?("tcp://") + uri = URI.parse(host_or_path) + @socket = TCPSocket.new(uri.host.not_nil!, uri.port.not_nil!) + else + uri = URI.parse("tcp://#{host_or_path}") + @socket = TCPSocket.new(uri.host.not_nil!, uri.port.not_nil!) + end + LOGGER.info("SigHelper: Using helper at '#{host_or_path}'") + + {% if flag?(:advanced_debug) %} + @io = IO::Hexdump.new(@socket, output: STDERR, read: true, write: true) + {% end %} + + @socket.sync = false + @socket.blocking = false + end + + def closed? : Bool + return @socket.closed? + end + + def close : Nil + @socket.close if !@socket.closed? + end + + def flush(*args, **options) + @socket.flush(*args, **options) + end + + def send(*args, **options) + @socket.send(*args, **options) + end + + # Wrap IO functions, with added debug tooling if needed + {% for function in %w(read read_bytes write write_bytes) %} + def {{function.id}}(*args, **options) + {% if flag?(:advanced_debug) %} + @io.{{function.id}}(*args, **options) + {% else %} + @socket.{{function.id}}(*args, **options) + {% end %} + end + {% end %} + end +end diff --git a/src/invidious/helpers/signatures.cr b/src/invidious/helpers/signatures.cr index ee09415b..82a28fc0 100644 --- a/src/invidious/helpers/signatures.cr +++ b/src/invidious/helpers/signatures.cr @@ -1,73 +1,53 @@ -alias SigProc = Proc(Array(String), Int32, Array(String)) +require "http/params" +require "./sig_helper" -struct DecryptFunction - @decrypt_function = [] of {SigProc, Int32} - @decrypt_time = Time.monotonic +class Invidious::DecryptFunction + @last_update : Time = Time.utc - 42.days - def initialize(@use_polling = true) + def initialize(uri_or_path) + @client = SigHelper::Client.new(uri_or_path) + self.check_update end - def update_decrypt_function - @decrypt_function = fetch_decrypt_function - end - - private def fetch_decrypt_function(id = "CvFH_6DNRCY") - document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en").body - url = document.match(/src="(?<url>\/s\/player\/[^\/]+\/player_ias[^\/]+\/en_US\/base.js)"/).not_nil!["url"] - player = YT_POOL.client &.get(url).body - - function_name = player.match(/^(?<name>[^=]+)=function\(\w\){\w=\w\.split\(""\);[^\. ]+\.[^( ]+/m).not_nil!["name"] - function_body = player.match(/^#{Regex.escape(function_name)}=function\(\w\){(?<body>[^}]+)}/m).not_nil!["body"] - function_body = function_body.split(";")[1..-2] - - var_name = function_body[0][0, 2] - var_body = player.delete("\n").match(/var #{Regex.escape(var_name)}={(?<body>(.*?))};/).not_nil!["body"] - - operations = {} of String => SigProc - var_body.split("},").each do |operation| - op_name = operation.match(/^[^:]+/).not_nil![0] - op_body = operation.match(/\{[^}]+/).not_nil![0] - - case op_body - when "{a.reverse()" - operations[op_name] = ->(a : Array(String), _b : Int32) { a.reverse } - when "{a.splice(0,b)" - operations[op_name] = ->(a : Array(String), b : Int32) { a.delete_at(0..(b - 1)); a } - else - operations[op_name] = ->(a : Array(String), b : Int32) { c = a[0]; a[0] = a[b % a.size]; a[b % a.size] = c; a } - end - end + def check_update + # If we have updated in the last 5 minutes, do nothing + return if (Time.utc - @last_update) < 5.minutes - decrypt_function = [] of {SigProc, Int32} - function_body.each do |function| - function = function.lchop(var_name).delete("[].") + # Get the amount of time elapsed since when the player was updated, in the + # event where multiple invidious processes are run in parallel. + update_time_elapsed = (@client.get_player_timestamp || 301).seconds - op_name = function.match(/[^\(]+/).not_nil![0] - value = function.match(/\(\w,(?<value>[\d]+)\)/).not_nil!["value"].to_i - - decrypt_function << {operations[op_name], value} + if update_time_elapsed > 5.minutes + LOGGER.debug("Signature: Player might be outdated, updating") + @client.force_update + @last_update = Time.utc end - - return decrypt_function end - def decrypt_signature(fmt : Hash(String, JSON::Any)) - return "" if !fmt["s"]? || !fmt["sp"]? - - sp = fmt["sp"].as_s - sig = fmt["s"].as_s.split("") - if !@use_polling - now = Time.monotonic - if now - @decrypt_time > 60.seconds || @decrypt_function.size == 0 - @decrypt_function = fetch_decrypt_function - @decrypt_time = Time.monotonic - end - end + def decrypt_nsig(n : String) : String? + self.check_update + return @client.decrypt_n_param(n) + rescue ex + LOGGER.debug(ex.message || "Signature: Unknown error") + LOGGER.trace(ex.inspect_with_backtrace) + return nil + end - @decrypt_function.each do |proc, value| - sig = proc.call(sig, value) - end + def decrypt_signature(str : String) : String? + self.check_update + return @client.decrypt_sig(str) + rescue ex + LOGGER.debug(ex.message || "Signature: Unknown error") + LOGGER.trace(ex.inspect_with_backtrace) + return nil + end - return "&#{sp}=#{sig.join("")}" + def get_sts : UInt64? + self.check_update + return @client.get_signature_timestamp + rescue ex + LOGGER.debug(ex.message || "Signature: Unknown error") + LOGGER.trace(ex.inspect_with_backtrace) + return nil end end diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index e438e3b9..4d9bb28d 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -52,9 +52,9 @@ def recode_length_seconds(time) end def decode_interval(string : String) : Time::Span - rawMinutes = string.try &.to_i32? + raw_minutes = string.try &.to_i32? - if !rawMinutes + if !raw_minutes hours = /(?<hours>\d+)h/.match(string).try &.["hours"].try &.to_i32 hours ||= 0 @@ -63,7 +63,7 @@ def decode_interval(string : String) : Time::Span time = Time::Span.new(hours: hours, minutes: minutes) else - time = Time::Span.new(minutes: rawMinutes) + time = Time::Span.new(minutes: raw_minutes) end return time @@ -323,68 +323,6 @@ def parse_range(range) return 0_i64, nil end -def fetch_random_instance - begin - instance_api_client = make_client(URI.parse("https://api.invidious.io")) - - # Timeouts - instance_api_client.connect_timeout = 10.seconds - instance_api_client.dns_timeout = 10.seconds - - instance_list = JSON.parse(instance_api_client.get("/instances.json").body).as_a - instance_api_client.close - rescue Socket::ConnectError | IO::TimeoutError | JSON::ParseException - instance_list = [] of JSON::Any - end - - filtered_instance_list = [] of String - - instance_list.each do |data| - # TODO Check if current URL is onion instance and use .onion types if so. - if data[1]["type"] == "https" - # Instances can have statistics disabled, which is an requirement of version validation. - # as_nil? doesn't exist. Thus we'll have to handle the error raised if as_nil fails. - begin - data[1]["stats"].as_nil - next - rescue TypeCastError - end - - # stats endpoint could also lack the software dict. - next if data[1]["stats"]["software"]?.nil? - - # Makes sure the instance isn't too outdated. - if remote_version = data[1]["stats"]?.try &.["software"]?.try &.["version"] - remote_commit_date = remote_version.as_s.match(/\d{4}\.\d{2}\.\d{2}/) - next if !remote_commit_date - - remote_commit_date = Time.parse(remote_commit_date[0], "%Y.%m.%d", Time::Location::UTC) - local_commit_date = Time.parse(CURRENT_VERSION, "%Y.%m.%d", Time::Location::UTC) - - next if (remote_commit_date - local_commit_date).abs.days > 30 - - begin - data[1]["monitor"].as_nil - health = data[1]["monitor"].as_h["dailyRatios"][0].as_h["ratio"] - filtered_instance_list << data[0].as_s if health.to_s.to_f > 90 - rescue TypeCastError - # We can't check the health if the monitoring is broken. Thus we'll just add it to the list - # and move on. Ideally we'll ignore any instance that has broken health monitoring but due to the fact that - # it's an error that often occurs with all the instances at the same time, we have to just skip the check. - filtered_instance_list << data[0].as_s - end - end - end - end - - # If for some reason no instances managed to get fetched successfully then we'll just redirect to redirect.invidious.io - if filtered_instance_list.size == 0 - return "redirect.invidious.io" - end - - return filtered_instance_list.sample(1)[0] -end - def reduce_uri(uri : URI | String, max_length : Int32 = 50, suffix : String = "…") : String str = uri.to_s.sub(/^https?:\/\//, "") if str.size > max_length diff --git a/src/invidious/http_server/utils.cr b/src/invidious/http_server/utils.cr index 222dfc4a..623a9177 100644 --- a/src/invidious/http_server/utils.cr +++ b/src/invidious/http_server/utils.cr @@ -11,11 +11,12 @@ module Invidious::HttpServer params = url.query_params params["host"] = url.host.not_nil! # Should never be nil, in theory params["region"] = region if !region.nil? + url.query_params = params if absolute - return "#{HOST_URL}#{url.request_target}?#{params}" + return "#{HOST_URL}#{url.request_target}" else - return "#{url.request_target}?#{params}" + return url.request_target end end diff --git a/src/invidious/jobs/instance_refresh_job.cr b/src/invidious/jobs/instance_refresh_job.cr new file mode 100644 index 00000000..cb4280b9 --- /dev/null +++ b/src/invidious/jobs/instance_refresh_job.cr @@ -0,0 +1,97 @@ +class Invidious::Jobs::InstanceListRefreshJob < Invidious::Jobs::BaseJob + # We update the internals of a constant as so it can be accessed from anywhere + # within the codebase + # + # "INSTANCES" => Array(Tuple(String, String)) # region, instance + + INSTANCES = {"INSTANCES" => [] of Tuple(String, String)} + + def initialize + end + + def begin + loop do + refresh_instances + LOGGER.info("InstanceListRefreshJob: Done, sleeping for 30 minutes") + sleep 30.minute + Fiber.yield + end + end + + # Refreshes the list of instances used for redirects. + # + # Does the following three checks for each instance + # - Is it a clear-net instance? + # - Is it an instance with a good uptime? + # - Is it an updated instance? + private def refresh_instances + raw_instance_list = self.fetch_instances + filtered_instance_list = [] of Tuple(String, String) + + raw_instance_list.each do |instance_data| + # TODO allow Tor hidden service instances when the current instance + # is also a hidden service. Same for i2p and any other non-clearnet instances. + begin + domain = instance_data[0] + info = instance_data[1] + stats = info["stats"] + + next unless info["type"] == "https" + next if bad_uptime?(info["monitor"]) + next if outdated?(stats["software"]["version"]) + + filtered_instance_list << {info["region"].as_s, domain.as_s} + rescue ex + if domain + LOGGER.info("InstanceListRefreshJob: failed to parse information from '#{domain}' because \"#{ex}\"\n\"#{ex.backtrace.join('\n')}\" ") + else + LOGGER.info("InstanceListRefreshJob: failed to parse information from an instance because \"#{ex}\"\n\"#{ex.backtrace.join('\n')}\" ") + end + end + end + + if !filtered_instance_list.empty? + INSTANCES["INSTANCES"] = filtered_instance_list + end + end + + # Fetches information regarding instances from api.invidious.io or an otherwise configured URL + private def fetch_instances : Array(JSON::Any) + begin + # We directly call the stdlib HTTP::Client here as it allows us to negate the effects + # of the force_resolve config option. This is needed as api.invidious.io does not support ipv6 + # and as such the following request raises if we were to use force_resolve with the ipv6 value. + instance_api_client = HTTP::Client.new(URI.parse("https://api.invidious.io")) + + # Timeouts + instance_api_client.connect_timeout = 10.seconds + instance_api_client.dns_timeout = 10.seconds + + raw_instance_list = JSON.parse(instance_api_client.get("/instances.json").body).as_a + instance_api_client.close + rescue ex : Socket::ConnectError | IO::TimeoutError | JSON::ParseException + raw_instance_list = [] of JSON::Any + end + + return raw_instance_list + end + + # Checks if the given target instance is outdated + private def outdated?(target_instance_version) : Bool + remote_commit_date = target_instance_version.as_s.match(/\d{4}\.\d{2}\.\d{2}/) + return false if !remote_commit_date + + remote_commit_date = Time.parse(remote_commit_date[0], "%Y.%m.%d", Time::Location::UTC) + local_commit_date = Time.parse(CURRENT_VERSION, "%Y.%m.%d", Time::Location::UTC) + + return (remote_commit_date - local_commit_date).abs.days > 30 + end + + # Checks if the uptime of the target instance is greater than 90% over a 30 day period + private def bad_uptime?(target_instance_health_monitor) : Bool + return true if !target_instance_health_monitor["down"].as_bool == false + return true if target_instance_health_monitor["uptime"].as_f < 90 + + return false + end +end diff --git a/src/invidious/jobs/notification_job.cr b/src/invidious/jobs/notification_job.cr index b445107b..f2c9d4be 100644 --- a/src/invidious/jobs/notification_job.cr +++ b/src/invidious/jobs/notification_job.cr @@ -1,8 +1,32 @@ +struct VideoNotification + getter video_id : String + getter channel_id : String + getter published : Time + + def_hash @channel_id, @video_id + + def ==(other) + video_id == other.video_id + end + + def self.from_video(video : ChannelVideo) : self + VideoNotification.new(video.id, video.ucid, video.published) + end + + def initialize(@video_id, @channel_id, @published) + end + + def clone : VideoNotification + VideoNotification.new(video_id.clone, channel_id.clone, published.clone) + end +end + class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob + private getter notification_channel : ::Channel(VideoNotification) private getter connection_channel : ::Channel({Bool, ::Channel(PQ::Notification)}) private getter pg_url : URI - def initialize(@connection_channel, @pg_url) + def initialize(@notification_channel, @connection_channel, @pg_url) end def begin @@ -10,6 +34,70 @@ class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob PG.connect_listen(pg_url, "notifications") { |event| connections.each(&.send(event)) } + # hash of channels to their videos (id+published) that need notifying + to_notify = Hash(String, Set(VideoNotification)).new( + ->(hash : Hash(String, Set(VideoNotification)), key : String) { + hash[key] = Set(VideoNotification).new + } + ) + notify_mutex = Mutex.new() + + # fiber to locally cache all incoming notifications (from pubsub webhooks and refresh channels job) + spawn do + begin + loop do + notification = notification_channel.receive + notify_mutex.synchronize do + to_notify[notification.channel_id] << notification + end + end + end + end + # fiber to regularly persist all cached notifications + spawn do + loop do + begin + LOGGER.debug("NotificationJob: waking up") + cloned = {} of String => Set(VideoNotification) + notify_mutex.synchronize do + cloned = to_notify.clone + to_notify.clear + end + + cloned.each do |channel_id, notifications| + if notifications.empty? + next + end + + LOGGER.info("NotificationJob: updating channel #{channel_id} with #{notifications.size} notifications") + if CONFIG.enable_user_notifications + video_ids = notifications.map { |n| n.video_id } + Invidious::Database::Users.add_multiple_notifications(channel_id, video_ids) + PG_DB.using_connection do |conn| + notifications.each do |n| + # Deliver notifications to `/api/v1/auth/notifications` + payload = { + "topic" => n.channel_id, + "videoId" => n.video_id, + "published" => n.published.to_unix, + }.to_json + conn.exec("NOTIFY notifications, E'#{payload}'") + end + end + else + Invidious::Database::Users.feed_needs_update(channel_id) + end + end + + LOGGER.trace("NotificationJob: Done, sleeping") + rescue ex + LOGGER.error("NotificationJob: #{ex.message}") + end + sleep 1.minute + Fiber.yield + end + end + loop do action, connection = connection_channel.receive diff --git a/src/invidious/jobs/update_decrypt_function_job.cr b/src/invidious/jobs/update_decrypt_function_job.cr deleted file mode 100644 index 6fa0ae1b..00000000 --- a/src/invidious/jobs/update_decrypt_function_job.cr +++ /dev/null @@ -1,14 +0,0 @@ -class Invidious::Jobs::UpdateDecryptFunctionJob < Invidious::Jobs::BaseJob - def begin - loop do - begin - DECRYPT_FUNCTION.update_decrypt_function - rescue ex - LOGGER.error("UpdateDecryptFunctionJob : #{ex.message}") - ensure - sleep 1.minute - Fiber.yield - end - end - end -end diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index 0dced80b..3439ae60 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -63,7 +63,7 @@ module Invidious::JSONify::APIv1 json.field "isListed", video.is_listed json.field "liveNow", video.live_now json.field "isPostLiveDvr", video.post_live_dvr - json.field "isUpcoming", video.is_upcoming + json.field "isUpcoming", video.upcoming? if video.premiere_timestamp json.field "premiereTimestamp", video.premiere_timestamp.try &.to_unix @@ -109,30 +109,36 @@ module Invidious::JSONify::APIv1 # On livestreams, it's not present, so always fall back to the # current unix timestamp (up to mS precision) for compatibility. last_modified = fmt["lastModified"]? - last_modified ||= "#{Time.utc.to_unix_ms.to_s}000" + last_modified ||= "#{Time.utc.to_unix_ms}000" json.field "lmt", last_modified json.field "projectionType", fmt["projectionType"] - if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) - fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 + height = fmt["height"]?.try &.as_i + width = fmt["width"]?.try &.as_i + + fps = fmt["fps"]?.try &.as_i + + if fps json.field "fps", fps - json.field "container", fmt_info["ext"] - json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] + end - if fmt_info["height"]? - json.field "resolution", "#{fmt_info["height"]}p" + if height && width + json.field "size", "#{width}x#{height}" + json.field "resolution", "#{height}p" - quality_label = "#{fmt_info["height"]}p" - if fps > 30 - quality_label += "60" - end - json.field "qualityLabel", quality_label + quality_label = "#{width > height ? height : width}p" - if fmt_info["width"]? - json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" - end + if fps && fps > 30 + quality_label += fps.to_s end + + json.field "qualityLabel", quality_label + end + + if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) + json.field "container", fmt_info["ext"] + json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] end # Livestream chunk infos @@ -156,33 +162,44 @@ module Invidious::JSONify::APIv1 json.array do video.fmt_stream.each do |fmt| json.object do - json.field "url", fmt["url"] + if proxy + json.field "url", Invidious::HttpServer::Utils.proxy_video_url( + fmt["url"].to_s, absolute: true + ) + else + json.field "url", fmt["url"] + end json.field "itag", fmt["itag"].as_i.to_s json.field "type", fmt["mimeType"] json.field "quality", fmt["quality"] json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]? - fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) - if fmt_info - fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 + height = fmt["height"]?.try &.as_i + width = fmt["width"]?.try &.as_i + + fps = fmt["fps"]?.try &.as_i + + if fps json.field "fps", fps - json.field "container", fmt_info["ext"] - json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] + end - if fmt_info["height"]? - json.field "resolution", "#{fmt_info["height"]}p" + if height && width + json.field "size", "#{width}x#{height}" + json.field "resolution", "#{height}p" - quality_label = "#{fmt_info["height"]}p" - if fps > 30 - quality_label += "60" - end - json.field "qualityLabel", quality_label + quality_label = "#{width > height ? height : width}p" - if fmt_info["width"]? - json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" - end + if fps && fps > 30 + quality_label += fps.to_s end + + json.field "qualityLabel", quality_label + end + + if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) + json.field "container", fmt_info["ext"] + json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] end end end @@ -250,6 +267,12 @@ module Invidious::JSONify::APIv1 json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i json.field "viewCountText", rv["short_view_count"]? json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 + json.field "published", rv["published"]? + if !rv["published"]?.nil? + json.field "publishedText", translate(locale, "`x` ago", recode_date(Time.parse_rfc3339(rv["published"].to_s), locale)) + else + json.field "publishedText", "" + end end end end @@ -260,17 +283,17 @@ module Invidious::JSONify::APIv1 def storyboards(json, id, storyboards) json.array do - storyboards.each do |storyboard| + storyboards.each do |sb| json.object do - json.field "url", "/api/v1/storyboards/#{id}?width=#{storyboard[:width]}&height=#{storyboard[:height]}" - json.field "templateUrl", storyboard[:url] - json.field "width", storyboard[:width] - json.field "height", storyboard[:height] - json.field "count", storyboard[:count] - json.field "interval", storyboard[:interval] - json.field "storyboardWidth", storyboard[:storyboard_width] - json.field "storyboardHeight", storyboard[:storyboard_height] - json.field "storyboardCount", storyboard[:storyboard_count] + json.field "url", "/api/v1/storyboards/#{id}?width=#{sb.width}&height=#{sb.height}" + json.field "templateUrl", sb.url.to_s + json.field "width", sb.width + json.field "height", sb.height + json.field "count", sb.count + json.field "interval", sb.interval + json.field "storyboardWidth", sb.columns + json.field "storyboardHeight", sb.rows + json.field "storyboardCount", sb.images_count end end end diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr index 823ca85b..28ff0ff6 100644 --- a/src/invidious/mixes.cr +++ b/src/invidious/mixes.cr @@ -81,7 +81,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil) }) end -def template_mix(mix) +def template_mix(mix, listen) html = <<-END_HTML <h3> <a href="/mix?list=#{mix["mixId"]}"> @@ -95,7 +95,7 @@ def template_mix(mix) mix["videos"].as_a.each do |video| html += <<-END_HTML <li class="pure-menu-item"> - <a href="/watch?v=#{video["videoId"]}&list=#{mix["mixId"]}"> + <a href="/watch?v=#{video["videoId"]}&list=#{mix["mixId"]}#{listen ? "&listen=1" : ""}"> <div class="thumbnail"> <img loading="lazy" class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg" alt="" /> <p class="length">#{recode_length_seconds(video["lengthSeconds"].as_i)}</p> diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 955e0855..b670c009 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -46,8 +46,14 @@ struct PlaylistVideo XML.build { |xml| to_xml(xml) } end + def to_json(locale : String?, json : JSON::Builder) + to_json(json) + end + def to_json(json : JSON::Builder, index : Int32? = nil) json.object do + json.field "type", "video" + json.field "title", self.title json.field "videoId", self.id @@ -67,6 +73,7 @@ struct PlaylistVideo end json.field "lengthSeconds", self.length_seconds + json.field "liveNow", self.live_now end end @@ -263,7 +270,7 @@ end def subscribe_playlist(user, playlist) playlist = InvidiousPlaylist.new({ - title: playlist.title.byte_slice(0, 150), + title: playlist.title[..150], id: playlist.id, author: user.email, description: "", # Max 5000 characters @@ -366,6 +373,8 @@ def fetch_playlist(plid : String) if text.includes? "video" video_count = text.gsub(/\D/, "").to_i? || 0 + elsif text.includes? "episode" + video_count = text.gsub(/\D/, "").to_i? || 0 elsif text.includes? "view" views = text.gsub(/\D/, "").to_i64? || 0_i64 else @@ -496,7 +505,7 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) return videos end -def template_playlist(playlist) +def template_playlist(playlist, listen) html = <<-END_HTML <h3> <a href="/playlist?list=#{playlist["playlistId"]}"> @@ -510,7 +519,7 @@ def template_playlist(playlist) playlist["videos"].as_a.each do |video| html += <<-END_HTML <li class="pure-menu-item" id="#{video["videoId"]}"> - <a href="/watch?v=#{video["videoId"]}&list=#{playlist["playlistId"]}&index=#{video["index"]}"> + <a href="/watch?v=#{video["videoId"]}&list=#{playlist["playlistId"]}&index=#{video["index"]}#{listen ? "&listen=1" : ""}"> <div class="thumbnail"> <img loading="lazy" class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg" alt="" /> <p class="length">#{recode_length_seconds(video["lengthSeconds"].as_i)}</p> diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr index 9d930841..c8db207c 100644 --- a/src/invidious/routes/account.cr +++ b/src/invidious/routes/account.cr @@ -53,7 +53,7 @@ module Invidious::Routes::Account return error_template(401, "Password is a required field") end - new_passwords = env.params.body.select { |k, v| k.match(/^new_password\[\d+\]$/) }.map { |k, v| v } + new_passwords = env.params.body.select { |k, _| k.match(/^new_password\[\d+\]$/) }.map { |_, v| v } if new_passwords.size <= 1 || new_passwords.uniq.size != 1 return error_template(400, "New passwords must match") @@ -240,7 +240,7 @@ module Invidious::Routes::Account return error_template(400, ex) end - scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v } + scopes = env.params.body.select { |k, _| k.match(/^scopes\[\d+\]$/) }.map { |_, v| v } callback_url = env.params.body["callbackUrl"]? expire = env.params.body["expire"]?.try &.to_i? @@ -328,17 +328,9 @@ module Invidious::Routes::Account end end - if env.params.query["action_revoke_token"]? - action = "action_revoke_token" - else - return env.redirect referer - end - - session = env.params.query["session"]? - session ||= "" - - case action - when .starts_with? "action_revoke_token" + case action = env.params.query["action"]? + when "revoke_token" + session = env.params.query["session"] Invidious::Database::SessionIDs.delete(sid: session, email: user.email) else return error_json(400, "Unsupported action #{action}") diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index d89e752c..6c4225e5 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -27,28 +27,21 @@ module Invidious::Routes::API::Manifest haltf env, status_code: response.status_code end - manifest = response.body - - manifest = manifest.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl| - url = baseurl.lchop("<BaseURL>") - url = url.rchop("</BaseURL>") - - if local - uri = URI.parse(url) - url = "#{HOST_URL}#{uri.request_target}host/#{uri.host}/" - end - + # Proxy URLs for video playback on invidious. + # Other API clients can get the original URLs by omiting `local=true`. + manifest = response.body.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl| + url = baseurl.lchop("<BaseURL>").rchop("</BaseURL>") + url = HttpServer::Utils.proxy_video_url(url, absolute: true) if local "<BaseURL>#{url}</BaseURL>" end return manifest end - adaptive_fmts = video.adaptive_fmts - + # Ditto, only proxify URLs if `local=true` is used if local - adaptive_fmts.each do |fmt| - fmt["url"] = JSON::Any.new("#{HOST_URL}#{URI.parse(fmt["url"].as_s).request_target}") + video.adaptive_fmts.each do |fmt| + fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s, absolute: true)) end end @@ -70,17 +63,23 @@ module Invidious::Routes::API::Manifest # OTF streams aren't supported yet (See https://github.com/TeamNewPipe/NewPipe/issues/2415) next if !(fmt.has_key?("indexRange") && fmt.has_key?("initRange")) + audio_track = fmt["audioTrack"]?.try &.as_h? || {} of String => JSON::Any + lang = audio_track["id"]?.try &.as_s.split('.')[0] || "und" + is_default = audio_track.has_key?("audioIsDefault") ? audio_track["audioIsDefault"].as_bool : i == 0 + displayname = audio_track["displayName"]?.try &.as_s || "Unknown" + bitrate = fmt["bitrate"] + # Different representations of the same audio should be groupped into one AdaptationSet. # However, most players don't support auto quality switching, so we have to trick them # into providing a quality selector. # See https://github.com/iv-org/invidious/issues/3074 for more details. - xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, label: fmt["bitrate"].to_s + "k") do + xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, label: "#{displayname} [#{bitrate}k]", lang: lang) do codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"') bandwidth = fmt["bitrate"].as_i itag = fmt["itag"].as_i url = fmt["url"].as_s - xml.element("Role", schemeIdUri: "urn:mpeg:dash:role:2011", value: i == 0 ? "main" : "alternate") + xml.element("Role", schemeIdUri: "urn:mpeg:dash:role:2011", value: is_default ? "main" : "alternate") xml.element("Representation", id: fmt["itag"], codecs: codecs, bandwidth: bandwidth) do xml.element("AudioChannelConfiguration", schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011", @@ -177,8 +176,9 @@ module Invidious::Routes::API::Manifest manifest = response.body if local - manifest = manifest.gsub(/^https:\/\/\w+---.{11}\.c\.youtube\.com[^\n]*/m) do |match| - path = URI.parse(match).path + manifest = manifest.gsub(/https:\/\/[^\n"]*/m) do |match| + uri = URI.parse(match) + path = uri.path path = path.lchop("/videoplayback/") path = path.rchop("/") @@ -207,7 +207,7 @@ module Invidious::Routes::API::Manifest raw_params["fvip"] = fvip["fvip"] end - raw_params["local"] = "true" + raw_params["host"] = uri.host.not_nil! "#{HOST_URL}/videoplayback?#{raw_params}" end diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 7faf200a..588bbc2a 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -27,10 +27,21 @@ module Invidious::Routes::API::V1::Channels # Retrieve "sort by" setting from URL parameters sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" - begin - videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by) - rescue ex - return error_json(500, ex) + if channel.is_age_gated + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULF")) + videos = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + videos = [] of PlaylistVideo + end + next_continuation = nil + else + begin + videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by) + rescue ex + return error_json(500, ex) + end end JSON.build do |json| @@ -84,6 +95,7 @@ module Invidious::Routes::API::V1::Channels json.field "joined", channel.joined.to_unix json.field "autoGenerated", channel.auto_generated + json.field "ageGated", channel.is_age_gated json.field "isFamilyFriendly", channel.is_family_friendly json.field "description", html_to_content(channel.description_html) json.field "descriptionHtml", channel.description_html @@ -142,12 +154,23 @@ module Invidious::Routes::API::V1::Channels sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" continuation = env.params.query["continuation"]? - begin - videos, next_continuation = Channel::Tabs.get_60_videos( - channel, continuation: continuation, sort_by: sort_by - ) - rescue ex - return error_json(500, ex) + if channel.is_age_gated + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULF")) + videos = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + videos = [] of PlaylistVideo + end + next_continuation = nil + else + begin + videos, next_continuation = Channel::Tabs.get_60_videos( + channel, continuation: continuation, sort_by: sort_by + ) + rescue ex + return error_json(500, ex) + end end return JSON.build do |json| @@ -174,14 +197,26 @@ module Invidious::Routes::API::V1::Channels get_channel() # Retrieve continuation from URL parameters + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" continuation = env.params.query["continuation"]? - begin - videos, next_continuation = Channel::Tabs.get_shorts( - channel, continuation: continuation - ) - rescue ex - return error_json(500, ex) + if channel.is_age_gated + begin + playlist = get_playlist(channel.ucid.sub("UC", "UUSH")) + videos = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + videos = [] of PlaylistVideo + end + next_continuation = nil + else + begin + videos, next_continuation = Channel::Tabs.get_shorts( + channel, continuation: continuation, sort_by: sort_by + ) + rescue ex + return error_json(500, ex) + end end return JSON.build do |json| @@ -208,14 +243,26 @@ module Invidious::Routes::API::V1::Channels get_channel() # Retrieve continuation from URL parameters + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" continuation = env.params.query["continuation"]? - begin - videos, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation - ) - rescue ex - return error_json(500, ex) + if channel.is_age_gated + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULV")) + videos = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + videos = [] of PlaylistVideo + end + next_continuation = nil + else + begin + videos, next_continuation = Channel::Tabs.get_60_livestreams( + channel, continuation: continuation, sort_by: sort_by + ) + rescue ex + return error_json(500, ex) + end end return JSON.build do |json| diff --git a/src/invidious/routes/api/v1/feeds.cr b/src/invidious/routes/api/v1/feeds.cr index 41865f34..fea2993c 100644 --- a/src/invidious/routes/api/v1/feeds.cr +++ b/src/invidious/routes/api/v1/feeds.cr @@ -31,7 +31,7 @@ module Invidious::Routes::API::V1::Feeds if !CONFIG.popular_enabled error_message = {"error" => "Administrator has disabled this endpoint."}.to_json - haltf env, 400, error_message + haltf env, 403, error_message end JSON.build do |json| diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index 12942906..4f5b58da 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -42,6 +42,9 @@ module Invidious::Routes::API::V1::Misc format = env.params.query["format"]? format ||= "json" + listen_param = env.params.query["listen"]? + listen = (listen_param == "true" || listen_param == "1") + if plid.starts_with? "RD" return env.redirect "/api/v1/mixes/#{plid}" end @@ -74,7 +77,9 @@ module Invidious::Routes::API::V1::Misc response = playlist.to_json(offset, video_id: video_id) json_response = JSON.parse(response) - if json_response["videos"].as_a[0]["index"] != offset + if json_response["videos"].as_a.empty? + json_response = JSON.parse(response) + elsif json_response["videos"].as_a[0]["index"] != offset offset = json_response["videos"].as_a[0]["index"].as_i lookback = offset < 50 ? offset : 50 response = playlist.to_json(offset - lookback) @@ -83,7 +88,7 @@ module Invidious::Routes::API::V1::Misc end if format == "html" - playlist_html = template_playlist(json_response) + playlist_html = template_playlist(json_response, listen) index, next_video = json_response["videos"].as_a.skip(1 + lookback).select { |video| !video["author"].as_s.empty? }[0]?.try { |v| {v["index"], v["videoId"]} } || {nil, nil} response = { @@ -109,6 +114,9 @@ module Invidious::Routes::API::V1::Misc format = env.params.query["format"]? format ||= "json" + listen_param = env.params.query["listen"]? + listen = (listen_param == "true" || listen_param == "1") + begin mix = fetch_mix(rdid, continuation, locale: locale) @@ -139,9 +147,7 @@ module Invidious::Routes::API::V1::Misc json.field "authorUrl", "/channel/#{video.ucid}" json.field "videoThumbnails" do - json.array do - Invidious::JSONify::APIv1.thumbnails(json, video.id) - end + Invidious::JSONify::APIv1.thumbnails(json, video.id) end json.field "index", video.index @@ -155,7 +161,7 @@ module Invidious::Routes::API::V1::Misc if format == "html" response = JSON.parse(response) - playlist_html = template_mix(response) + playlist_html = template_mix(response, listen) next_video = response["videos"].as_a.select { |video| !video["author"].as_s.empty? }[0]?.try &.["videoId"] response = { @@ -177,8 +183,8 @@ module Invidious::Routes::API::V1::Misc begin resolved_url = YoutubeAPI.resolve_url(url.as(String)) endpoint = resolved_url["endpoint"] - pageType = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || "" - if pageType == "WEB_PAGE_TYPE_UNKNOWN" + page_type = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || "" + if page_type == "WEB_PAGE_TYPE_UNKNOWN" return error_json(400, "Unknown url") end @@ -194,7 +200,7 @@ module Invidious::Routes::API::V1::Misc json.field "playlistId", sub_endpoint["playlistId"].as_s if sub_endpoint["playlistId"]? json.field "startTimeSeconds", sub_endpoint["startTimeSeconds"].as_i if sub_endpoint["startTimeSeconds"]? json.field "params", params.try &.as_s - json.field "pageType", pageType + json.field "pageType", page_type end end end diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index 2922b060..59a30745 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -31,9 +31,7 @@ module Invidious::Routes::API::V1::Search query = env.params.query["q"]? || "" begin - client = HTTP::Client.new("suggestqueries-clients6.youtube.com") - client.before_request { |r| add_yt_headers(r) } - + client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers: true) url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt" response = client.get(url).body diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 9281f4dd..368304ac 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -1,3 +1,5 @@ +require "html" + module Invidious::Routes::API::V1::Videos def self.videos(env) locale = env.get("preferences").as(Preferences).locale @@ -89,9 +91,14 @@ module Invidious::Routes::API::V1::Videos if CONFIG.use_innertube_for_captions params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated) - initial_data = YoutubeAPI.get_transcript(params) - webvtt = Invidious::Videos::Transcript.convert_transcripts_to_vtt(initial_data, caption.language_code) + transcript = Invidious::Videos::Transcript.from_raw( + YoutubeAPI.get_transcript(params), + caption.language_code, + caption.auto_generated + ) + + webvtt = transcript.to_vtt else # Timedtext API handling url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target @@ -111,7 +118,7 @@ module Invidious::Routes::API::V1::Videos else caption_xml = XML.parse(caption_xml) - webvtt = WebVTT.build(settings_field) do |webvtt| + webvtt = WebVTT.build(settings_field) do |builder| caption_nodes = caption_xml.xpath_nodes("//transcript/text") caption_nodes.each_with_index do |node, i| start_time = node["start"].to_f.seconds @@ -131,12 +138,16 @@ module Invidious::Routes::API::V1::Videos text = "<v #{md["name"]}>#{md["text"]}</v>" end - webvtt.cue(start_time, end_time, text) + builder.cue(start_time, end_time, text) end end end else - webvtt = YT_POOL.client &.get("#{url}&fmt=vtt").body + uri = URI.parse(url) + query_params = uri.query_params + query_params["fmt"] = "vtt" + uri.query_params = query_params + webvtt = YT_POOL.client &.get(uri.request_target).body if webvtt.starts_with?("<?xml") webvtt = caption.timedtext_to_vtt(webvtt) @@ -178,15 +189,14 @@ module Invidious::Routes::API::V1::Videos haltf env, 500 end - storyboards = video.storyboards - width = env.params.query["width"]? - height = env.params.query["height"]? + width = env.params.query["width"]?.try &.to_i + height = env.params.query["height"]?.try &.to_i if !width && !height response = JSON.build do |json| json.object do json.field "storyboards" do - Invidious::JSONify::APIv1.storyboards(json, id, storyboards) + Invidious::JSONify::APIv1.storyboards(json, id, video.storyboards) end end end @@ -196,35 +206,48 @@ module Invidious::Routes::API::V1::Videos env.response.content_type = "text/vtt" - storyboard = storyboards.select { |sb| width == "#{sb[:width]}" || height == "#{sb[:height]}" } + # Select a storyboard matching the user's provided width/height + storyboard = video.storyboards.select { |x| x.width == width || x.height == height } + haltf env, 404 if storyboard.empty? - if storyboard.empty? - haltf env, 404 - else - storyboard = storyboard[0] - end + # Alias variable, to make the code below esaier to read + sb = storyboard[0] + + # Some base URL segments that we'll use to craft the final URLs + work_url = sb.proxied_url.dup + template_path = sb.proxied_url.path - WebVTT.build do |vtt| - start_time = 0.milliseconds - end_time = storyboard[:interval].milliseconds + # Initialize cue timing variables + # NOTE: videojs-vtt-thumbnails gets lost when the cue times don't overlap + # (i.e: if cue[n] end time is 1:06:25.000, cue[n+1] start time should be 1:06:25.000) + time_delta = sb.interval.milliseconds + start_time = 0.milliseconds + end_time = time_delta - storyboard[:storyboard_count].times do |i| - url = storyboard[:url] - authority = /(i\d?).ytimg.com/.match(url).not_nil![1]? - url = url.gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "") - url = "#{HOST_URL}/sb/#{authority}/#{url}" + # Build a VTT file for VideoJS-vtt plugin + vtt_file = WebVTT.build do |vtt| + sb.images_count.times do |i| + # Replace the variable component part of the path + work_url.path = template_path.sub("$M", i) - storyboard[:storyboard_height].times do |j| - storyboard[:storyboard_width].times do |k| - current_cue_url = "#{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]}" - vtt.cue(start_time, end_time, current_cue_url) + sb.rows.times do |j| + sb.columns.times do |k| + # The URL fragment represents the offset of the thumbnail inside the storyboard image + work_url.fragment = "xywh=#{sb.width * k},#{sb.height * j},#{sb.width - 2},#{sb.height}" - start_time += storyboard[:interval].milliseconds - end_time += storyboard[:interval].milliseconds + vtt.cue(start_time, end_time, work_url.to_s) + + start_time += time_delta + end_time += time_delta end end end end + + # videojs-vtt-thumbnails is not compliant to the VTT specification, it + # doesn't unescape the HTML entities, so we have to do it here: + # TODO: remove this when we migrate to VideoJS 8 + return HTML.unescape(vtt_file) end def self.annotations(env) @@ -245,7 +268,7 @@ module Invidious::Routes::API::V1::Videos if CONFIG.cache_annotations && (cached_annotation = Invidious::Database::Annotations.select(id)) annotations = cached_annotation.annotations else - index = CHARS_SAFE.index(id[0]).not_nil!.to_s.rjust(2, '0') + index = CHARS_SAFE.index!(id[0]).to_s.rjust(2, '0') # IA doesn't handle leading hyphens, # so we use https://archive.org/details/youtubeannotations_64 diff --git a/src/invidious/routes/before_all.cr b/src/invidious/routes/before_all.cr index 396840a4..5695dee9 100644 --- a/src/invidious/routes/before_all.cr +++ b/src/invidious/routes/before_all.cr @@ -30,7 +30,7 @@ module Invidious::Routes::BeforeAll # Only allow the pages at /embed/* to be embedded if env.request.resource.starts_with?("/embed") - frame_ancestors = "'self' http: https:" + frame_ancestors = "'self' file: http: https:" else frame_ancestors = "'none'" end diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index fea49bbe..7d634cbb 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -20,10 +20,11 @@ module Invidious::Routes::Channels sort_by = env.params.query["sort_by"]?.try &.downcase if channel.auto_generated + sort_by ||= "last" sort_options = {"last", "oldest", "newest"} items, next_continuation = fetch_channel_playlists( - channel.ucid, channel.author, continuation, (sort_by || "last") + channel.ucid, channel.author, continuation, sort_by ) items.uniq! do |item| @@ -36,12 +37,26 @@ module Invidious::Routes::Channels items = items.select(SearchPlaylist) items.each(&.author = "") else - sort_options = {"newest", "oldest", "popular"} - # Fetch items and continuation token - items, next_continuation = Channel::Tabs.get_videos( - channel, continuation: continuation, sort_by: (sort_by || "newest") - ) + if channel.is_age_gated + sort_by = "" + sort_options = [] of String + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULF")) + items = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + items = [] of PlaylistVideo + end + next_continuation = nil + else + sort_by ||= "newest" + sort_options = {"newest", "oldest", "popular"} + + items, next_continuation = Channel::Tabs.get_60_videos( + channel, continuation: continuation, sort_by: sort_by + ) + end end selected_tab = Frontend::ChannelPage::TabsAvailable::Videos @@ -58,14 +73,26 @@ module Invidious::Routes::Channels return env.redirect "/channel/#{channel.ucid}" end - # TODO: support sort option for shorts - sort_by = "" - sort_options = [] of String + if channel.is_age_gated + sort_by = "" + sort_options = [] of String + begin + playlist = get_playlist(channel.ucid.sub("UC", "UUSH")) + items = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + items = [] of PlaylistVideo + end + next_continuation = nil + else + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" + sort_options = {"newest", "oldest", "popular"} - # Fetch items and continuation token - items, next_continuation = Channel::Tabs.get_shorts( - channel, continuation: continuation - ) + # Fetch items and continuation token + items, next_continuation = Channel::Tabs.get_shorts( + channel, continuation: continuation, sort_by: sort_by + ) + end selected_tab = Frontend::ChannelPage::TabsAvailable::Shorts templated "channel" @@ -81,14 +108,26 @@ module Invidious::Routes::Channels return env.redirect "/channel/#{channel.ucid}" end - # TODO: support sort option for livestreams - sort_by = "" - sort_options = [] of String + if channel.is_age_gated + sort_by = "" + sort_options = [] of String + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULV")) + items = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + items = [] of PlaylistVideo + end + next_continuation = nil + else + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" + sort_options = {"newest", "oldest", "popular"} - # Fetch items and continuation token - items, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation - ) + # Fetch items and continuation token + items, next_continuation = Channel::Tabs.get_60_livestreams( + channel, continuation: continuation, sort_by: sort_by + ) + end selected_tab = Frontend::ChannelPage::TabsAvailable::Streams templated "channel" diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 266f7ba4..00f24159 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -157,10 +157,12 @@ module Invidious::Routes::Embed adaptive_fmts = video.adaptive_fmts if params.local - fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) } - adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) } + fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) } end + # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback + adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) } + video_streams = video.video_streams audio_streams = video.audio_streams diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index e20a7139..7f9a0edb 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -143,32 +143,25 @@ module Invidious::Routes::Feeds # RSS feeds def self.rss_channel(env) - locale = env.get("preferences").as(Preferences).locale - env.response.headers["Content-Type"] = "application/atom+xml" env.response.content_type = "application/atom+xml" - ucid = env.params.url["ucid"] + if env.params.url["ucid"].matches?(/^[\w-]+$/) + ucid = env.params.url["ucid"] + else + return error_atom(400, InfoException.new("Invalid channel ucid provided.")) + end params = HTTP::Params.parse(env.params.query["params"]? || "") - begin - channel = get_about_info(ucid, locale) - rescue ex : ChannelRedirect - return env.redirect env.request.resource.gsub(ucid, ex.channel_id) - rescue ex : NotFoundException - return error_atom(404, ex) - rescue ex - return error_atom(500, ex) - end - namespaces = { "yt" => "http://www.youtube.com/xml/schemas/2015", "media" => "http://search.yahoo.com/mrss/", "default" => "http://www.w3.org/2005/Atom", } - response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}") + response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}") + return error_atom(404, NotFoundException.new("Channel does not exist.")) if response.status_code == 404 rss = XML.parse(response.body) videos = rss.xpath_nodes("//default:feed/default:entry", namespaces).map do |entry| @@ -179,7 +172,7 @@ module Invidious::Routes::Feeds updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content) author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content - ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content + video_ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content description_html = entry.xpath_node("media:group/media:description", namespaces).not_nil!.to_s views = entry.xpath_node("media:group/media:community/media:statistics", namespaces).not_nil!.["views"].to_i64 @@ -187,43 +180,44 @@ module Invidious::Routes::Feeds title: title, id: video_id, author: author, - ucid: ucid, + ucid: video_ucid, published: published, views: views, description_html: description_html, length_seconds: 0, - live_now: false, - paid: false, - premium: false, premiere_timestamp: nil, author_verified: false, + author_thumbnail: nil, + badges: VideoBadges::None, }) end + author = "" + author = videos[0].author if videos.size > 0 + XML.build(indent: " ", encoding: "UTF-8") do |xml| xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", "xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom", "xml:lang": "en-US") do xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}") - xml.element("id") { xml.text "yt:channel:#{channel.ucid}" } - xml.element("yt:channelId") { xml.text channel.ucid } - xml.element("icon") { xml.text channel.author_thumbnail } - xml.element("title") { xml.text channel.author } - xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{channel.ucid}") + xml.element("id") { xml.text "yt:channel:#{ucid}" } + xml.element("yt:channelId") { xml.text ucid } + xml.element("title") { author } + xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{ucid}") xml.element("author") do - xml.element("name") { xml.text channel.author } - xml.element("uri") { xml.text "#{HOST_URL}/channel/#{channel.ucid}" } + xml.element("name") { xml.text author } + xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" } end xml.element("image") do - xml.element("url") { xml.text channel.author_thumbnail } - xml.element("title") { xml.text channel.author } + xml.element("url") { xml.text "" } + xml.element("title") { xml.text author } xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}") end videos.each do |video| - video.to_xml(channel.auto_generated, params, xml) + video.to_xml(false, params, xml) end end end @@ -311,8 +305,9 @@ module Invidious::Routes::Feeds end response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}") - document = XML.parse(response.body) + return error_atom(404, NotFoundException.new("Playlist does not exist.")) if response.status_code == 404 + document = XML.parse(response.body) document.xpath_nodes(%q(//*[@href]|//*[@url])).each do |node| node.attributes.each do |attribute| case attribute.name @@ -425,16 +420,6 @@ module Invidious::Routes::Feeds next # skip this video since it raised an exception (e.g. it is a scheduled live event) end - if CONFIG.enable_user_notifications - # Deliver notifications to `/api/v1/auth/notifications` - payload = { - "topic" => video.ucid, - "videoId" => video.id, - "published" => published.to_unix, - }.to_json - PG_DB.exec("NOTIFY notifications, E'#{payload}'") - end - video = ChannelVideo.new({ id: id, title: video.title, @@ -450,11 +435,7 @@ module Invidious::Routes::Feeds was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true) if was_insert - if CONFIG.enable_user_notifications - Invidious::Database::Users.add_notification(video) - else - Invidious::Database::Users.feed_needs_update(video) - end + NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video)) end end end diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index b6a2e110..639697db 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -11,29 +11,9 @@ module Invidious::Routes::Images end end - # We're encapsulating this into a proc in order to easily reuse this - # portion of the code for each request block below. - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - env.response.headers.delete("Transfer-Encoding") - return - end - - proxy_file(response, env) - } - begin - HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| - return request_proc.call(resp) + GGPHT_POOL.client &.get(url, headers) do |resp| + return self.proxy_image(env, resp) end rescue ex end @@ -61,27 +41,10 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Connection"] = "close" - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - return env.response.headers.delete("Transfer-Encoding") - end - - proxy_file(response, env) - } - begin - HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| - return request_proc.call(resp) + get_ytimg_pool(authority).client &.get(url, headers) do |resp| + env.response.headers["Connection"] = "close" + return self.proxy_image(env, resp) end rescue ex end @@ -101,26 +64,9 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - return env.response.headers.delete("Transfer-Encoding") - end - - proxy_file(response, env) - } - begin - HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| - return request_proc.call(resp) + get_ytimg_pool("i9").client &.get(url, headers) do |resp| + return self.proxy_image(env, resp) end rescue ex end @@ -165,8 +111,7 @@ module Invidious::Routes::Images if name == "maxres.jpg" build_thumbnails(id).each do |thumb| thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" - # This can likely be optimized into a (small) pool sometime in the future. - if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 + if get_ytimg_pool("i9").client &.head(thumbnail_resource_path, headers).status_code == 200 name = thumb[:url] + ".jpg" break end @@ -181,29 +126,28 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end + begin + get_ytimg_pool("i").client &.get(url, headers) do |resp| + return self.proxy_image(env, resp) end + rescue ex + end + end - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - return env.response.headers.delete("Transfer-Encoding") + private def self.proxy_image(env, response) + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value end + end - proxy_file(response, env) - } + env.response.headers["Access-Control-Allow-Origin"] = "*" - begin - # This can likely be optimized into a (small) pool sometime in the future. - HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end - rescue ex + if response.status_code >= 300 + return env.response.headers.delete("Transfer-Encoding") end + + return proxy_file(response, env) end end diff --git a/src/invidious/routes/misc.cr b/src/invidious/routes/misc.cr index d6bd9571..8b620d63 100644 --- a/src/invidious/routes/misc.cr +++ b/src/invidious/routes/misc.cr @@ -40,7 +40,16 @@ module Invidious::Routes::Misc def self.cross_instance_redirect(env) referer = get_referer(env) - instance_url = fetch_random_instance + + instance_list = Invidious::Jobs::InstanceListRefreshJob::INSTANCES["INSTANCES"] + if instance_list.empty? + instance_url = "redirect.invidious.io" + else + # Sample returns an array + # Instances are packaged as {region, domain} in the instance list + instance_url = instance_list.sample(1)[0][1] + end + env.redirect "https://#{instance_url}#{referer}" end end diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index 9c6843e9..f2213da4 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -304,23 +304,6 @@ module Invidious::Routes::Playlists end end - if env.params.query["action_create_playlist"]? - action = "action_create_playlist" - elsif env.params.query["action_delete_playlist"]? - action = "action_delete_playlist" - elsif env.params.query["action_edit_playlist"]? - action = "action_edit_playlist" - elsif env.params.query["action_add_video"]? - action = "action_add_video" - video_id = env.params.query["video_id"] - elsif env.params.query["action_remove_video"]? - action = "action_remove_video" - elsif env.params.query["action_move_video_before"]? - action = "action_move_video_before" - else - return env.redirect referer - end - begin playlist_id = env.params.query["playlist_id"] playlist = get_playlist(playlist_id).as(InvidiousPlaylist) @@ -335,12 +318,8 @@ module Invidious::Routes::Playlists end end - email = user.email - - case action - when "action_edit_playlist" - # TODO: Playlist stub - when "action_add_video" + case action = env.params.query["action"]? + when "add_video" if playlist.index.size >= CONFIG.playlist_length_limit if redirect return error_template(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos") @@ -377,12 +356,14 @@ module Invidious::Routes::Playlists Invidious::Database::PlaylistVideos.insert(playlist_video) Invidious::Database::Playlists.update_video_added(playlist_id, playlist_video.index) - when "action_remove_video" + when "remove_video" index = env.params.query["set_video_id"] Invidious::Database::PlaylistVideos.delete(index) Invidious::Database::Playlists.update_video_removed(playlist_id, index) - when "action_move_video_before" + when "move_video_before" # TODO: Playlist stub + when nil + return error_json(400, "Missing action") else return error_json(400, "Unsupported action #{action}") end diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index 112535bd..39ca77c0 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -27,6 +27,10 @@ module Invidious::Routes::PreferencesRoute annotations_subscribed ||= "off" annotations_subscribed = annotations_subscribed == "on" + preload = env.params.body["preload"]?.try &.as(String) + preload ||= "off" + preload = preload == "on" + autoplay = env.params.body["autoplay"]?.try &.as(String) autoplay ||= "off" autoplay = autoplay == "on" @@ -144,6 +148,7 @@ module Invidious::Routes::PreferencesRoute preferences = Preferences.from_json({ annotations: annotations, annotations_subscribed: annotations_subscribed, + preload: preload, autoplay: autoplay, captions: captions, comments: comments, @@ -214,7 +219,7 @@ module Invidious::Routes::PreferencesRoute statistics_enabled ||= "off" CONFIG.statistics_enabled = statistics_enabled == "on" - CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.try &.as(String) + CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.presence File.write("config/config.yml", CONFIG.to_yaml) end diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index 5be33533..44970922 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -51,6 +51,12 @@ module Invidious::Routes::Search else user = env.get? "user" + # An URL was copy/pasted in the search box. + # Redirect the user to the appropriate page. + if query.url? + return env.redirect UrlSanitizer.process(query.text).to_s + end + begin items = query.process rescue ex : ChannelSearchException diff --git a/src/invidious/routes/subscriptions.cr b/src/invidious/routes/subscriptions.cr index 7f9ec592..1de655d2 100644 --- a/src/invidious/routes/subscriptions.cr +++ b/src/invidious/routes/subscriptions.cr @@ -32,24 +32,16 @@ module Invidious::Routes::Subscriptions end end - if env.params.query["action_create_subscription_to_channel"]?.try &.to_i?.try &.== 1 - action = "action_create_subscription_to_channel" - elsif env.params.query["action_remove_subscriptions"]?.try &.to_i?.try &.== 1 - action = "action_remove_subscriptions" - else - return env.redirect referer - end - channel_id = env.params.query["c"]? channel_id ||= "" - case action - when "action_create_subscription_to_channel" + case action = env.params.query["action"]? + when "create_subscription_to_channel" if !user.subscriptions.includes? channel_id get_channel(channel_id) Invidious::Database::Users.subscribe_channel(user, channel_id) end - when "action_remove_subscriptions" + when "remove_subscriptions" Invidious::Database::Users.unsubscribe_channel(user, channel_id) else return error_json(400, "Unsupported action #{action}") diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index ec18f3b8..a8f9f665 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -42,7 +42,7 @@ module Invidious::Routes::VideoPlayback headers["Range"] = "bytes=#{range_for_head}" end - client = make_client(URI.parse(host), region, force_resolve = true) + client = make_client(URI.parse(host), region, force_resolve: true) response = HTTP::Client::Response.new(500) error = "" 5.times do @@ -57,7 +57,7 @@ module Invidious::Routes::VideoPlayback if new_host != host host = new_host client.close - client = make_client(URI.parse(new_host), region, force_resolve = true) + client = make_client(URI.parse(new_host), region, force_resolve: true) end url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" @@ -71,7 +71,7 @@ module Invidious::Routes::VideoPlayback fvip = "3" host = "https://r#{fvip}---#{mn}.googlevideo.com" - client = make_client(URI.parse(host), region, force_resolve = true) + client = make_client(URI.parse(host), region, force_resolve: true) rescue ex error = ex.message end @@ -131,7 +131,7 @@ module Invidious::Routes::VideoPlayback end # TODO: Record bytes written so we can restart after a chunk fails - while true + loop do if !range_end && content_length range_end = content_length end @@ -164,10 +164,13 @@ module Invidious::Routes::VideoPlayback env.response.headers["Access-Control-Allow-Origin"] = "*" if location = resp.headers["Location"]? - location = URI.parse(location) - location = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" + url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region) - env.redirect location + if title = query_params["title"]? + url = "#{url}&title=#{URI.encode_www_form(title)}" + end + + env.redirect url break end @@ -196,7 +199,7 @@ module Invidious::Routes::VideoPlayback break else client.close - client = make_client(URI.parse(host), region, force_resolve = true) + client = make_client(URI.parse(host), region, force_resolve: true) end end diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index aabe8dfc..1f384546 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -121,10 +121,12 @@ module Invidious::Routes::Watch adaptive_fmts = video.adaptive_fmts if params.local - fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) } - adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) } + fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) } end + # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback + adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) } + video_streams = video.video_streams audio_streams = video.audio_streams @@ -241,18 +243,10 @@ module Invidious::Routes::Watch end end - if env.params.query["action_mark_watched"]? - action = "action_mark_watched" - elsif env.params.query["action_mark_unwatched"]? - action = "action_mark_unwatched" - else - return env.redirect referer - end - - case action - when "action_mark_watched" + case action = env.params.query["action"]? + when "mark_watched" Invidious::Database::Users.mark_watched(user, id) - when "action_mark_unwatched" + when "mark_unwatched" Invidious::Database::Users.mark_unwatched(user, id) else return error_json(400, "Unsupported action #{action}") diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index ba05da19..9009062f 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -243,17 +243,16 @@ module Invidious::Routing # Channels get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home + get "/api/v1/channels/:ucid/latest", {{namespace}}::Channels, :latest + get "/api/v1/channels/:ucid/videos", {{namespace}}::Channels, :videos get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams get "/api/v1/channels/:ucid/podcasts", {{namespace}}::Channels, :podcasts get "/api/v1/channels/:ucid/releases", {{namespace}}::Channels, :releases - + get "/api/v1/channels/:ucid/playlists", {{namespace}}::Channels, :playlists + get "/api/v1/channels/:ucid/community", {{namespace}}::Channels, :community get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels - - {% for route in {"videos", "latest", "playlists", "community", "search"} %} - get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}} - get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}} - {% end %} + get "/api/v1/channels/:ucid/search", {{namespace}}::Channels, :search # Posts get "/api/v1/post/:id", {{namespace}}::Channels, :post @@ -271,11 +270,6 @@ module Invidious::Routing # Authenticated - # The notification APIs cannot be extracted yet! They require the *local* notifications constant defined in invidious.cr - # - # Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications - # Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications - get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr index bf968734..bc2715cf 100644 --- a/src/invidious/search/filters.cr +++ b/src/invidious/search/filters.cr @@ -75,7 +75,7 @@ module Invidious::Search @type : Type = Type::All, @duration : Duration = Duration::None, @features : Features = Features::None, - @sort : Sort = Sort::Relevance + @sort : Sort = Sort::Relevance, ) end diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr index e38845d9..94a92e23 100644 --- a/src/invidious/search/query.cr +++ b/src/invidious/search/query.cr @@ -20,6 +20,9 @@ module Invidious::Search property region : String? property channel : String = "" + # Flag that indicates if the smart search features have been disabled. + @inhibit_ssf : Bool = false + # Return true if @raw_query is either `nil` or empty private def empty_raw_query? return @raw_query.empty? @@ -44,14 +47,22 @@ module Invidious::Search def initialize( params : HTTP::Params, @type : Type = Type::Regular, - @region : String? = nil + @region : String? = nil, ) # Get the raw search query string (common to all search types). In # Regular search mode, also look for the `search_query` URL parameter - if @type.regular? - @raw_query = params["q"]? || params["search_query"]? || "" - else - @raw_query = params["q"]? || "" + _raw_query = params["q"]? + _raw_query ||= params["search_query"]? if @type.regular? + _raw_query ||= "" + + # Remove surrounding whitespaces. Mostly useful for copy/pasted URLs. + @raw_query = _raw_query.strip + + # Check for smart features (ex: URL search) inhibitor (backslash). + # If inhibitor is present, remove it. + if @raw_query.starts_with?('\\') + @inhibit_ssf = true + @raw_query = @raw_query[1..] end # Get the page number (also common to all search types) @@ -85,7 +96,7 @@ module Invidious::Search @filters = Filters.from_iv_params(params) @channel = params["channel"]? || "" - if @filters.default? && @raw_query.includes?(':') + if @filters.default? && @raw_query.index(/\w:\w/) # Parse legacy filters from query @filters, @channel, @query, subs = Filters.from_legacy_filters(@raw_query) else @@ -136,5 +147,22 @@ module Invidious::Search return params end + + # Checks if the query is a standalone URL + def url? : Bool + # If the smart features have been inhibited, don't go further. + return false if @inhibit_ssf + + # Only supported in regular search mode + return false if !@type.regular? + + # If filters are present, that's a regular search + return false if !@filters.default? + + # Simple heuristics: domain name + return @raw_query.starts_with?( + /(https?:\/\/)?(www\.)?(m\.)?youtu(\.be|be\.com)\// + ) + end end end diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 108f2ccc..007eb666 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -115,7 +115,7 @@ struct Invidious::User playlists.each do |item| title = item["title"]?.try &.as_s?.try &.delete("<>") description = item["description"]?.try &.as_s?.try &.delete("\r") - privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy } + privacy = item["privacy"]?.try &.as_s?.try { |raw_pl_privacy_state| PlaylistPrivacy.parse? raw_pl_privacy_state } next if !title next if !description @@ -124,7 +124,7 @@ struct Invidious::User playlist = create_playlist(title, privacy, user) Invidious::Database::Playlists.update_description(playlist.id, description) - videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| + item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| if idx > CONFIG.playlist_length_limit raise InfoException.new("Playlist cannot have more than #{CONFIG.playlist_length_limit} videos") end @@ -161,7 +161,7 @@ struct Invidious::User # Youtube # ------------------- - private def is_opml?(mimetype : String, extension : String) + private def opml?(mimetype : String, extension : String) opml_mimetypes = [ "application/xml", "text/xml", @@ -179,10 +179,10 @@ struct Invidious::User def from_youtube(user : User, body : String, filename : String, type : String) : Bool extension = filename.split(".").last - if is_opml?(type, extension) + if opml?(type, extension) subscriptions = XML.parse(body) user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| - channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] + channel["xmlUrl"].match!(/UC[a-zA-Z0-9_-]{22}/)[0] end elsif extension == "json" || type == "application/json" subscriptions = JSON.parse(body) @@ -290,42 +290,39 @@ struct Invidious::User end def from_newpipe(user : User, body : String) : Bool - io = IO::Memory.new(body) + Compress::Zip::File.open(IO::Memory.new(body), true) do |file| + entry = file.entries.find { |file_entry| file_entry.filename == "newpipe.db" } + return false if entry.nil? + entry.open do |file_io| + # Ensure max size of 4MB + io_sized = IO::Sized.new(file_io, 0x400000) - Compress::Zip::File.open(io) do |file| - file.entries.each do |entry| - entry.open do |file_io| - # Ensure max size of 4MB - io_sized = IO::Sized.new(file_io, 0x400000) - - next if entry.filename != "newpipe.db" - - tempfile = File.tempfile(".db") - - begin - File.write(tempfile.path, io_sized.gets_to_end) - rescue - return false - end - - db = DB.open("sqlite3://" + tempfile.path) - - user.watched += db.query_all("SELECT url FROM streams", as: String) - .map(&.lchop("https://www.youtube.com/watch?v=")) + begin + temp = File.tempfile(".db") do |tempfile| + begin + File.write(tempfile.path, io_sized.gets_to_end) + rescue + return false + end - user.watched.uniq! - Invidious::Database::Users.update_watch_history(user) + DB.open("sqlite3://" + tempfile.path) do |db| + user.watched += db.query_all("SELECT url FROM streams", as: String) + .map(&.lchop("https://www.youtube.com/watch?v=")) - user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String) - .map(&.lchop("https://www.youtube.com/channel/")) + user.watched.uniq! + Invidious::Database::Users.update_watch_history(user) - user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions) + user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String) + .map(&.lchop("https://www.youtube.com/channel/")) - Invidious::Database::Users.update_subscriptions(user) + user.subscriptions.uniq! + user.subscriptions = get_batch_channels(user.subscriptions) - db.close - tempfile.delete + Invidious::Database::Users.update_subscriptions(user) + end + end + ensure + temp.delete if !temp.nil? end end end diff --git a/src/invidious/user/preferences.cr b/src/invidious/user/preferences.cr index b3059403..0a8525f3 100644 --- a/src/invidious/user/preferences.cr +++ b/src/invidious/user/preferences.cr @@ -4,6 +4,7 @@ struct Preferences property annotations : Bool = CONFIG.default_user_preferences.annotations property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed + property preload : Bool = CONFIG.default_user_preferences.preload property autoplay : Bool = CONFIG.default_user_preferences.autoplay property automatic_instance_redirect : Bool = CONFIG.default_user_preferences.automatic_instance_redirect diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index c218b4ef..962f87bd 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -27,12 +27,6 @@ struct Video @captions = [] of Invidious::Videos::Captions::Metadata @[DB::Field(ignore: true)] - property adaptive_fmts : Array(Hash(String, JSON::Any))? - - @[DB::Field(ignore: true)] - property fmt_stream : Array(Hash(String, JSON::Any))? - - @[DB::Field(ignore: true)] property description : String? module JSONConverter @@ -98,45 +92,24 @@ struct Video # Methods for parsing streaming data - def fmt_stream - return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream - - fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any) - fmt_stream.each do |fmt| - if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) } - s.each do |k, v| - fmt[k] = JSON::Any.new(v) - end - fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}") - end - - fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}") - fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]? + def fmt_stream : Array(Hash(String, JSON::Any)) + if formats = info.dig?("streamingData", "formats") + return formats + .as_a.map(&.as_h) + .sort_by! { |f| f["width"]?.try &.as_i || 0 } + else + return [] of Hash(String, JSON::Any) end - - fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 } - @fmt_stream = fmt_stream - return @fmt_stream.as(Array(Hash(String, JSON::Any))) end - def adaptive_fmts - return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts - fmt_stream = info["streamingData"]?.try &.["adaptiveFormats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any) - fmt_stream.each do |fmt| - if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) } - s.each do |k, v| - fmt[k] = JSON::Any.new(v) - end - fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}") - end - - fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}") - fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]? + def adaptive_fmts : Array(Hash(String, JSON::Any)) + if formats = info.dig?("streamingData", "adaptiveFormats") + return formats + .as_a.map(&.as_h) + .sort_by! { |f| f["width"]?.try &.as_i || f["audioTrack"]?.try { |a| a["audioIsDefault"]?.try { |v| v.as_bool ? -1 : 0 } } || 0 } + else + return [] of Hash(String, JSON::Any) end - - fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 } - @adaptive_fmts = fmt_stream - return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) end def video_streams @@ -150,65 +123,8 @@ struct Video # Misc. methods def storyboards - storyboards = info.dig?("storyboards", "playerStoryboardSpecRenderer", "spec") - .try &.as_s.split("|") - - if !storyboards - if storyboard = info.dig?("storyboards", "playerLiveStoryboardSpecRenderer", "spec").try &.as_s - return [{ - url: storyboard.split("#")[0], - width: 106, - height: 60, - count: -1, - interval: 5000, - storyboard_width: 3, - storyboard_height: 3, - storyboard_count: -1, - }] - end - end - - items = [] of NamedTuple( - url: String, - width: Int32, - height: Int32, - count: Int32, - interval: Int32, - storyboard_width: Int32, - storyboard_height: Int32, - storyboard_count: Int32) - - return items if !storyboards - - url = URI.parse(storyboards.shift) - params = HTTP::Params.parse(url.query || "") - - storyboards.each_with_index do |sb, i| - width, height, count, storyboard_width, storyboard_height, interval, _, sigh = sb.split("#") - params["sigh"] = sigh - url.query = params.to_s - - width = width.to_i - height = height.to_i - count = count.to_i - interval = interval.to_i - storyboard_width = storyboard_width.to_i - storyboard_height = storyboard_height.to_i - storyboard_count = (count / (storyboard_width * storyboard_height)).ceil.to_i - - items << { - url: url.to_s.sub("$L", i).sub("$N", "M$M"), - width: width, - height: height, - count: count, - interval: interval, - storyboard_width: storyboard_width, - storyboard_height: storyboard_height, - storyboard_count: storyboard_count, - } - end - - items + container = info.dig?("storyboards") || JSON::Any.new("{}") + return IV::Videos::Storyboard.from_yt_json(container, self.length_seconds) end def paid @@ -250,10 +166,10 @@ struct Video end def genre_url : String? - info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil + info["genreUcid"].try &.as_s? ? "/channel/#{info["genreUcid"]}" : nil end - def is_vr : Bool? + def vr? : Bool? return {"EQUIRECTANGULAR", "MESH"}.includes? self.projection_type end @@ -334,6 +250,21 @@ struct Video {% if flag?(:debug_macros) %} {{debug}} {% end %} end + # Macro to generate ? and = accessor methods for attributes in `info` + private macro predicate_bool(method_name, name) + # Return {{name.stringify}} from `info` + def {{method_name.id.underscore}}? : Bool + return info[{{name.stringify}}]?.try &.as_bool || false + end + + # Update {{name.stringify}} into `info` + def {{method_name.id.underscore}}=(value : Bool) + info[{{name.stringify}}] = JSON::Any.new(value) + end + + {% if flag?(:debug_macros) %} {{debug}} {% end %} + end + # Method definitions, using the macros above getset_string author @@ -355,11 +286,12 @@ struct Video getset_i64 likes getset_i64 views + # TODO: Make predicate_bool the default as to adhere to Crystal conventions getset_bool allowRatings getset_bool authorVerified getset_bool isFamilyFriendly getset_bool isListed - getset_bool isUpcoming + predicate_bool upcoming, isUpcoming end def get_video(id, refresh = true, region = nil, force_refresh = false) @@ -394,10 +326,6 @@ end def fetch_video(id, region) info = extract_video_info(video_id: id) - allowed_regions = info - .dig?("microformat", "playerMicroformatRenderer", "availableCountries") - .try &.as_a.map &.as_s || [] of String - if reason = info["reason"]? if reason == "Video unavailable" raise NotFoundException.new(reason.as_s || "") diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr index 484e61d2..c811cfe1 100644 --- a/src/invidious/videos/caption.cr +++ b/src/invidious/videos/caption.cr @@ -123,6 +123,7 @@ module Invidious::Videos "Esperanto", "Estonian", "Filipino", + "Filipino (auto-generated)", "Finnish", "French", "French (auto-generated)", diff --git a/src/invidious/videos/description.cr b/src/invidious/videos/description.cr index c7191dec..1371bebb 100644 --- a/src/invidious/videos/description.cr +++ b/src/invidious/videos/description.cr @@ -36,7 +36,13 @@ def parse_description(desc, video_id : String) : String? return "" if content.empty? commands = desc["commandRuns"]?.try &.as_a - return content if commands.nil? + if commands.nil? + # Slightly faster than HTML.escape, as we're only doing one pass on + # the string instead of five for the standard library + return String.build do |str| + copy_string(str, content.each_codepoint, content.size) + end + end # Not everything is stored in UTF-8 on youtube's side. The SMP codepoints # (0x10000 and above) are encoded as UTF-16 surrogate pairs, which are diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 0e1a947c..5ca4bdb2 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -36,6 +36,13 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container") + if published_time_text = related["publishedTimeText"]? + decoded_time = decode_date(published_time_text["simpleText"].to_s) + published = decoded_time.to_rfc3339.to_s + else + published = nil + end + # TODO: when refactoring video types, make a struct for related videos # or reuse an existing type, if that fits. return { @@ -47,6 +54,7 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? "view_count" => JSON::Any.new(view_count || "0"), "short_view_count" => JSON::Any.new(short_view_count || "0"), "author_verified" => JSON::Any.new(author_verified), + "published" => JSON::Any.new(published || ""), } end @@ -55,7 +63,7 @@ def extract_video_info(video_id : String) client_config = YoutubeAPI::ClientConfig.new # Fetch data from the player endpoint - player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) + player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config) playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s @@ -102,7 +110,9 @@ def extract_video_info(video_id : String) new_player_response = nil - if reason.nil? + # Don't use Android test suite client if po_token is passed because po_token doesn't + # work for Android test suite client. + if reason.nil? && CONFIG.po_token.nil? # Fetch the video streams using an Android client in order to get the # decrypted URLs and maybe fix throttling issues (#2194). See the # following issue for an explanation about decrypted URLs: @@ -111,12 +121,6 @@ def extract_video_info(video_id : String) new_player_response = try_fetch_streaming_data(video_id, client_config) end - # Last hope - if new_player_response.nil? - client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed - new_player_response = try_fetch_streaming_data(video_id, client_config) - end - # Replace player response and reset reason if !new_player_response.nil? # Preserve captions & storyboard data before replacement @@ -127,10 +131,21 @@ def extract_video_info(video_id : String) params.delete("reason") end - {"captions", "playabilityStatus", "playerConfig", "storyboards", "streamingData"}.each do |f| + {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f| params[f] = player_response[f] if player_response[f]? end + # Convert URLs, if those are present + if streaming_data = player_response["streamingData"]? + %w[formats adaptiveFormats].each do |key| + streaming_data.as_h[key]?.try &.as_a.each do |format| + format.as_h["url"] = JSON::Any.new(convert_url(format)) + end + end + + params["streamingData"] = streaming_data + end + # Data structure version, for cache control params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64) @@ -180,10 +195,11 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any end video_details = player_response.dig?("videoDetails") - microformat = player_response.dig?("microformat", "playerMicroformatRenderer") + if !(microformat = player_response.dig?("microformat", "playerMicroformatRenderer")) + microformat = {} of String => JSON::Any + end raise BrokenTubeException.new("videoDetails") if !video_details - raise BrokenTubeException.new("microformat") if !microformat # Basic video infos @@ -208,8 +224,17 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp") .try { |t| Time.parse_rfc3339(t.as_s) } + premiere_timestamp ||= player_response.dig?( + "playabilityStatus", "liveStreamability", + "liveStreamabilityRenderer", "offlineSlate", + "liveStreamOfflineSlateRenderer", "scheduledStartTime" + ) + .try &.as_s.to_i64 + .try { |t| Time.unix(t) } + live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow") - .try &.as_bool || false + .try &.as_bool + live_now ||= video_details.dig?("isLive").try &.as_bool || false post_live_dvr = video_details.dig?("isPostLiveDvr") .try &.as_bool || false @@ -220,7 +245,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any .try &.as_a.map &.as_s || [] of String allow_ratings = video_details["allowRatings"]?.try &.as_bool - family_friendly = microformat["isFamilySafe"].try &.as_bool + family_friendly = microformat["isFamilySafe"]?.try &.as_bool is_listed = video_details["isCrawlable"]?.try &.as_bool is_upcoming = video_details["isUpcoming"]?.try &.as_bool @@ -424,7 +449,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil), # Video metadata "genre" => JSON::Any.new(genre.try &.as_s || ""), - "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""), + "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s?), "license" => JSON::Any.new(license.try &.as_s || ""), # Music section "music" => JSON.parse(music_list.to_json), @@ -438,3 +463,35 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any return params end + +private def convert_url(fmt) + if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) } + sp = cfr["sp"] + url = URI.parse(cfr["url"]) + params = url.query_params + + LOGGER.debug("convert_url: Decoding '#{cfr}'") + + unsig = DECRYPT_FUNCTION.try &.decrypt_signature(cfr["s"]) + params[sp] = unsig if unsig + else + url = URI.parse(fmt["url"].as_s) + params = url.query_params + end + + n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"]) + params["n"] = n if n + + if token = CONFIG.po_token + params["pot"] = token + end + + url.query_params = params + LOGGER.trace("convert_url: new url is '#{url}'") + + return url.to_s +rescue ex + LOGGER.debug("convert_url: Error when parsing video URL") + LOGGER.trace(ex.inspect_with_backtrace) + return "" +end diff --git a/src/invidious/videos/storyboard.cr b/src/invidious/videos/storyboard.cr new file mode 100644 index 00000000..bd0eef59 --- /dev/null +++ b/src/invidious/videos/storyboard.cr @@ -0,0 +1,122 @@ +require "uri" +require "http/params" + +module Invidious::Videos + struct Storyboard + # Template URL + getter url : URI + getter proxied_url : URI + + # Thumbnail parameters + getter width : Int32 + getter height : Int32 + getter count : Int32 + getter interval : Int32 + + # Image (storyboard) parameters + getter rows : Int32 + getter columns : Int32 + getter images_count : Int32 + + def initialize( + *, @url, @width, @height, @count, @interval, + @rows, @columns, @images_count, + ) + authority = /(i\d?).ytimg.com/.match!(@url.host.not_nil!)[1]? + + @proxied_url = URI.parse(HOST_URL) + @proxied_url.path = "/sb/#{authority}/#{@url.path.lchop("/sb/")}" + @proxied_url.query = @url.query + end + + # Parse the JSON structure from Youtube + def self.from_yt_json(container : JSON::Any, length_seconds : Int32) : Array(Storyboard) + # Livestream storyboards are a bit different + # TODO: document exactly how + if storyboard = container.dig?("playerLiveStoryboardSpecRenderer", "spec").try &.as_s + return [Storyboard.new( + url: URI.parse(storyboard.split("#")[0]), + width: 106, + height: 60, + count: -1, + interval: 5000, + rows: 3, + columns: 3, + images_count: -1 + )] + end + + # Split the storyboard string into chunks + # + # General format (whitespaces added for legibility): + # https://i.ytimg.com/sb/<video_id>/storyboard3_L$L/$N.jpg?sqp=<sig0> + # | 48 # 27 # 100 # 10 # 10 # 0 # default # rs$<sig1> + # | 80 # 45 # 95 # 10 # 10 # 10000 # M$M # rs$<sig2> + # | 160 # 90 # 95 # 5 # 5 # 10000 # M$M # rs$<sig3> + # + storyboards = container.dig?("playerStoryboardSpecRenderer", "spec") + .try &.as_s.split("|") + + return [] of Storyboard if !storyboards + + # The base URL is the first chunk + base_url = URI.parse(storyboards.shift) + + return storyboards.map_with_index do |sb, i| + # Separate the different storyboard parameters: + # width/height: respective dimensions, in pixels, of a single thumbnail + # count: how many thumbnails are displayed across the full video + # columns/rows: maximum amount of thumbnails that can be stuffed in a + # single image, horizontally and vertically. + # interval: interval between two thumbnails, in milliseconds + # name: storyboard filename. Usually "M$M" or "default" + # sigh: URL cryptographic signature + width, height, count, columns, rows, interval, name, sigh = sb.split("#") + + width = width.to_i + height = height.to_i + count = count.to_i + interval = interval.to_i + columns = columns.to_i + rows = rows.to_i + + # Copy base URL object, so that we can modify it + url = base_url.dup + + # Add the signature to the URL + params = url.query_params + params["sigh"] = sigh + url.query_params = params + + # Replace the template parts with what we have + url.path = url.path.sub("$L", i).sub("$N", name) + + # This value represents the maximum amount of thumbnails that can fit + # in a single image. The last image (or the only one for short videos) + # will contain less thumbnails than that. + thumbnails_per_image = columns * rows + + # This value represents the total amount of storyboards required to + # hold all of the thumbnails. It can't be less than 1. + images_count = (count / thumbnails_per_image).ceil.to_i + + # Compute the interval when needed (in general, that's only required + # for the first "default" storyboard). + if interval == 0 + interval = ((length_seconds / count) * 1_000).to_i + end + + Storyboard.new( + url: url, + width: width, + height: height, + count: count, + interval: interval, + rows: rows, + columns: columns, + images_count: images_count, + ) + end + end + end +end diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr index dac00eea..4bd9f820 100644 --- a/src/invidious/videos/transcript.cr +++ b/src/invidious/videos/transcript.cr @@ -1,8 +1,26 @@ module Invidious::Videos - # Namespace for methods primarily relating to Transcripts - module Transcript - record TranscriptLine, start_ms : Time::Span, end_ms : Time::Span, line : String + # A `Transcripts` struct encapsulates a sequence of lines that together forms the whole transcript for a given YouTube video. + # These lines can be categorized into two types: section headings and regular lines representing content from the video. + struct Transcript + # Types + record HeadingLine, start_ms : Time::Span, end_ms : Time::Span, line : String + record RegularLine, start_ms : Time::Span, end_ms : Time::Span, line : String + alias TranscriptLine = HeadingLine | RegularLine + property lines : Array(TranscriptLine) + + property language_code : String + property auto_generated : Bool + + # User friendly label for the current transcript. + # Example: "English (auto-generated)" + property label : String + + # Initializes a new Transcript struct with the contents and associated metadata describing it + def initialize(@lines : Array(TranscriptLine), @language_code : String, @auto_generated : Bool, @label : String) + end + + # Generates a protobuf string to fetch the requested transcript from YouTube def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String kind = auto_generated ? "asr" : "" @@ -30,48 +48,79 @@ module Invidious::Videos return params end - def self.convert_transcripts_to_vtt(initial_data : Hash(String, JSON::Any), target_language : String) : String - # Convert into array of TranscriptLine - lines = self.parse(initial_data) + # Constructs a Transcripts struct from the initial YouTube response + def self.from_raw(initial_data : Hash(String, JSON::Any), language_code : String, auto_generated : Bool) + transcript_panel = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", + "content", "transcriptSearchPanelRenderer") - settings_field = { - "Kind" => "captions", - "Language" => target_language, - } + segment_list = transcript_panel.dig("body", "transcriptSegmentListRenderer") - # Taken from Invidious::Videos::Captions::Metadata.timedtext_to_vtt() - vtt = WebVTT.build(settings_field) do |vtt| - lines.each do |line| - vtt.cue(line.start_ms, line.end_ms, line.line) - end + if !segment_list["initialSegments"]? + raise NotFoundException.new("Requested transcript does not exist") end - return vtt - end + # Extract user-friendly label for the current transcript + + footer_language_menu = transcript_panel.dig?( + "footer", "transcriptFooterRenderer", "languageMenu", "sortFilterSubMenuRenderer", "subMenuItems" + ) - private def self.parse(initial_data : Hash(String, JSON::Any)) - body = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", - "content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer", - "initialSegments").as_a + if footer_language_menu + label = footer_language_menu.as_a.select(&.["selected"].as_bool)[0]["title"].as_s + else + label = language_code + end + + # Extract transcript lines + + initial_segments = segment_list["initialSegments"].as_a lines = [] of TranscriptLine - body.each do |line| - # Transcript section headers. They are not apart of the captions and as such we can safely skip them. - if line.as_h.has_key?("transcriptSectionHeaderRenderer") - next + + initial_segments.each do |line| + if unpacked_line = line["transcriptSectionHeaderRenderer"]? + line_type = HeadingLine + else + unpacked_line = line["transcriptSegmentRenderer"] + line_type = RegularLine end - line = line["transcriptSegmentRenderer"] + start_ms = unpacked_line["startMs"].as_s.to_i.millisecond + end_ms = unpacked_line["endMs"].as_s.to_i.millisecond + text = extract_text(unpacked_line["snippet"]) || "" + + lines << line_type.new(start_ms, end_ms, text) + end + + return Transcript.new( + lines: lines, + language_code: language_code, + auto_generated: auto_generated, + label: label + ) + end - start_ms = line["startMs"].as_s.to_i.millisecond - end_ms = line["endMs"].as_s.to_i.millisecond + # Converts transcript lines to a WebVTT file + # + # This is used within Invidious to replace subtitles + # as to workaround YouTube's rate-limited timedtext endpoint. + def to_vtt + settings_field = { + "Kind" => "captions", + "Language" => @language_code, + } - text = extract_text(line["snippet"]) || "" + vtt = WebVTT.build(settings_field) do |builder| + @lines.each do |line| + # Section headers are excluded from the VTT conversion as to + # match the regular captions returned from YouTube as much as possible + next if line.is_a? HeadingLine - lines << TranscriptLine.new(start_ms, end_ms, text) + builder.cue(line.start_ms, line.end_ms, line.line) + end end - return lines + return vtt end end end diff --git a/src/invidious/videos/video_preferences.cr b/src/invidious/videos/video_preferences.cr index 34cf7ff0..48177bd8 100644 --- a/src/invidious/videos/video_preferences.cr +++ b/src/invidious/videos/video_preferences.cr @@ -2,6 +2,7 @@ struct VideoPreferences include JSON::Serializable property annotations : Bool + property preload : Bool property autoplay : Bool property comments : Array(String) property continue : Bool @@ -28,6 +29,7 @@ end def process_video_params(query, preferences) annotations = query["iv_load_policy"]?.try &.to_i? + preload = query["preload"]?.try { |q| (q == "true" || q == "1").to_unsafe } autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } comments = query["comments"]?.try &.split(",").map(&.downcase) continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe } @@ -50,6 +52,7 @@ def process_video_params(query, preferences) if preferences # region ||= preferences.region annotations ||= preferences.annotations.to_unsafe + preload ||= preferences.preload.to_unsafe autoplay ||= preferences.autoplay.to_unsafe comments ||= preferences.comments continue ||= preferences.continue.to_unsafe @@ -70,6 +73,7 @@ def process_video_params(query, preferences) end annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe + preload ||= CONFIG.default_user_preferences.preload.to_unsafe autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe comments ||= CONFIG.default_user_preferences.comments continue ||= CONFIG.default_user_preferences.continue.to_unsafe @@ -89,6 +93,7 @@ def process_video_params(query, preferences) save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe annotations = annotations == 1 + preload = preload == 1 autoplay = autoplay == 1 continue = continue == 1 continue_autoplay = continue_autoplay == 1 @@ -128,6 +133,7 @@ def process_video_params(query, preferences) params = VideoPreferences.new({ annotations: annotations, + preload: preload, autoplay: autoplay, comments: comments, continue: continue, diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr index 09df106d..1fe8ab7e 100644 --- a/src/invidious/views/channel.ecr +++ b/src/invidious/views/channel.ecr @@ -20,7 +20,9 @@ page_nav_html = IV::Frontend::Pagination.nav_ctoken(locale, base_url: relative_url, - ctoken: next_continuation + ctoken: next_continuation, + first_page: continuation.nil?, + params: env.params.query, ) %> @@ -30,16 +32,18 @@ <meta property="og:site_name" content="Invidious"> <meta property="og:url" content="<%= HOST_URL %>/channel/<%= ucid %>"> <meta property="og:title" content="<%= author %>"> -<meta property="og:image" content="/ggpht<%= channel_profile_pic %>"> +<meta property="og:image" content="<%= HOST_URL %>/ggpht<%= channel_profile_pic %>"> <meta property="og:description" content="<%= channel.description %>"> <meta name="twitter:card" content="summary"> <meta name="twitter:url" content="<%= HOST_URL %>/channel/<%= ucid %>"> <meta name="twitter:title" content="<%= author %>"> <meta name="twitter:description" content="<%= channel.description %>"> -<meta name="twitter:image" content="/ggpht<%= channel_profile_pic %>"> +<meta name="twitter:image" content="<%= HOST_URL %>/ggpht<%= channel_profile_pic %>"> <link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/channel/<%= ucid %>" /> <%- end -%> +<script src="/js/pagination.js?v=<%= ASSET_COMMIT %>"></script> + <link rel="alternate" href="<%= youtube_url %>"> <title><%= author %> - Invidious</title> <% end %> diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index 6d227cfc..c966a926 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -128,7 +128,7 @@ <div class="top-left-overlay"> <%- if env.get? "show_watched" -%> - <form data-onsubmit="return_false" action="/watch_ajax?action_mark_watched=1&id=<%= item.id %>&referer=<%= env.get("current_page") %>" method="post"> + <form data-onsubmit="return_false" action="/watch_ajax?action=mark_watched&id=<%= item.id %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>"> <button type="submit" class="pure-button pure-button-secondary low-profile" data-onclick="mark_watched" data-id="<%= item.id %>"> @@ -138,14 +138,14 @@ <%- end -%> <%- if plid_form = env.get?("add_playlist_items") -%> - <%- form_parameters = "action_add_video=1&video_id=#{item.id}&playlist_id=#{plid_form}&referer=#{env.get("current_page")}" -%> + <%- form_parameters = "action=add_video&video_id=#{item.id}&playlist_id=#{plid_form}&referer=#{env.get("current_page")}" -%> <form data-onsubmit="return_false" action="/playlist_ajax?<%= form_parameters %>" method="post"> <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>"> <button type="submit" class="pure-button pure-button-secondary low-profile" data-onclick="add_playlist_item" data-id="<%= item.id %>" data-plid="<%= plid_form %>"><i class="icon ion-md-add"></i></button> </form> <%- elsif item.is_a?(PlaylistVideo) && (plid_form = env.get?("remove_playlist_items")) -%> - <%- form_parameters = "action_remove_video=1&set_video_id=#{item.index}&playlist_id=#{plid_form}&referer=#{env.get("current_page")}" -%> + <%- form_parameters = "action=remove_video&set_video_id=#{item.index}&playlist_id=#{plid_form}&referer=#{env.get("current_page")}" -%> <form data-onsubmit="return_false" action="/playlist_ajax?<%= form_parameters %>" method="post"> <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>"> <button type="submit" class="pure-button pure-button-secondary low-profile" diff --git a/src/invidious/views/components/items_paginated.ecr b/src/invidious/views/components/items_paginated.ecr index 4534a0a3..f69df3fe 100644 --- a/src/invidious/views/components/items_paginated.ecr +++ b/src/invidious/views/components/items_paginated.ecr @@ -8,4 +8,14 @@ <%= page_nav_html %> +<script id="pagination-data" type="application/json"> +<%= +{ + "next_page" => translate(locale, "Next page"), + "prev_page" => translate(locale, "Previous page"), + "is_rtl" => locale_is_rtl?(locale) +}.to_pretty_json +%> +</script> + <script src="/js/watched_indicator.js"></script> diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index c3c02df0..5c28358b 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -1,5 +1,6 @@ <video style="outline:none;width:100%;background-color:#000" playsinline poster="<%= thumbnail %>" id="player" class="on-video_player video-js player-style-<%= params.player_style %>" + preload="<% if params.preload %>auto<% else %>none<% end %>" <% if params.autoplay %>autoplay<% end %> <% if params.video_loop %>loop<% end %> <% if params.controls %>controls<% end %>> diff --git a/src/invidious/views/components/search_box.ecr b/src/invidious/views/components/search_box.ecr index a03785d1..29da2c52 100644 --- a/src/invidious/views/components/search_box.ecr +++ b/src/invidious/views/components/search_box.ecr @@ -6,4 +6,7 @@ title="<%= translate(locale, "search") %>" value="<%= env.get?("search").try {|x| HTML.escape(x.as(String)) } %>"> </fieldset> + <button type="submit" id="searchbutton" aria-label="<%= translate(locale, "search") %>"> + <i class="icon ion-ios-search"></i> + </button> </form> diff --git a/src/invidious/views/components/subscribe_widget.ecr b/src/invidious/views/components/subscribe_widget.ecr index 05e4e253..3cfcb0eb 100644 --- a/src/invidious/views/components/subscribe_widget.ecr +++ b/src/invidious/views/components/subscribe_widget.ecr @@ -1,13 +1,13 @@ <% if user %> <% if subscriptions.includes? ucid %> - <form action="/subscription_ajax?action_remove_subscriptions=1&c=<%= ucid %>&referer=<%= env.get("current_page") %>" method="post"> + <form action="/subscription_ajax?action=remove_subscriptions&c=<%= ucid %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>"> <button data-type="unsubscribe" id="subscribe" class="pure-button pure-button-primary"> <b><input style="all:unset" type="submit" value="<%= translate(locale, "Unsubscribe") %> | <%= sub_count_text %>"></b> </button> </form> <% else %> - <form action="/subscription_ajax?action_create_subscription_to_channel=1&c=<%= ucid %>&referer=<%= env.get("current_page") %>" method="post"> + <form action="/subscription_ajax?action=create_subscription_to_channel&c=<%= ucid %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>"> <button data-type="subscribe" id="subscribe" class="pure-button pure-button-primary"> <b><input style="all:unset" type="submit" value="<%= translate(locale, "Subscribe") %> | <%= sub_count_text %>"></b> diff --git a/src/invidious/views/components/video-context-buttons.ecr b/src/invidious/views/components/video-context-buttons.ecr index 385ed6b3..22458a03 100644 --- a/src/invidious/views/components/video-context-buttons.ecr +++ b/src/invidious/views/components/video-context-buttons.ecr @@ -1,6 +1,6 @@ <div class="flex-right flexible"> <div class="icon-buttons"> - <a title="<%=translate(locale, "videoinfo_watch_on_youTube")%>" href="https://www.youtube.com/watch<%=endpoint_params%>"> + <a title="<%=translate(locale, "videoinfo_watch_on_youTube")%>" rel="noreferrer noopener" href="https://www.youtube.com/watch<%=endpoint_params%>"> <i class="icon ion-logo-youtube"></i> </a> <a title="<%=translate(locale, "Audio mode")%>" href="/watch<%=endpoint_params%>&listen=1"> diff --git a/src/invidious/views/feeds/history.ecr b/src/invidious/views/feeds/history.ecr index bda4e1f3..13fe4147 100644 --- a/src/invidious/views/feeds/history.ecr +++ b/src/invidious/views/feeds/history.ecr @@ -37,7 +37,7 @@ </a> <div class="top-left-overlay"><div class="watched"> - <form data-onsubmit="return_false" action="/watch_ajax?action_mark_unwatched=1&id=<%= item %>&referer=<%= env.get("current_page") %>" method="post"> + <form data-onsubmit="return_false" action="/watch_ajax?action=mark_unwatched&id=<%= item %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> <button type="submit" class="pure-button pure-button-secondary low-profile" data-onclick="mark_unwatched" data-id="<%= item %>"><i class="icon ion-md-trash"></i></button> diff --git a/src/invidious/views/playlist.ecr b/src/invidious/views/playlist.ecr index 24ba437d..c27ddba6 100644 --- a/src/invidious/views/playlist.ecr +++ b/src/invidious/views/playlist.ecr @@ -83,7 +83,7 @@ <% if !playlist.is_a? InvidiousPlaylist %> <div class="pure-u-2-3"> - <a href="https://www.youtube.com/playlist?list=<%= playlist.id %>"> + <a rel="noreferrer noopener" href="https://www.youtube.com/playlist?list=<%= playlist.id %>"> <%= translate(locale, "View playlist on YouTube") %> </a> <span> | </span> diff --git a/src/invidious/views/user/preferences.ecr b/src/invidious/views/user/preferences.ecr index 55349c5a..cf8b5593 100644 --- a/src/invidious/views/user/preferences.ecr +++ b/src/invidious/views/user/preferences.ecr @@ -13,6 +13,11 @@ </div> <div class="pure-control-group"> + <label for="preload"><%= translate(locale, "preferences_preload_label") %></label> + <input name="preload" id="preload" type="checkbox" <% if preferences.preload %>checked<% end %>> + </div> + + <div class="pure-control-group"> <label for="autoplay"><%= translate(locale, "preferences_autoplay_label") %></label> <input name="autoplay" id="autoplay" type="checkbox" <% if preferences.autoplay %>checked<% end %>> </div> @@ -310,7 +315,7 @@ <div class="pure-control-group"> <label for="modified_source_code_url"><%= translate(locale, "adminprefs_modified_source_code_url_label") %></label> - <input name="modified_source_code_url" id="modified_source_code_url" type="input" <% if CONFIG.modified_source_code_url %>checked<% end %>> + <input name="modified_source_code_url" id="modified_source_code_url" type="url" value="<%= CONFIG.modified_source_code_url %>"> </div> <% end %> diff --git a/src/invidious/views/user/subscription_manager.ecr b/src/invidious/views/user/subscription_manager.ecr index c9801f09..d566e228 100644 --- a/src/invidious/views/user/subscription_manager.ecr +++ b/src/invidious/views/user/subscription_manager.ecr @@ -37,7 +37,7 @@ <div class="pure-u-2-5"></div> <div class="pure-u-1-5" style="text-align:right"> <h3 style="padding-right:0.5em"> - <form data-onsubmit="return_false" action="/subscription_ajax?action_remove_subscriptions=1&c=<%= channel.id %>&referer=<%= env.get("current_page") %>" method="post"> + <form data-onsubmit="return_false" action="/subscription_ajax?action=remove_subscriptions&c=<%= channel.id %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>"> <input style="all:unset" type="submit" data-onclick="remove_subscription" data-ucid="<%= channel.id %>" value="<%= translate(locale, "unsubscribe") %>"> </form> diff --git a/src/invidious/views/user/token_manager.ecr b/src/invidious/views/user/token_manager.ecr index a73fa048..8431deb0 100644 --- a/src/invidious/views/user/token_manager.ecr +++ b/src/invidious/views/user/token_manager.ecr @@ -29,7 +29,7 @@ </div> <div class="pure-u-1-5" style="text-align:right"> <h3 style="padding-right:0.5em"> - <form data-onsubmit="return_false" action="/token_ajax?action_revoke_token=1&session=<%= token[:session] %>&referer=<%= env.get("current_page") %>" method="post"> + <form data-onsubmit="return_false" action="/token_ajax?action=revoke_token&session=<%= token[:session] %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>"> <input style="all:unset" type="submit" data-onclick="revoke_token" data-session="<%= token[:session] %>" value="<%= translate(locale, "revoke") %>"> </form> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 7a1cf2c3..6f9ced6f 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -10,7 +10,7 @@ <meta property="og:site_name" content="<%= author %> | Invidious"> <meta property="og:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>"> <meta property="og:title" content="<%= title %>"> -<meta property="og:image" content="/vi/<%= video.id %>/maxres.jpg"> +<meta property="og:image" content="<%= HOST_URL %>/vi/<%= video.id %>/maxres.jpg"> <meta property="og:description" content="<%= HTML.escape(video.short_description) %>"> <meta property="og:type" content="video.other"> <meta property="og:video:url" content="<%= HOST_URL %>/embed/<%= video.id %>"> @@ -62,7 +62,7 @@ we're going to need to do it here in order to allow for translations. "params" => params, "preferences" => preferences, "premiere_timestamp" => video.premiere_timestamp.try &.to_unix, - "vr" => video.is_vr, + "vr" => video.vr?, "projection_type" => video.projection_type, "local_disabled" => CONFIG.disabled?("local"), "support_reddit" => true @@ -123,8 +123,8 @@ we're going to need to do it here in order to allow for translations. link_yt_embed = IV::HttpServer::Utils.add_params_to_url(link_yt_embed, link_yt_param) end -%> - <a id="link-yt-watch" data-base-url="<%= link_yt_watch %>" href="<%= link_yt_watch %>"><%= translate(locale, "videoinfo_watch_on_youTube") %></a> - (<a id="link-yt-embed" data-base-url="<%= link_yt_embed %>" href="<%= link_yt_embed %>"><%= translate(locale, "videoinfo_youTube_embed_link") %></a>) + <a id="link-yt-watch" rel="noreferrer noopener" data-base-url="<%= link_yt_watch %>" href="<%= link_yt_watch %>"><%= translate(locale, "videoinfo_watch_on_youTube") %></a> + (<a id="link-yt-embed" rel="noreferrer noopener" data-base-url="<%= link_yt_embed %>" href="<%= link_yt_embed %>"><%= translate(locale, "videoinfo_youTube_embed_link") %></a>) </span> <p id="watch-on-another-invidious-instance"> @@ -158,7 +158,7 @@ we're going to need to do it here in order to allow for translations. <% if user %> <% playlists = Invidious::Database::Playlists.select_user_created_playlists(user.email) %> <% if !playlists.empty? %> - <form data-onsubmit="return_false" class="pure-form pure-form-stacked" action="/playlist_ajax" method="post" target="_blank"> + <form data-onsubmit="return_false" class="pure-form pure-form-stacked" action="/playlist_ajax?action=add_video" method="post" target="_blank"> <div class="pure-control-group"> <label for="playlist_id"><%= translate(locale, "Add to playlist: ") %></label> <select style="width:100%" name="playlist_id" id="playlist_id"> @@ -169,7 +169,6 @@ we're going to need to do it here in order to allow for translations. </div> <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> - <input type="hidden" name="action_add_video" value="1"> <input type="hidden" name="video_id" value="<%= video.id %>"> <button data-onclick="add_playlist_video" data-id="<%= video.id %>" type="submit" class="pure-button pure-button-primary"> <b><%= translate(locale, "Add to playlist") %></b> diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index d3dbcc0e..c4a73aa7 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -1,17 +1,6 @@ -def add_yt_headers(request) - request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" - request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" - - request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" - request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" - request.headers["Accept-Language"] ||= "en-us,en;q=0.5" - - # Preserve original cookies and add new YT consent cookie for EU servers - request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}" - if !CONFIG.cookies.empty? - request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" - end -end +# Mapping of subdomain => YoutubeConnectionPool +# This is needed as we may need to access arbitrary subdomains of ytimg +private YTIMG_POOLS = {} of String => YoutubeConnectionPool struct YoutubeConnectionPool property! url : URI @@ -24,17 +13,17 @@ struct YoutubeConnectionPool @pool = build_pool() end - def client(&block) + def client(&) conn = pool.checkout + # Proxy needs to be reinstated every time we get a client from the pool + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + begin response = yield conn rescue ex conn.close - conn = HTTP::Client.new(url) + conn = make_client(url, force_resolve: true) - conn.family = CONFIG.force_resolve - conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC - conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" response = yield conn ensure pool.release(conn) @@ -44,36 +33,84 @@ struct YoutubeConnectionPool end private def build_pool - DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do - conn = HTTP::Client.new(url) - conn.family = CONFIG.force_resolve - conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC - conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" - conn + options = DB::Pool::Options.new( + initial_pool_size: 0, + max_pool_size: capacity, + max_idle_pool_size: capacity, + checkout_timeout: timeout + ) + + DB::Pool(HTTP::Client).new(options) do + next make_client(url, force_resolve: true) end end end -def make_client(url : URI, region = nil, force_resolve : Bool = false) +def add_yt_headers(request) + request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" + request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" + + request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" + request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + request.headers["Accept-Language"] ||= "en-us,en;q=0.5" + + # Preserve original cookies and add new YT consent cookie for EU servers + request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}" + if !CONFIG.cookies.empty? + request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" + end +end + +def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false) client = HTTP::Client.new(url) + client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy # Force the usage of a specific configured IP Family if force_resolve client.family = CONFIG.force_resolve + client.family = Socket::Family::INET if client.family == Socket::Family::UNSPEC end - client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" + client.before_request { |r| add_yt_headers(r) } if url.host.try &.ends_with?("youtube.com") || force_youtube_headers client.read_timeout = 10.seconds client.connect_timeout = 10.seconds return client end -def make_client(url : URI, region = nil, force_resolve : Bool = false, &block) - client = make_client(url, region, force_resolve) +def make_client(url : URI, region = nil, force_resolve : Bool = false, &) + client = make_client(url, region, force_resolve: force_resolve) begin yield client ensure client.close end end + +def make_configured_http_proxy_client + # This method is only called when configuration for an HTTP proxy are set + config_proxy = CONFIG.http_proxy.not_nil! + + return HTTP::Proxy::Client.new( + config_proxy.host, + config_proxy.port, + + username: config_proxy.user, + password: config_proxy.password, + ) +end + +# Fetches a HTTP pool for the specified subdomain of ytimg.com +# +# Creates a new one when the specified pool for the subdomain does not exist +def get_ytimg_pool(subdomain) + if pool = YTIMG_POOLS[subdomain]? + return pool + else + LOGGER.info("ytimg_pool: Creating a new HTTP pool for \"https://#{subdomain}.ytimg.com\"") + pool = YoutubeConnectionPool.new(URI.parse("https://#{subdomain}.ytimg.com"), capacity: CONFIG.pool_size) + YTIMG_POOLS[subdomain] = pool + + return pool + end +end diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 0e72957e..edd7bf1b 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -21,6 +21,7 @@ private ITEM_PARSERS = { Parsers::ItemSectionRendererParser, Parsers::ContinuationItemRendererParser, Parsers::HashtagRendererParser, + Parsers::LockupViewModelParser, } private alias InitialData = Hash(String, JSON::Any) @@ -66,6 +67,8 @@ private module Parsers author_id = author_fallback.id end + author_thumbnail = item_contents.dig?("channelThumbnailSupportedRenderers", "channelThumbnailWithLinkRenderer", "thumbnail", "thumbnails", 0, "url").try &.as_s + author_verified = has_verified_badge?(item_contents["ownerBadges"]?) # For live videos (and possibly recently premiered videos) there is no published information. @@ -108,22 +111,30 @@ private module Parsers length_seconds = 0 end - live_now = false - paid = false - premium = false - premiere_timestamp = item_contents.dig?("upcomingEventData", "startTime").try { |t| Time.unix(t.as_s.to_i64) } - + badges = VideoBadges::None item_contents["badges"]?.try &.as_a.each do |badge| b = badge["metadataBadgeRenderer"] case b["label"].as_s - when "LIVE NOW" - live_now = true - when "New", "4K", "CC" - # TODO + when "LIVE" + badges |= VideoBadges::LiveNow + when "New" + badges |= VideoBadges::New + when "4K" + badges |= VideoBadges::FourK + when "8K" + badges |= VideoBadges::EightK + when "VR180" + badges |= VideoBadges::VR180 + when "360°" + badges |= VideoBadges::VR360 + when "3D" + badges |= VideoBadges::ThreeD + when "CC" + badges |= VideoBadges::ClosedCaptions when "Premium" # TODO: Potentially available as item_contents["topStandaloneBadge"]["metadataBadgeRenderer"] - premium = true + badges |= VideoBadges::Premium else nil # Ignore end end @@ -137,10 +148,10 @@ private module Parsers views: view_count, description_html: description_html, length_seconds: length_seconds, - live_now: live_now, - premium: premium, premiere_timestamp: premiere_timestamp, author_verified: author_verified, + author_thumbnail: author_thumbnail, + badges: badges, }) end @@ -460,9 +471,9 @@ private module Parsers # Parses an InnerTube richItemRenderer into a SearchVideo. # Returns nil when the given object isn't a RichItemRenderer # - # A richItemRenderer seems to be a simple wrapper for a videoRenderer, used - # by the result page for hashtags and for the podcast tab on channels. - # It is located inside a continuationItems container for hashtags. + # A richItemRenderer seems to be a simple wrapper for a various other types, + # used on the hashtags result page and the channel podcast tab. It is located + # itself inside a richGridRenderer container. # module RichItemRendererParser def self.process(item : JSON::Any, author_fallback : AuthorFallback) @@ -475,6 +486,8 @@ private module Parsers child = VideoRendererParser.process(item_contents, author_fallback) child ||= ReelItemRendererParser.process(item_contents, author_fallback) child ||= PlaylistRendererParser.process(item_contents, author_fallback) + child ||= LockupViewModelParser.process(item_contents, author_fallback) + child ||= ShortsLockupViewModelParser.process(item_contents, author_fallback) return child end @@ -489,6 +502,9 @@ private module Parsers # reelItemRenderer items are used in the new (2022) channel layout, # in the "shorts" tab. # + # NOTE: As of 10/2024, it might have been fully replaced by shortsLockupViewModel + # TODO: Confirm that hypothesis + # module ReelItemRendererParser def self.process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item["reelItemRenderer"]? @@ -564,10 +580,140 @@ private module Parsers views: view_count, description_html: "", length_seconds: duration, - live_now: false, - premium: false, premiere_timestamp: Time.unix(0), author_verified: false, + author_thumbnail: nil, + badges: VideoBadges::None, + }) + end + + def self.parser_name + return {{@type.name}} + end + end + + # Parses an InnerTube lockupViewModel into a SearchPlaylist. + # Returns nil when the given object is not a lockupViewModel. + # + # This structure is present since November 2024 on the "podcasts" and + # "playlists" tabs of the channel page. It is usually encapsulated in either + # a richItemRenderer or a richGridRenderer. + # + module LockupViewModelParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["lockupViewModel"]? + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + playlist_id = item_contents["contentId"].as_s + + thumbnail_view_model = item_contents.dig( + "contentImage", "collectionThumbnailViewModel", + "primaryThumbnail", "thumbnailViewModel" + ) + + thumbnail = thumbnail_view_model.dig("image", "sources", 0, "url").as_s + + # This complicated sequences tries to extract the following data structure: + # "overlays": [{ + # "thumbnailOverlayBadgeViewModel": { + # "thumbnailBadges": [{ + # "thumbnailBadgeViewModel": { + # "text": "430 episodes", + # "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT" + # } + # }] + # } + # }] + # + # NOTE: this simplistic `.to_i` conversion might not work on larger + # playlists and hasn't been tested. + video_count = thumbnail_view_model.dig("overlays").as_a + .compact_map(&.dig?("thumbnailOverlayBadgeViewModel", "thumbnailBadges").try &.as_a) + .flatten + .find(nil, &.dig?("thumbnailBadgeViewModel", "text").try { |node| + {"episodes", "videos"}.any? { |str| node.as_s.ends_with?(str) } + }) + .try &.dig("thumbnailBadgeViewModel", "text").as_s.to_i(strict: false) + + metadata = item_contents.dig("metadata", "lockupMetadataViewModel") + title = metadata.dig("title", "content").as_s + + # TODO: Retrieve "updated" info from metadata parts + # rows = metadata.dig("metadata", "contentMetadataViewModel", "metadataRows").as_a + # parts_text = rows.map(&.dig?("metadataParts", "text", "content").try &.as_s) + # One of these parts should contain a string like: "Updated 2 days ago" + + # TODO: Maybe add a button to access the first video of the playlist? + # item_contents.dig("rendererContext", "commandContext", "onTap", "innertubeCommand", "watchEndpoint") + # Available fields: "videoId", "playlistId", "params" + + return SearchPlaylist.new({ + title: title, + id: playlist_id, + author: author_fallback.name, + ucid: author_fallback.id, + video_count: video_count || -1, + videos: [] of SearchPlaylistVideo, + thumbnail: thumbnail, + author_verified: false, + }) + end + + def self.parser_name + return {{@type.name}} + end + end + + # Parses an InnerTube shortsLockupViewModel into a SearchVideo. + # Returns nil when the given object is not a shortsLockupViewModel. + # + # This structure is present since around October 2024 on the "shorts" tab of + # the channel page and likely replaces the reelItemRenderer structure. It is + # usually (always?) encapsulated in a richItemRenderer. + # + module ShortsLockupViewModelParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["shortsLockupViewModel"]? + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + # TODO: Maybe add support for "oardefault.jpg" thumbnails? + # thumbnail = item_contents.dig("thumbnail", "sources", 0, "url").as_s + # Gives: https://i.ytimg.com/vi/{video_id}/oardefault.jpg?... + + video_id = item_contents.dig( + "onTap", "innertubeCommand", "reelWatchEndpoint", "videoId" + ).as_s + + title = item_contents.dig("overlayMetadata", "primaryText", "content").as_s + + view_count = short_text_to_number( + item_contents.dig("overlayMetadata", "secondaryText", "content").as_s + ) + + # Approximate to one minute, as "shorts" generally don't exceed that. + # NOTE: The actual duration is not provided by Youtube anymore. + # TODO: Maybe use -1 as an error value and handle that on the frontend? + duration = 60_i32 + + SearchVideo.new({ + title: title, + id: video_id, + author: author_fallback.name, + ucid: author_fallback.id, + published: Time.unix(0), + views: view_count, + description_html: "", + length_seconds: duration, + premiere_timestamp: Time.unix(0), + author_verified: false, + author_thumbnail: nil, + badges: VideoBadges::None, }) end @@ -856,7 +1002,7 @@ end # # This function yields the container so that items can be parsed separately. # -def extract_items(initial_data : InitialData, &block) +def extract_items(initial_data : InitialData, &) if unpackaged_data = initial_data["contents"]?.try &.as_h elsif unpackaged_data = initial_data["response"]?.try &.as_h elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 1).try &.as_h @@ -883,7 +1029,7 @@ end def extract_items( initial_data : InitialData, author_fallback : String? = nil, - author_id_fallback : String? = nil + author_id_fallback : String? = nil, ) : {Array(SearchItem), String?} items = [] of SearchItem continuation = nil diff --git a/src/invidious/yt_backend/extractors_utils.cr b/src/invidious/yt_backend/extractors_utils.cr index 11d95958..c83a2de5 100644 --- a/src/invidious/yt_backend/extractors_utils.cr +++ b/src/invidious/yt_backend/extractors_utils.cr @@ -83,5 +83,5 @@ end def extract_selected_tab(tabs) # Extract the selected tab from the array of tabs Youtube returns - return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"]?.try &.as_bool)[0]["tabRenderer"] + return tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"]?.try &.as_bool)[0]["tabRenderer"] end diff --git a/src/invidious/yt_backend/url_sanitizer.cr b/src/invidious/yt_backend/url_sanitizer.cr new file mode 100644 index 00000000..d539dadb --- /dev/null +++ b/src/invidious/yt_backend/url_sanitizer.cr @@ -0,0 +1,121 @@ +require "uri" + +module UrlSanitizer + extend self + + ALLOWED_QUERY_PARAMS = { + channel: ["u", "user", "lb"], + playlist: ["list"], + search: ["q", "search_query", "sp"], + watch: [ + "v", # Video ID + "list", "index", # Playlist-related + "playlist", # Unnamed playlist (id,id,id,...) (embed-only?) + "t", "time_continue", "start", "end", # Timestamp + "lc", # Highlighted comment (watch page only) + ], + } + + # Returns whether the given string is an ASCII word. This is the same as + # running the following regex in US-ASCII locale: /^[\w-]+$/ + private def ascii_word?(str : String) : Bool + return false if str.bytesize != str.size + + str.each_byte do |byte| + next if 'a'.ord <= byte <= 'z'.ord + next if 'A'.ord <= byte <= 'Z'.ord + next if '0'.ord <= byte <= '9'.ord + next if byte == '-'.ord || byte == '_'.ord + + return false + end + + return true + end + + # Return which kind of parameters are allowed based on the + # first path component (breadcrumb 0). + private def determine_allowed(path_root : String) + case path_root + when "watch", "w", "v", "embed", "e", "shorts", "clip" + return :watch + when .starts_with?("@"), "c", "channel", "user", "profile", "attribution_link" + return :channel + when "playlist", "mix" + return :playlist + when "results", "search" + return :search + else # hashtag, post, trending, brand URLs, etc.. + return nil + end + end + + # Create a new URI::Param containing only the allowed parameters + private def copy_params(unsafe_params : URI::Params, allowed_type) : URI::Params + new_params = URI::Params.new + + ALLOWED_QUERY_PARAMS[allowed_type].each do |name| + if unsafe_params[name]? + # Only copy the last parameter, in case there is more than one + new_params[name] = unsafe_params.fetch_all(name)[-1] + end + end + + return new_params + end + + # Transform any user-supplied youtube URL into something we can trust + # and use across the code. + def process(str : String) : URI + # Because URI follows RFC3986 specifications, URL without a scheme + # will be parsed as a relative path. So we have to add a scheme ourselves. + str = "https://#{str}" if !str.starts_with?(/https?:\/\//) + + unsafe_uri = URI.parse(str) + unsafe_host = unsafe_uri.host + unsafe_path = unsafe_uri.path + + new_uri = URI.new(path: "/") + + # Redirect to homepage for bogus URLs + return new_uri if (unsafe_host.nil? || unsafe_path.nil?) + + breadcrumbs = unsafe_path + .split('/', remove_empty: true) + .compact_map do |bc| + # Exclude attempts at path trasversal + next if bc == "." || bc == ".." + + # Non-alnum characters are unlikely in a genuine URL + next if !ascii_word?(bc) + + bc + end + + # If nothing remains, it's either a legit URL to the homepage + # (who does that!?) or because we filtered some junk earlier. + return new_uri if breadcrumbs.empty? + + # Replace the original query parameters with the sanitized ones + case unsafe_host + when .ends_with?("youtube.com") + # Use our sanitized path (not forgetting the leading '/') + new_uri.path = "/#{breadcrumbs.join('/')}" + + # Then determine which params are allowed, and copy them over + if allowed = determine_allowed(breadcrumbs[0]) + new_uri.query_params = copy_params(unsafe_uri.query_params, allowed) + end + when "youtu.be" + # Always redirect to the watch page + new_uri.path = "/watch" + + new_params = copy_params(unsafe_uri.query_params, :watch) + new_params["v"] = breadcrumbs[0] + + new_uri.query_params = new_params + end + + return new_uri + end +end diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 727ce9a3..ec080d8c 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -5,14 +5,11 @@ module YoutubeAPI extend self - private DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" - private ANDROID_API_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w" - # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history - private ANDROID_APP_VERSION = "19.14.42" - private ANDROID_USER_AGENT = "com.google.android.youtube/19.14.42 (Linux; U; Android 12; US) gzip" - private ANDROID_SDK_VERSION = 31_i64 + private ANDROID_APP_VERSION = "19.32.34" private ANDROID_VERSION = "12" + private ANDROID_USER_AGENT = "com.google.android.youtube/#{ANDROID_APP_VERSION} (Linux; U; Android #{ANDROID_VERSION}; US) gzip" + private ANDROID_SDK_VERSION = 31_i64 private ANDROID_TS_APP_VERSION = "1.9" private ANDROID_TS_USER_AGENT = "com.google.android.youtube/1.9 (Linux; U; Android 12; US) gzip" @@ -20,9 +17,9 @@ module YoutubeAPI # For Apple device names, see https://gist.github.com/adamawolf/3048717 # For iOS versions, see https://en.wikipedia.org/wiki/IOS_version_history#Releases, # then go to the dedicated article of the major version you want. - private IOS_APP_VERSION = "19.16.3" - private IOS_USER_AGENT = "com.google.ios.youtube/19.16.3 (iPhone14,5; U; CPU iOS 17_4 like Mac OS X;)" - private IOS_VERSION = "17.4.0.21E219" # Major.Minor.Patch.Build + private IOS_APP_VERSION = "19.32.8" + private IOS_USER_AGENT = "com.google.ios.youtube/#{IOS_APP_VERSION} (iPhone14,5; U; CPU iOS 17_6 like Mac OS X;)" + private IOS_VERSION = "17.6.1.21G93" # Major.Minor.Patch.Build private WINDOWS_VERSION = "10.0" @@ -32,6 +29,7 @@ module YoutubeAPI WebEmbeddedPlayer WebMobile WebScreenEmbed + WebCreator Android AndroidEmbeddedPlayer @@ -51,8 +49,7 @@ module YoutubeAPI ClientType::Web => { name: "WEB", name_proto: "1", - version: "2.20240304.00.00", - api_key: DEFAULT_API_KEY, + version: "2.20240814.00.00", screen: "WATCH_FULL_SCREEN", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -61,8 +58,7 @@ module YoutubeAPI ClientType::WebEmbeddedPlayer => { name: "WEB_EMBEDDED_PLAYER", name_proto: "56", - version: "1.20240303.00.00", - api_key: DEFAULT_API_KEY, + version: "1.20240812.01.00", screen: "EMBED", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -71,8 +67,7 @@ module YoutubeAPI ClientType::WebMobile => { name: "MWEB", name_proto: "2", - version: "2.20240304.08.00", - api_key: DEFAULT_API_KEY, + version: "2.20240813.02.00", os_name: "Android", os_version: ANDROID_VERSION, platform: "MOBILE", @@ -80,13 +75,20 @@ module YoutubeAPI ClientType::WebScreenEmbed => { name: "WEB", name_proto: "1", - version: "2.20240304.00.00", - api_key: DEFAULT_API_KEY, + version: "2.20240814.00.00", screen: "EMBED", os_name: "Windows", os_version: WINDOWS_VERSION, platform: "DESKTOP", }, + ClientType::WebCreator => { + name: "WEB_CREATOR", + name_proto: "62", + version: "1.20240918.03.00", + os_name: "Windows", + os_version: WINDOWS_VERSION, + platform: "DESKTOP", + }, # Android @@ -94,7 +96,6 @@ module YoutubeAPI name: "ANDROID", name_proto: "3", version: ANDROID_APP_VERSION, - api_key: ANDROID_API_KEY, android_sdk_version: ANDROID_SDK_VERSION, user_agent: ANDROID_USER_AGENT, os_name: "Android", @@ -105,13 +106,11 @@ module YoutubeAPI name: "ANDROID_EMBEDDED_PLAYER", name_proto: "55", version: ANDROID_APP_VERSION, - api_key: "AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw", }, ClientType::AndroidScreenEmbed => { name: "ANDROID", name_proto: "3", version: ANDROID_APP_VERSION, - api_key: DEFAULT_API_KEY, screen: "EMBED", android_sdk_version: ANDROID_SDK_VERSION, user_agent: ANDROID_USER_AGENT, @@ -123,7 +122,6 @@ module YoutubeAPI name: "ANDROID_TESTSUITE", name_proto: "30", version: ANDROID_TS_APP_VERSION, - api_key: ANDROID_API_KEY, android_sdk_version: ANDROID_SDK_VERSION, user_agent: ANDROID_TS_USER_AGENT, os_name: "Android", @@ -137,7 +135,6 @@ module YoutubeAPI name: "IOS", name_proto: "5", version: IOS_APP_VERSION, - api_key: "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc", user_agent: IOS_USER_AGENT, device_make: "Apple", device_model: "iPhone14,5", @@ -149,7 +146,6 @@ module YoutubeAPI name: "IOS_MESSAGES_EXTENSION", name_proto: "66", version: IOS_APP_VERSION, - api_key: DEFAULT_API_KEY, user_agent: IOS_USER_AGENT, device_make: "Apple", device_model: "iPhone14,5", @@ -160,9 +156,8 @@ module YoutubeAPI ClientType::IOSMusic => { name: "IOS_MUSIC", name_proto: "26", - version: "6.42", - api_key: "AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s", - user_agent: "com.google.ios.youtubemusic/6.42 (iPhone14,5; U; CPU iOS 17_4 like Mac OS X;)", + version: "7.14", + user_agent: "com.google.ios.youtubemusic/7.14 (iPhone14,5; U; CPU iOS 17_6 like Mac OS X;)", device_make: "Apple", device_model: "iPhone14,5", os_name: "iPhone", @@ -175,14 +170,12 @@ module YoutubeAPI ClientType::TvHtml5 => { name: "TVHTML5", name_proto: "7", - version: "7.20240304.10.00", - api_key: DEFAULT_API_KEY, + version: "7.20240813.07.00", }, ClientType::TvHtml5ScreenEmbed => { name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER", name_proto: "85", version: "2.0", - api_key: DEFAULT_API_KEY, screen: "EMBED", }, } @@ -218,7 +211,7 @@ module YoutubeAPI def initialize( *, @client_type = ClientType::Web, - @region = "US" + @region = "US", ) end @@ -238,11 +231,6 @@ module YoutubeAPI end # :ditto: - def api_key : String - HARDCODED_CLIENTS[@client_type][:api_key] - end - - # :ditto: def screen : String HARDCODED_CLIENTS[@client_type][:screen]? || "" end @@ -293,7 +281,7 @@ module YoutubeAPI # Return, as a Hash, the "context" data required to request the # youtube API endpoints. # - private def make_context(client_config : ClientConfig | Nil) : Hash + private def make_context(client_config : ClientConfig | Nil, video_id = "dQw4w9WgXcQ") : Hash # Use the default client config if nil is passed client_config ||= DEFAULT_CLIENT_CONFIG @@ -313,7 +301,7 @@ module YoutubeAPI if client_config.screen == "EMBED" client_context["thirdParty"] = { - "embedUrl" => "https://www.youtube.com/embed/dQw4w9WgXcQ", + "embedUrl" => "https://www.youtube.com/embed/#{video_id}", } of String => String | Int64 end @@ -341,6 +329,10 @@ module YoutubeAPI client_context["client"]["platform"] = platform end + if CONFIG.visitor_data.is_a?(String) + client_context["client"]["visitorData"] = CONFIG.visitor_data.as(String) + end + return client_context end @@ -378,7 +370,7 @@ module YoutubeAPI browse_id : String, *, # Force the following parameters to be passed by name params : String, - client_config : ClientConfig | Nil = nil + client_config : ClientConfig | Nil = nil, ) # JSON Request data, required by the API data = { @@ -472,21 +464,34 @@ module YoutubeAPI video_id : String, *, # Force the following parameters to be passed by name params : String, - client_config : ClientConfig | Nil = nil + client_config : ClientConfig | Nil = nil, ) + # Playback context, separate because it can be different between clients + playback_ctx = { + "html5Preference" => "HTML5_PREF_WANTS", + "referer" => "https://www.youtube.com/watch?v=#{video_id}", + } of String => String | Int64 + + if {"WEB", "TVHTML5"}.any? { |s| client_config.name.starts_with? s } + if sts = DECRYPT_FUNCTION.try &.get_sts + playback_ctx["signatureTimestamp"] = sts.to_i64 + end + end + # JSON Request data, required by the API data = { "contentCheckOk" => true, "videoId" => video_id, - "context" => self.make_context(client_config), + "context" => self.make_context(client_config, video_id), "racyCheckOk" => true, "user" => { "lockedSafetyMode" => false, }, "playbackContext" => { - "contentPlaybackContext" => { - "html5Preference": "HTML5_PREF_WANTS", - }, + "contentPlaybackContext" => playback_ctx, + }, + "serviceIntegrityDimensions" => { + "poToken" => CONFIG.po_token, }, } @@ -552,7 +557,7 @@ module YoutubeAPI def search( search_query : String, params : String, - client_config : ClientConfig | Nil = nil + client_config : ClientConfig | Nil = nil, ) # JSON Request data, required by the API data = { @@ -578,7 +583,7 @@ module YoutubeAPI def get_transcript( params : String, - client_config : ClientConfig | Nil = nil + client_config : ClientConfig | Nil = nil, ) : Hash(String, JSON::Any) data = { "context" => self.make_context(client_config), @@ -600,13 +605,13 @@ module YoutubeAPI def _post_json( endpoint : String, data : Hash, - client_config : ClientConfig | Nil + client_config : ClientConfig | Nil, ) : Hash(String, JSON::Any) # Use the default client config if nil is passed client_config ||= DEFAULT_CLIENT_CONFIG # Query parameters - url = "#{endpoint}?key=#{client_config.api_key}&prettyPrint=false" + url = "#{endpoint}?prettyPrint=false" headers = HTTP::Headers{ "Content-Type" => "application/json; charset=UTF-8", @@ -620,6 +625,10 @@ module YoutubeAPI headers["User-Agent"] = user_agent end + if CONFIG.visitor_data.is_a?(String) + headers["X-Goog-Visitor-Id"] = CONFIG.visitor_data.as(String) + end + # Logging LOGGER.debug("YoutubeAPI: Using endpoint: \"#{endpoint}\"") LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}") @@ -628,6 +637,11 @@ module YoutubeAPI # Send the POST request body = YT_POOL.client() do |client| client.post(url, headers: headers, body: data.to_json) do |response| + if response.status_code != 200 + raise InfoException.new("Error: non 200 status code. Youtube API returned \ + status code #{response.status_code}. See <a href=\"https://docs.invidious.io/youtube-errors-explained/\"> \ + https://docs.invidious.io/youtube-errors-explained/</a> for troubleshooting.") + end self._decompress(response.body_io, response.headers["Content-Encoding"]?) end end |