diff options
Diffstat (limited to 'src')
90 files changed, 3118 insertions, 2112 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index e0bd0101..b422dcbb 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -23,6 +23,7 @@ require "kilt" require "./ext/kemal_content_for.cr" require "./ext/kemal_static_file_handler.cr" +require "http_proxy" require "athena-negotiation" require "openssl/hmac" require "option_parser" @@ -92,6 +93,10 @@ SOFTWARE = { YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size) +# Image request pool + +GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size) + # CLI Kemal.config.extra_options do |parser| parser.banner = "Usage: invidious [arguments]" @@ -117,6 +122,9 @@ Kemal.config.extra_options do |parser| parser.on("-l LEVEL", "--log-level=LEVEL", "Log level, one of #{LogLevel.values} (default: #{CONFIG.log_level})") do |log_level| CONFIG.log_level = LogLevel.parse(log_level) end + parser.on("-k", "--colorize", "Colorize logs") do + CONFIG.colorize_logs = true + end parser.on("-v", "--version", "Print version") do puts SOFTWARE.to_pretty_json exit @@ -133,7 +141,7 @@ if CONFIG.output.upcase != "STDOUT" FileUtils.mkdir_p(File.dirname(CONFIG.output)) end OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mode: "a") -LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) +LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level, CONFIG.colorize_logs) # Check table integrity Invidious::Database.check_integrity(CONFIG) @@ -153,6 +161,15 @@ Invidious::Database.check_integrity(CONFIG) {% puts "\nDone checking player dependencies, now compiling Invidious...\n" %} {% end %} +# Misc + +DECRYPT_FUNCTION = + if sig_helper_address = CONFIG.signature_server.presence + IV::DecryptFunction.new(sig_helper_address) + else + nil + end + # Start jobs if CONFIG.channel_threads > 0 @@ -163,11 +180,6 @@ if CONFIG.feed_threads > 0 Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB) end -DECRYPT_FUNCTION = DecryptFunction.new(CONFIG.decrypt_polling) -if CONFIG.decrypt_polling - Invidious::Jobs.register Invidious::Jobs::UpdateDecryptFunctionJob.new -end - if CONFIG.statistics_enabled Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, SOFTWARE) end @@ -185,6 +197,8 @@ Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new +Invidious::Jobs.register Invidious::Jobs::InstanceListRefreshJob.new + Invidious::Jobs.start_all def popular_videos diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index 0054f8f2..13909527 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -14,12 +14,14 @@ record AboutChannel, is_family_friendly : Bool, allowed_regions : Array(String), tabs : Array(String), - verified : Bool + tags : Array(String), + verified : Bool, + is_age_gated : Bool def get_about_info(ucid, locale) : AboutChannel begin - # "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"} - initdata = YoutubeAPI.browse(browse_id: ucid, params: "EgVhYm91dA==") + # Fetch channel information from channel home page + initdata = YoutubeAPI.browse(browse_id: ucid, params: "") rescue raise InfoException.new("Could not get channel info.") end @@ -43,36 +45,102 @@ def get_about_info(ucid, locale) : AboutChannel auto_generated = true end - if auto_generated - author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s - author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s - author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s - - # Raises a KeyError on failure. - banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? - banner = banners.try &.[-1]?.try &.["url"].as_s? + tags = [] of String + tab_names = [] of String + total_views = 0_i64 + joined = Time.unix(0) - description_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] + if age_gate_renderer = initdata.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs", 0, "tabRenderer", "content", "sectionListRenderer", "contents", 0, "channelAgeGateRenderer") + description_node = nil + author = age_gate_renderer["channelTitle"].as_s + ucid = initdata.dig("responseContext", "serviceTrackingParams", 0, "params", 0, "value").as_s + author_url = "https://www.youtube.com/channel/#{ucid}" + author_thumbnail = age_gate_renderer.dig("avatar", "thumbnails", 0, "url").as_s + banner = nil + is_family_friendly = false + is_age_gated = true + tab_names = ["videos", "shorts", "streams"] + auto_generated = false else - author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s - author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s - author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s - author_verified = has_verified_badge?(initdata.dig?("header", "c4TabbedHeaderRenderer", "badges")) + if auto_generated + author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s + author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s + author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s + + # Raises a KeyError on failure. + banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? + banner = banners.try &.[-1]?.try &.["url"].as_s? + + description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] + # some channels have the description in a simpleText + # ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/ + description_node = description_base_node.dig?("simpleText") || description_base_node + + tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges") + .try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String + else + author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s + author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s + author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s + author_verified = has_verified_badge?(initdata.dig?("header", "c4TabbedHeaderRenderer", "badges")) - ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s + ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s - # Raises a KeyError on failure. - banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? - banner = banners.try &.[-1]?.try &.["url"].as_s? + # Raises a KeyError on failure. + banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? + banners ||= initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "banner", "imageBannerViewModel", "image", "sources") + banner = banners.try &.[-1]?.try &.["url"].as_s? - # if banner.includes? "channels/c4/default_banner" - # banner = nil - # end + # if banner.includes? "channels/c4/default_banner" + # banner = nil + # end - description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]? - end + description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]? + tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String + end - is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool + is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool + if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]? + # Get the name of the tabs available on this channel + tab_names = tabs_json.as_a.compact_map do |entry| + name = entry.dig?("tabRenderer", "title").try &.as_s.downcase + + # This is a small fix to not add extra code on the HTML side + # I.e, the URL for the "live" tab is .../streams, so use "streams" + # everywhere for the sake of simplicity + (name == "live") ? "streams" : name + end + + # Get the currently active tab ("About") + about_tab = extract_selected_tab(tabs_json) + + # Try to find the about metadata section + channel_about_meta = about_tab.dig?( + "content", + "sectionListRenderer", "contents", 0, + "itemSectionRenderer", "contents", 0, + "channelAboutFullMetadataRenderer" + ) + + if !channel_about_meta.nil? + total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 + + # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. + joined = extract_text(channel_about_meta["joinedDateText"]?) + .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) + + # Normal Auto-generated channels + # https://support.google.com/youtube/answer/2579942 + # For auto-generated channels, channel_about_meta only has + # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] + auto_generated = ( + (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \ + extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" || + channel_about_meta.dig?("links", 0, "channelExternalLinkViewModel", "title", "content").try &.as_s == "Auto-generated by YouTube" + ) + end + end + end allowed_regions = initdata .dig?("microformat", "microformatDataRenderer", "availableCountries") @@ -91,55 +159,18 @@ def get_about_info(ucid, locale) : AboutChannel end end - total_views = 0_i64 - joined = Time.unix(0) - - tab_names = [] of String - - if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]? - # Get the name of the tabs available on this channel - tab_names = tabs_json.as_a.compact_map do |entry| - name = entry.dig?("tabRenderer", "title").try &.as_s.downcase + sub_count = 0 - # This is a small fix to not add extra code on the HTML side - # I.e, the URL for the "live" tab is .../streams, so use "streams" - # everywhere for the sake of simplicity - (name == "live") ? "streams" : name - end - - # Get the currently active tab ("About") - about_tab = extract_selected_tab(tabs_json) - - # Try to find the about metadata section - channel_about_meta = about_tab.dig?( - "content", - "sectionListRenderer", "contents", 0, - "itemSectionRenderer", "contents", 0, - "channelAboutFullMetadataRenderer" - ) - - if !channel_about_meta.nil? - total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 - - # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. - joined = extract_text(channel_about_meta["joinedDateText"]?) - .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) - - # Normal Auto-generated channels - # https://support.google.com/youtube/answer/2579942 - # For auto-generated channels, channel_about_meta only has - # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] - auto_generated = ( - (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \ - extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" - ) + if (metadata_rows = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows").try &.as_a) + metadata_rows.each do |row| + metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") } + if !metadata_part.nil? + sub_count = short_text_to_number(metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32 + end + break if sub_count != 0 end end - sub_count = initdata - .dig?("header", "c4TabbedHeaderRenderer", "subscriberCountText", "simpleText").try &.as_s? - .try { |text| short_text_to_number(text.split(" ")[0]).to_i32 } || 0 - AboutChannel.new( ucid: ucid, author: author, @@ -155,7 +186,9 @@ def get_about_info(ucid, locale) : AboutChannel is_family_friendly: is_family_friendly, allowed_regions: allowed_regions, tabs: tab_names, + tags: tags, verified: author_verified || false, + is_age_gated: is_age_gated || false, ) end diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index c3d6124f..1478c8fc 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -93,7 +93,7 @@ struct ChannelVideo def to_tuple {% begin %} { - {{*@type.instance_vars.map(&.name)}} + {{@type.instance_vars.map(&.name).splat}} } {% end %} end @@ -223,7 +223,7 @@ def fetch_channel(ucid, pull_all_videos : Bool) length_seconds = channel_video.try &.length_seconds length_seconds ||= 0 - live_now = channel_video.try &.live_now + live_now = channel_video.try &.badges.live_now? live_now ||= false premiere_timestamp = channel_video.try &.premiere_timestamp @@ -232,7 +232,7 @@ def fetch_channel(ucid, pull_all_videos : Bool) id: video_id, title: title, published: published, - updated: Time.utc, + updated: updated, ucid: ucid, author: author, length_seconds: length_seconds, @@ -275,7 +275,7 @@ def fetch_channel(ucid, pull_all_videos : Bool) ucid: video.ucid, author: video.author, length_seconds: video.length_seconds, - live_now: video.live_now, + live_now: video.badges.live_now?, premiere_timestamp: video.premiere_timestamp, views: video.views, }) diff --git a/src/invidious/channels/community.cr b/src/invidious/channels/community.cr index 791f1641..49ffd990 100644 --- a/src/invidious/channels/community.cr +++ b/src/invidious/channels/community.cr @@ -24,7 +24,33 @@ def fetch_channel_community(ucid, cursor, locale, format, thin_mode) return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode) end -def extract_channel_community(items, *, ucid, locale, format, thin_mode) +def fetch_channel_community_post(ucid, post_id, locale, format, thin_mode) + object = { + "2:string" => "community", + "25:embedded" => { + "22:string" => post_id.to_s, + }, + "45:embedded" => { + "2:varint" => 1_i64, + "3:varint" => 1_i64, + }, + } + params = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + initial_data = YoutubeAPI.browse(ucid, params: params) + + items = [] of JSON::Any + extract_items(initial_data) do |item| + items << item + end + + return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode, is_single_post: true) +end + +def extract_channel_community(items, *, ucid, locale, format, thin_mode, is_single_post : Bool = false) if message = items[0]["messageRenderer"]? error_message = (message["text"]["simpleText"]? || message["text"]["runs"]?.try &.[0]?.try &.["text"]?) @@ -39,6 +65,9 @@ def extract_channel_community(items, *, ucid, locale, format, thin_mode) response = JSON.build do |json| json.object do json.field "authorId", ucid + if is_single_post + json.field "singlePost", true + end json.field "comments" do json.array do items.each do |post| @@ -240,8 +269,10 @@ def extract_channel_community(items, *, ucid, locale, format, thin_mode) end end end - if cont = items.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token") - json.field "continuation", extract_channel_community_cursor(cont.as_s) + if !is_single_post + if cont = items.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token") + json.field "continuation", extract_channel_community_cursor(cont.as_s) + end end end end diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index beb86e08..96400f47 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -1,73 +1,3 @@ -def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - object_inner_2 = { - "2:0:embedded" => { - "1:0:varint" => 0_i64, - }, - "5:varint" => 50_i64, - "6:varint" => 1_i64, - "7:varint" => (page * 30).to_i64, - "9:varint" => 1_i64, - "10:varint" => 0_i64, - } - - object_inner_2_encoded = object_inner_2 - .try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - sort_by_numerical = - case sort_by - when "newest" then 1_i64 - when "popular" then 2_i64 - when "oldest" then 4_i64 - else 1_i64 # Fallback to "newest" - end - - object_inner_1 = { - "110:embedded" => { - "3:embedded" => { - "15:embedded" => { - "1:embedded" => { - "1:string" => object_inner_2_encoded, - }, - "2:embedded" => { - "1:string" => "00000000-0000-0000-0000-000000000000", - }, - "3:varint" => sort_by_numerical, - }, - }, - }, - } - - object_inner_1_encoded = object_inner_1 - .try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:string" => object_inner_1_encoded, - "35:string" => "browse-feed#{ucid}videos102", - }, - } - - continuation = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return continuation -end - -# Used in bypass_captcha_job.cr -def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2) - return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" -end - module Invidious::Channel::Tabs extend self @@ -75,10 +5,6 @@ module Invidious::Channel::Tabs # Regular videos # ------------------- - def make_initial_video_ctoken(ucid, sort_by) : String - return produce_channel_videos_continuation(ucid, sort_by: sort_by) - end - # Wrapper for AboutChannel, as we still need to call get_videos with # an author name and ucid directly (e.g in RSS feeds). # TODO: figure out how to get rid of that @@ -100,7 +26,7 @@ module Invidious::Channel::Tabs end def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest") - continuation ||= make_initial_video_ctoken(ucid, sort_by) + continuation ||= make_initial_videos_ctoken(ucid, sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, author, ucid) @@ -129,14 +55,10 @@ module Invidious::Channel::Tabs # Shorts # ------------------- - def get_shorts(channel : AboutChannel, continuation : String? = nil) - if continuation.nil? - # EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts" - # TODO: try to extract the continuation tokens that allows other sorting options - initial_data = YoutubeAPI.browse(channel.ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D") - else - initial_data = YoutubeAPI.browse(continuation: continuation) - end + def get_shorts(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") + continuation ||= make_initial_shorts_ctoken(channel.ucid, sort_by) + initial_data = YoutubeAPI.browse(continuation: continuation) + return extract_items(initial_data, channel.author, channel.ucid) end @@ -144,21 +66,17 @@ module Invidious::Channel::Tabs # Livestreams # ------------------- - def get_livestreams(channel : AboutChannel, continuation : String? = nil) - if continuation.nil? - # EgdzdHJlYW1z8gYECgJ6AA%3D%3D is the protobuf object to load "streams" - initial_data = YoutubeAPI.browse(channel.ucid, params: "EgdzdHJlYW1z8gYECgJ6AA%3D%3D") - else - initial_data = YoutubeAPI.browse(continuation: continuation) - end + def get_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") + continuation ||= make_initial_livestreams_ctoken(channel.ucid, sort_by) + initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, channel.author, channel.ucid) end - def get_60_livestreams(channel : AboutChannel, continuation : String? = nil) + def get_60_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") if continuation.nil? - # Fetch the first "page" of streams - items, next_continuation = get_livestreams(channel) + # Fetch the first "page" of stream + items, next_continuation = get_livestreams(channel, sort_by: sort_by) else # Fetch a "page" of streams using the given continuation token items, next_continuation = get_livestreams(channel, continuation: continuation) @@ -173,4 +91,102 @@ module Invidious::Channel::Tabs return items, next_continuation end + + # ------------------- + # C-tokens + # ------------------- + + private def sort_options_videos_short(sort_by : String) + case sort_by + when "newest" then return 4_i64 + when "popular" then return 2_i64 + when "oldest" then return 5_i64 + else return 4_i64 # Fallback to "newest" + end + end + + # Generate the initial "continuation token" to get the first page of the + # "videos" tab. The following page requires the ctoken provided in that + # first page, and so on. + private def make_initial_videos_ctoken(ucid : String, sort_by = "newest") + object = { + "15:embedded" => { + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "4:varint" => sort_options_videos_short(sort_by), + }, + } + + return channel_ctoken_wrap(ucid, object) + end + + # Generate the initial "continuation token" to get the first page of the + # "shorts" tab. The following page requires the ctoken provided in that + # first page, and so on. + private def make_initial_shorts_ctoken(ucid : String, sort_by = "newest") + object = { + "10:embedded" => { + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "4:varint" => sort_options_videos_short(sort_by), + }, + } + + return channel_ctoken_wrap(ucid, object) + end + + # Generate the initial "continuation token" to get the first page of the + # "livestreams" tab. The following page requires the ctoken provided in that + # first page, and so on. + private def make_initial_livestreams_ctoken(ucid : String, sort_by = "newest") + sort_by_numerical = + case sort_by + when "newest" then 12_i64 + when "popular" then 14_i64 + when "oldest" then 13_i64 + else 12_i64 # Fallback to "newest" + end + + object = { + "14:embedded" => { + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "5:varint" => sort_by_numerical, + }, + } + + return channel_ctoken_wrap(ucid, object) + end + + # The protobuf structure common between videos/shorts/livestreams + private def channel_ctoken_wrap(ucid : String, object) + object_inner = { + "110:embedded" => { + "3:embedded" => object, + }, + } + + object_inner_encoded = object_inner + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + object = { + "80226972:embedded" => { + "2:string" => ucid, + "3:string" => object_inner_encoded, + }, + } + + continuation = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + return continuation + end end diff --git a/src/invidious/comments/content.cr b/src/invidious/comments/content.cr index c8cdc2df..1f55bfe6 100644 --- a/src/invidious/comments/content.cr +++ b/src/invidious/comments/content.cr @@ -5,35 +5,35 @@ def text_to_parsed_content(text : String) : JSON::Any # In first case line is just a simple node before # check patterns inside line # { 'text': line } - currentNodes = [] of JSON::Any - initialNode = {"text" => line} - currentNodes << (JSON.parse(initialNode.to_json)) + current_nodes = [] of JSON::Any + initial_node = {"text" => line} + current_nodes << (JSON.parse(initial_node.to_json)) # For each match with url pattern, get last node and preserve # last node before create new node with url information # { 'text': match, 'navigationEndpoint': { 'urlEndpoint' : 'url': match } } - line.scan(/https?:\/\/[^ ]*/).each do |urlMatch| + line.scan(/https?:\/\/[^ ]*/).each do |url_match| # Retrieve last node and update node without match - lastNode = currentNodes[currentNodes.size - 1].as_h - splittedLastNode = lastNode["text"].as_s.split(urlMatch[0]) - lastNode["text"] = JSON.parse(splittedLastNode[0].to_json) - currentNodes[currentNodes.size - 1] = JSON.parse(lastNode.to_json) + last_node = current_nodes[-1].as_h + splitted_last_node = last_node["text"].as_s.split(url_match[0]) + last_node["text"] = JSON.parse(splitted_last_node[0].to_json) + current_nodes[-1] = JSON.parse(last_node.to_json) # Create new node with match and navigation infos - currentNode = {"text" => urlMatch[0], "navigationEndpoint" => {"urlEndpoint" => {"url" => urlMatch[0]}}} - currentNodes << (JSON.parse(currentNode.to_json)) + current_node = {"text" => url_match[0], "navigationEndpoint" => {"urlEndpoint" => {"url" => url_match[0]}}} + current_nodes << (JSON.parse(current_node.to_json)) # If text remain after match create new simple node with text after match - afterNode = {"text" => splittedLastNode.size > 1 ? splittedLastNode[1] : ""} - currentNodes << (JSON.parse(afterNode.to_json)) + after_node = {"text" => splitted_last_node.size > 1 ? splitted_last_node[1] : ""} + current_nodes << (JSON.parse(after_node.to_json)) end # After processing of matches inside line # Add \n at end of last node for preserve carriage return - lastNode = currentNodes[currentNodes.size - 1].as_h - lastNode["text"] = JSON.parse("#{currentNodes[currentNodes.size - 1]["text"]}\n".to_json) - currentNodes[currentNodes.size - 1] = JSON.parse(lastNode.to_json) + last_node = current_nodes[-1].as_h + last_node["text"] = JSON.parse("#{last_node["text"]}\n".to_json) + current_nodes[-1] = JSON.parse(last_node.to_json) # Finally add final nodes to nodes returned - currentNodes.each do |node| + current_nodes.each do |node| nodes << (node) end end @@ -53,8 +53,8 @@ def content_to_comment_html(content, video_id : String? = "") text = HTML.escape(run["text"].as_s) - if navigationEndpoint = run.dig?("navigationEndpoint") - text = parse_link_endpoint(navigationEndpoint, text, video_id) + if navigation_endpoint = run.dig?("navigationEndpoint") + text = parse_link_endpoint(navigation_endpoint, text, video_id) end text = "<b>#{text}</b>" if run["bold"]? @@ -64,15 +64,15 @@ def content_to_comment_html(content, video_id : String? = "") # check for custom emojis if run["emoji"]? if run["emoji"]["isCustomEmoji"]?.try &.as_bool - if emojiImage = run.dig?("emoji", "image") - emojiAlt = emojiImage.dig?("accessibility", "accessibilityData", "label").try &.as_s || text - emojiThumb = emojiImage["thumbnails"][0] + if emoji_image = run.dig?("emoji", "image") + emoji_alt = emoji_image.dig?("accessibility", "accessibilityData", "label").try &.as_s || text + emoji_thumb = emoji_image["thumbnails"][0] text = String.build do |str| - str << %(<img alt=") << emojiAlt << "\" " - str << %(src="/ggpht) << URI.parse(emojiThumb["url"].as_s).request_target << "\" " - str << %(title=") << emojiAlt << "\" " - str << %(width=") << emojiThumb["width"] << "\" " - str << %(height=") << emojiThumb["height"] << "\" " + str << %(<img alt=") << emoji_alt << "\" " + str << %(src="/ggpht) << URI.parse(emoji_thumb["url"].as_s).request_target << "\" " + str << %(title=") << emoji_alt << "\" " + str << %(width=") << emoji_thumb["width"] << "\" " + str << %(height=") << emoji_thumb["height"] << "\" " str << %(class="channel-emoji" />) end else diff --git a/src/invidious/comments/youtube.cr b/src/invidious/comments/youtube.cr index 1ba1b534..0716fcde 100644 --- a/src/invidious/comments/youtube.cr +++ b/src/invidious/comments/youtube.cr @@ -13,6 +13,51 @@ module Invidious::Comments client_config = YoutubeAPI::ClientConfig.new(region: region) response = YoutubeAPI.next(continuation: ctoken, client_config: client_config) + return parse_youtube(id, response, format, locale, thin_mode, sort_by) + end + + def fetch_community_post_comments(ucid, post_id) + object = { + "2:string" => "community", + "25:embedded" => { + "22:string" => post_id, + }, + "45:embedded" => { + "2:varint" => 1_i64, + "3:varint" => 1_i64, + }, + "53:embedded" => { + "4:embedded" => { + "6:varint" => 0_i64, + "27:varint" => 1_i64, + "29:string" => post_id, + "30:string" => ucid, + }, + "8:string" => "comments-section", + }, + } + + object_parsed = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + + object2 = { + "80226972:embedded" => { + "2:string" => ucid, + "3:string" => object_parsed, + }, + } + + continuation = object2.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + initial_data = YoutubeAPI.browse(continuation: continuation) + return initial_data + end + + def parse_youtube(id, response, format, locale, thin_mode, sort_by = "top", is_post = false) contents = nil if on_response_received_endpoints = response["onResponseReceivedEndpoints"]? @@ -59,6 +104,8 @@ module Invidious::Comments end end + mutations = response.dig?("frameworkUpdates", "entityBatchUpdate", "mutations").try &.as_a || [] of JSON::Any + response = JSON.build do |json| json.object do if header @@ -68,7 +115,11 @@ module Invidious::Comments json.field "commentCount", comment_count end - json.field "videoId", id + if is_post + json.field "postId", id + else + json.field "videoId", id + end json.field "comments" do json.array do @@ -82,73 +133,138 @@ module Invidious::Comments node_replies = node["replies"]["commentRepliesRenderer"] end - if node["comment"]? - node_comment = node["comment"]["commentRenderer"] - else - node_comment = node["commentRenderer"] - end + if cvm = node["commentViewModel"]? + # two commentViewModels for inital request + # one commentViewModel when getting a replies to a comment + cvm = cvm["commentViewModel"] if cvm["commentViewModel"]? + + comment_key = cvm["commentKey"] + toolbar_key = cvm["toolbarStateKey"] + comment_mutation = mutations.find { |i| i.dig?("payload", "commentEntityPayload", "key") == comment_key } + toolbar_mutation = mutations.find { |i| i.dig?("entityKey") == toolbar_key } + + if !comment_mutation.nil? && !toolbar_mutation.nil? + # todo parse styleRuns, commandRuns and attachmentRuns for comments + html_content = parse_description(comment_mutation.dig("payload", "commentEntityPayload", "properties", "content"), id) + comment_author = comment_mutation.dig("payload", "commentEntityPayload", "author") + json.field "authorId", comment_author["channelId"].as_s + json.field "authorUrl", "/channel/#{comment_author["channelId"].as_s}" + json.field "author", comment_author["displayName"].as_s + json.field "verified", comment_author["isVerified"].as_bool + json.field "authorThumbnails" do + json.array do + comment_mutation.dig?("payload", "commentEntityPayload", "avatar", "image", "sources").try &.as_a.each do |thumbnail| + json.object do + json.field "url", thumbnail["url"] + json.field "width", thumbnail["width"] + json.field "height", thumbnail["height"] + end + end + end + end - content_html = node_comment["contentText"]?.try { |t| parse_content(t, id) } || "" - author = node_comment["authorText"]?.try &.["simpleText"]? || "" + json.field "authorIsChannelOwner", comment_author["isCreator"].as_bool + json.field "isSponsor", (comment_author["sponsorBadgeUrl"]? != nil) - json.field "verified", (node_comment["authorCommentBadge"]? != nil) + if sponsor_badge_url = comment_author["sponsorBadgeUrl"]? + # Sponsor icon thumbnails always have one object and there's only ever the url property in it + json.field "sponsorIconUrl", sponsor_badge_url + end - json.field "author", author - json.field "authorThumbnails" do - json.array do - node_comment["authorThumbnail"]["thumbnails"].as_a.each do |thumbnail| - json.object do - json.field "url", thumbnail["url"] - json.field "width", thumbnail["width"] - json.field "height", thumbnail["height"] + comment_toolbar = comment_mutation.dig("payload", "commentEntityPayload", "toolbar") + json.field "likeCount", short_text_to_number(comment_toolbar["likeCountNotliked"].as_s) + reply_count = short_text_to_number(comment_toolbar["replyCount"]?.try &.as_s || "0") + + if heart_state = toolbar_mutation.dig?("payload", "engagementToolbarStateEntityPayload", "heartState") + if heart_state.as_s == "TOOLBAR_HEART_STATE_HEARTED" + json.field "creatorHeart" do + json.object do + json.field "creatorThumbnail", comment_toolbar["creatorThumbnailUrl"].as_s + json.field "creatorName", comment_toolbar["heartActiveTooltip"].as_s.sub("❤ by ", "") + end + end end end + + published_text = comment_mutation.dig?("payload", "commentEntityPayload", "properties", "publishedTime").try &.as_s end - end - if node_comment["authorEndpoint"]? - json.field "authorId", node_comment["authorEndpoint"]["browseEndpoint"]["browseId"] - json.field "authorUrl", node_comment["authorEndpoint"]["browseEndpoint"]["canonicalBaseUrl"] + json.field "isPinned", (cvm.dig?("pinnedText") != nil) + json.field "commentId", cvm["commentId"] else - json.field "authorId", "" - json.field "authorUrl", "" - end + if node["comment"]? + node_comment = node["comment"]["commentRenderer"] + else + node_comment = node["commentRenderer"] + end + json.field "commentId", node_comment["commentId"] + html_content = node_comment["contentText"]?.try { |t| parse_content(t, id) } + + json.field "verified", (node_comment["authorCommentBadge"]? != nil) + + json.field "author", node_comment["authorText"]?.try &.["simpleText"]? || "" + json.field "authorThumbnails" do + json.array do + node_comment["authorThumbnail"]["thumbnails"].as_a.each do |thumbnail| + json.object do + json.field "url", thumbnail["url"] + json.field "width", thumbnail["width"] + json.field "height", thumbnail["height"] + end + end + end + end - published_text = node_comment["publishedTimeText"]["runs"][0]["text"].as_s - published = decode_date(published_text.rchop(" (edited)")) + if comment_action_buttons_renderer = node_comment.dig?("actionButtons", "commentActionButtonsRenderer") + json.field "likeCount", comment_action_buttons_renderer["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"].as_s.scan(/\d/).map(&.[0]).join.to_i + if comment_action_buttons_renderer["creatorHeart"]? + heart_data = comment_action_buttons_renderer["creatorHeart"]["creatorHeartRenderer"]["creatorThumbnail"] + json.field "creatorHeart" do + json.object do + json.field "creatorThumbnail", heart_data["thumbnails"][-1]["url"] + json.field "creatorName", heart_data["accessibility"]["accessibilityData"]["label"] + end + end + end + end - if published_text.includes?(" (edited)") - json.field "isEdited", true - else - json.field "isEdited", false - end + if node_comment["authorEndpoint"]? + json.field "authorId", node_comment["authorEndpoint"]["browseEndpoint"]["browseId"] + json.field "authorUrl", node_comment["authorEndpoint"]["browseEndpoint"]["canonicalBaseUrl"] + else + json.field "authorId", "" + json.field "authorUrl", "" + end - json.field "content", html_to_content(content_html) - json.field "contentHtml", content_html + json.field "authorIsChannelOwner", node_comment["authorIsChannelOwner"] + json.field "isPinned", (node_comment["pinnedCommentBadge"]? != nil) + published_text = node_comment["publishedTimeText"]["runs"][0]["text"].as_s - json.field "isPinned", (node_comment["pinnedCommentBadge"]? != nil) - json.field "isSponsor", (node_comment["sponsorCommentBadge"]? != nil) - if node_comment["sponsorCommentBadge"]? - # Sponsor icon thumbnails always have one object and there's only ever the url property in it - json.field "sponsorIconUrl", node_comment.dig("sponsorCommentBadge", "sponsorCommentBadgeRenderer", "customBadge", "thumbnails", 0, "url").to_s - end - json.field "published", published.to_unix - json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale)) + json.field "isSponsor", (node_comment["sponsorCommentBadge"]? != nil) + if node_comment["sponsorCommentBadge"]? + # Sponsor icon thumbnails always have one object and there's only ever the url property in it + json.field "sponsorIconUrl", node_comment.dig("sponsorCommentBadge", "sponsorCommentBadgeRenderer", "customBadge", "thumbnails", 0, "url").to_s + end - comment_action_buttons_renderer = node_comment["actionButtons"]["commentActionButtonsRenderer"] + reply_count = node_comment["replyCount"]? + end - json.field "likeCount", comment_action_buttons_renderer["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"].as_s.scan(/\d/).map(&.[0]).join.to_i - json.field "commentId", node_comment["commentId"] - json.field "authorIsChannelOwner", node_comment["authorIsChannelOwner"] + content_html = html_content || "" + json.field "content", html_to_content(content_html) + json.field "contentHtml", content_html - if comment_action_buttons_renderer["creatorHeart"]? - hearth_data = comment_action_buttons_renderer["creatorHeart"]["creatorHeartRenderer"]["creatorThumbnail"] - json.field "creatorHeart" do - json.object do - json.field "creatorThumbnail", hearth_data["thumbnails"][-1]["url"] - json.field "creatorName", hearth_data["accessibility"]["accessibilityData"]["label"] - end + if published_text != nil + published_text = published_text.to_s + if published_text.includes?(" (edited)") + json.field "isEdited", true + published = decode_date(published_text.rchop(" (edited)")) + else + json.field "isEdited", false + published = decode_date(published_text) end + + json.field "published", published.to_unix + json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale)) end if node_replies && !response["commentRepliesContinuation"]? @@ -161,7 +277,7 @@ module Invidious::Comments json.field "replies" do json.object do - json.field "replyCount", node_comment["replyCount"]? || 1 + json.field "replyCount", reply_count || 1 json.field "continuation", continuation end end @@ -187,7 +303,6 @@ module Invidious::Comments if format == "html" response = JSON.parse(response) content_html = Frontend::Comments.template_youtube(response, locale, thin_mode) - response = JSON.build do |json| json.object do json.field "contentHtml", content_html diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 429d9246..4b3bdafc 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -13,6 +13,7 @@ struct ConfigPreferences property annotations : Bool = false property annotations_subscribed : Bool = false + property preload : Bool = true property autoplay : Bool = false property captions : Array(String) = ["", "", ""] property comments : Array(String) = ["youtube", ""] @@ -48,12 +49,21 @@ struct ConfigPreferences def to_tuple {% begin %} { - {{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}} + {{(@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }).splat}} } {% end %} end end +struct HTTPProxyConfig + include YAML::Serializable + + property user : String + property password : String + property host : String + property port : Int32 +end + class Config include YAML::Serializable @@ -68,14 +78,14 @@ class Config property output : String = "STDOUT" # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr property log_level : LogLevel = LogLevel::Info + # Enables colors in logs. Useful for debugging purposes + property colorize_logs : Bool = false # Database configuration with separate parameters (username, hostname, etc) property db : DBConfig? = nil # Database configuration using 12-Factor "Database URL" syntax @[YAML::Field(converter: Preferences::URIConverter)] property database_url : URI = URI.parse("") - # Use polling to keep decryption function up to date - property decrypt_polling : Bool = false # Used for crawling channels: threads should check all videos uploaded by a channel property full_refresh : Bool = false @@ -120,23 +130,30 @@ class Config # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729) @[YAML::Field(converter: Preferences::FamilyConverter)] property force_resolve : Socket::Family = Socket::Family::UNSPEC + + # External signature solver server socket (either a path to a UNIX domain socket or "<IP>:<Port>") + property signature_server : String? = nil + # Port to listen for connections (overridden by command line argument) property port : Int32 = 3000 # Host to bind (overridden by command line argument) property host_binding : String = "0.0.0.0" # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) property pool_size : Int32 = 100 + # HTTP Proxy configuration + property http_proxy : HTTPProxyConfig? = nil # Use Innertube's transcripts API instead of timedtext for closed captions property use_innertube_for_captions : Bool = false + # visitor data ID for Google session + property visitor_data : String? = nil + # poToken for passing bot attestation + property po_token : String? = nil + # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] property cookies : HTTP::Cookies = HTTP::Cookies.new - # Key for Anti-Captcha - property captcha_key : String? = nil - # API URL for Anti-Captcha - property captcha_api_url : String = "https://api.anti-captcha.com" # Playlist length limit property playlist_length_limit : Int32 = 500 @@ -167,6 +184,9 @@ class Config config = Config.from_yaml(config_yaml) # Update config from env vars (upcased and prefixed with "INVIDIOUS_") + # + # Also checks if any top-level config options are set to "CHANGE_ME!!" + # TODO: Support non-top-level config options such as the ones in DBConfig {% for ivar in Config.instance_vars %} {% env_id = "INVIDIOUS_#{ivar.id.upcase}" %} @@ -203,6 +223,12 @@ class Config exit(1) end end + + # Warn when any config attribute is set to "CHANGE_ME!!" + if config.{{ivar.id}} == "CHANGE_ME!!" + puts "Config: The value of '#{ {{ivar.stringify}} }' needs to be changed!!" + exit(1) + end {% end %} # HMAC_key is mandatory @@ -210,9 +236,6 @@ class Config if config.hmac_key.empty? puts "Config: 'hmac_key' is required/can't be empty" exit(1) - elsif config.hmac_key == "CHANGE_ME!!" - puts "Config: The value of 'hmac_key' needs to be changed!!" - exit(1) end # Build database_url from db.* if it's not set directly diff --git a/src/invidious/database/playlists.cr b/src/invidious/database/playlists.cr index c6754a1e..08aa719a 100644 --- a/src/invidious/database/playlists.cr +++ b/src/invidious/database/playlists.cr @@ -140,6 +140,7 @@ module Invidious::Database::Playlists request = <<-SQL SELECT id,title FROM playlists WHERE author = $1 AND id LIKE 'IV%' + ORDER BY title SQL PG_DB.query_all(request, email, as: {String, String}) diff --git a/src/invidious/database/statistics.cr b/src/invidious/database/statistics.cr index 1df549e2..9e4963fd 100644 --- a/src/invidious/database/statistics.cr +++ b/src/invidious/database/statistics.cr @@ -15,7 +15,7 @@ module Invidious::Database::Statistics PG_DB.query_one(request, as: Int64) end - def count_users_active_1m : Int64 + def count_users_active_6m : Int64 request = <<-SQL SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '6 months' @@ -24,7 +24,7 @@ module Invidious::Database::Statistics PG_DB.query_one(request, as: Int64) end - def count_users_active_6m : Int64 + def count_users_active_1m : Int64 request = <<-SQL SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '1 month' diff --git a/src/invidious/frontend/comments_reddit.cr b/src/invidious/frontend/comments_reddit.cr index b5647bae..4dda683e 100644 --- a/src/invidious/frontend/comments_reddit.cr +++ b/src/invidious/frontend/comments_reddit.cr @@ -33,7 +33,7 @@ module Invidious::Frontend::Comments <a href="javascript:void(0)" data-onclick="toggle_parent">[ − ]</a> <b><a href="https://www.reddit.com/user/#{child.author}">#{child.author}</a></b> #{translate_count(locale, "comments_points_count", child.score, NumberFormatting::Separator)} - <span title="#{child.created_utc.to_s(translate(locale, "%a %B %-d %T %Y UTC"))}">#{translate(locale, "`x` ago", recode_date(child.created_utc, locale))}</span> + <span title="#{child.created_utc.to_s("%a %B %-d %T %Y UTC")}">#{translate(locale, "`x` ago", recode_date(child.created_utc, locale))}</span> <a href="https://www.reddit.com#{child.permalink}" title="#{translate(locale, "permalink")}">#{translate(locale, "permalink")}</a> </p> <div> diff --git a/src/invidious/frontend/comments_youtube.cr b/src/invidious/frontend/comments_youtube.cr index 41f43f04..a0e1d783 100644 --- a/src/invidious/frontend/comments_youtube.cr +++ b/src/invidious/frontend/comments_youtube.cr @@ -23,6 +23,24 @@ module Invidious::Frontend::Comments </div> </div> END_HTML + elsif comments["authorId"]? && !comments["singlePost"]? + # for posts we should display a link to the post + replies_count_text = translate_count(locale, + "comments_view_x_replies", + child["replyCount"].as_i64 || 0, + NumberFormatting::Separator + ) + + replies_html = <<-END_HTML + <div class="pure-g"> + <div class="pure-u-1-24"></div> + <div class="pure-u-23-24"> + <p> + <a href="/post/#{child["commentId"]}?ucid=#{comments["authorId"]}">#{replies_count_text}</a> + </p> + </div> + </div> + END_HTML end if !thin_mode @@ -89,6 +107,36 @@ module Invidious::Frontend::Comments </div> END_HTML end + when "multiImage" + html << <<-END_HTML + <section class="carousel"> + <a class="skip-link" href="#skip-#{child["commentId"]}">#{translate(locale, "carousel_skip")}</a> + <div class="slides"> + END_HTML + image_array = attachment["images"].as_a + + image_array.each_index do |i| + html << <<-END_HTML + <div class="slides-item slide-#{i + 1}" id="#{child["commentId"]}-slide-#{i + 1}" aria-label="#{translate(locale, "carousel_slide", {"current" => (i + 1).to_s, "total" => image_array.size.to_s})}" tabindex="0"> + <img loading="lazy" src="/ggpht#{URI.parse(image_array[i][1]["url"].as_s).request_target}" alt="" /> + </div> + END_HTML + end + + html << <<-END_HTML + </div> + <div class="carousel__nav"> + END_HTML + attachment["images"].as_a.each_index do |i| + html << <<-END_HTML + <a class="slider-nav" href="##{child["commentId"]}-slide-#{i + 1}" aria-label="#{translate(locale, "carousel_go_to", (i + 1).to_s)}" tabindex="-1" aria-hidden="true">#{i + 1}</a> + END_HTML + end + html << <<-END_HTML + </div> + <div id="skip-#{child["commentId"]}"></div> + </section> + END_HTML else nil # Ignore end end @@ -101,12 +149,12 @@ module Invidious::Frontend::Comments if comments["videoId"]? html << <<-END_HTML - <a href="https://www.youtube.com/watch?v=#{comments["videoId"]}&lc=#{child["commentId"]}" title="#{translate(locale, "YouTube comment permalink")}">[YT]</a> + <a rel="noreferrer noopener" href="https://www.youtube.com/watch?v=#{comments["videoId"]}&lc=#{child["commentId"]}" title="#{translate(locale, "YouTube comment permalink")}">[YT]</a> | END_HTML elsif comments["authorId"]? html << <<-END_HTML - <a href="https://www.youtube.com/channel/#{comments["authorId"]}/community?lb=#{child["commentId"]}" title="#{translate(locale, "YouTube comment permalink")}">[YT]</a> + <a rel="noreferrer noopener" href="https://www.youtube.com/channel/#{comments["authorId"]}/community?lb=#{child["commentId"]}" title="#{translate(locale, "YouTube comment permalink")}">[YT]</a> | END_HTML end diff --git a/src/invidious/frontend/misc.cr b/src/invidious/frontend/misc.cr index 43ba9f5c..7a6cf79d 100644 --- a/src/invidious/frontend/misc.cr +++ b/src/invidious/frontend/misc.cr @@ -6,9 +6,9 @@ module Invidious::Frontend::Misc if prefs.automatic_instance_redirect current_page = env.get?("current_page").as(String) - redirect_url = "/redirect?referer=#{current_page}" + return "/redirect?referer=#{current_page}" else - redirect_url = "https://redirect.invidious.io#{env.request.resource}" + return "https://redirect.invidious.io#{env.request.resource}" end end end diff --git a/src/invidious/frontend/watch_page.cr b/src/invidious/frontend/watch_page.cr index 5fd81168..2e2f6ad0 100644 --- a/src/invidious/frontend/watch_page.cr +++ b/src/invidious/frontend/watch_page.cr @@ -13,7 +13,7 @@ module Invidious::Frontend::WatchPage @full_videos, @video_streams, @audio_streams, - @captions + @captions, ) end end @@ -42,8 +42,7 @@ module Invidious::Frontend::WatchPage str << translate(locale, "Download as: ") str << "</label>\n" - # TODO: remove inline style - str << "\t\t<select style=\"width:100%\" name='download_widget' id='download_widget'>\n" + str << "\t\t<select name='download_widget' id='download_widget'>\n" # Non-DASH videos (audio+video) diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr index bf56d826..fec3f62c 100644 --- a/src/invidious/helpers/crystal_class_overrides.cr +++ b/src/invidious/helpers/crystal_class_overrides.cr @@ -3,9 +3,9 @@ # IPv6 addresses. # class TCPSocket - def initialize(host : String, port, dns_timeout = nil, connect_timeout = nil, family = Socket::Family::UNSPEC) + def initialize(host, port, dns_timeout = nil, connect_timeout = nil, blocking = false, family = Socket::Family::UNSPEC) Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo| - super(addrinfo.family, addrinfo.type, addrinfo.protocol) + super(addrinfo.family, addrinfo.type, addrinfo.protocol, blocking) connect(addrinfo, timeout: connect_timeout) do |error| close error @@ -26,7 +26,7 @@ class HTTP::Client end hostname = @host.starts_with?('[') && @host.ends_with?(']') ? @host[1..-2] : @host - io = TCPSocket.new hostname, @port, @dns_timeout, @connect_timeout, @family + io = TCPSocket.new hostname, @port, @dns_timeout, @connect_timeout, family: @family io.read_timeout = @read_timeout if @read_timeout io.write_timeout = @write_timeout if @write_timeout io.sync = false @@ -35,7 +35,7 @@ class HTTP::Client if tls = @tls tcp_socket = io begin - io = OpenSSL::SSL::Socket::Client.new(tcp_socket, context: tls, sync_close: true, hostname: @host) + io = OpenSSL::SSL::Socket::Client.new(tcp_socket, context: tls, sync_close: true, hostname: @host.rchop('.')) rescue exc # don't leak the TCP socket when the SSL connection failed tcp_socket.close diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr index 6e5a975d..900cb0c6 100644 --- a/src/invidious/helpers/errors.cr +++ b/src/invidious/helpers/errors.cr @@ -3,7 +3,7 @@ # ------------------- macro error_template(*args) - error_template_helper(env, {{*args}}) + error_template_helper(env, {{args.splat}}) end def github_details(summary : String, content : String) @@ -43,6 +43,8 @@ def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exce # URLs for the error message below url_faq = "https://github.com/iv-org/documentation/blob/master/docs/faq.md" url_search_issues = "https://github.com/iv-org/invidious/issues" + url_search_issues += "?q=is:issue+is:open+" + url_search_issues += URI.encode_www_form("[Bug] #{issue_title}") url_switch = "https://redirect.invidious.io" + env.request.resource @@ -95,7 +97,7 @@ end # ------------------- macro error_atom(*args) - error_atom_helper(env, {{*args}}) + error_atom_helper(env, {{args.splat}}) end def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception) @@ -121,14 +123,14 @@ end # ------------------- macro error_json(*args) - error_json_helper(env, {{*args}}) + error_json_helper(env, {{args.splat}}) end def error_json_helper( env : HTTP::Server::Context, status_code : Int32, exception : Exception, - additional_fields : Hash(String, Object) | Nil = nil + additional_fields : Hash(String, Object) | Nil = nil, ) if exception.is_a?(InfoException) return error_json_helper(env, status_code, exception.message || "", additional_fields) @@ -150,7 +152,7 @@ def error_json_helper( env : HTTP::Server::Context, status_code : Int32, message : String, - additional_fields : Hash(String, Object) | Nil = nil + additional_fields : Hash(String, Object) | Nil = nil, ) env.response.content_type = "application/json" env.response.status_code = status_code @@ -190,7 +192,7 @@ def error_redirect_helper(env : HTTP::Server::Context) <a href="/redirect?referer=#{env.get("current_page")}">#{switch_instance}</a> </li> <li> - <a href="https://youtube.com#{env.request.resource}">#{go_to_youtube}</a> + <a rel="noreferrer noopener" href="https://youtube.com#{env.request.resource}">#{go_to_youtube}</a> </li> </ul> END_HTML diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index d140a858..13ea9fe9 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -27,6 +27,7 @@ class Kemal::RouteHandler # Processes the route if it's a match. Otherwise renders 404. private def process_request(context) raise Kemal::Exceptions::RouteNotFound.new(context) unless context.route_found? + return if context.response.closed? content = context.route.handler.call(context) if !Kemal.config.error_handlers.empty? && Kemal.config.error_handlers.has_key?(context.response.status_code) && exclude_match?(context) @@ -97,7 +98,7 @@ class AuthHandler < Kemal::Handler if token = env.request.headers["Authorization"]? token = JSON.parse(URI.decode_www_form(token.lchop("Bearer "))) session = URI.decode_www_form(token["session"].as_s) - scopes, expire, signature = validate_request(token, session, env.request, HMAC_KEY, nil) + scopes, _, _ = validate_request(token, session, env.request, HMAC_KEY, nil) if email = Invidious::Database::SessionIDs.select_email(session) user = Invidious::Database::Users.select!(email: email) @@ -142,63 +143,8 @@ class APIHandler < Kemal::Handler exclude ["/api/v1/auth/notifications"], "POST" def call(env) - return call_next env unless only_match? env - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - # Since /api/v1/notifications is an event-stream, we don't want - # to wrap the response - return call_next env if exclude_match? env - - # Here we swap out the socket IO so we can modify the response as needed - output = env.response.output - env.response.output = IO::Memory.new - - begin - call_next env - - env.response.output.rewind - - if env.response.output.as(IO::Memory).size != 0 && - env.response.headers.includes_word?("Content-Type", "application/json") - response = JSON.parse(env.response.output) - - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - - if env.params.query["pretty"]?.try &.== "1" - response = response.to_pretty_json - else - response = response.to_json - end - else - response = env.response.output.gets_to_end - end - rescue ex - env.response.content_type = "application/json" if env.response.headers.includes_word?("Content-Type", "text/html") - env.response.status_code = 500 - - if env.response.headers.includes_word?("Content-Type", "application/json") - response = {"error" => ex.message || "Unspecified error"} - - if env.params.query["pretty"]?.try &.== "1" - response = response.to_pretty_json - else - response = response.to_json - end - end - ensure - env.response.output = output - env.response.print response - - env.response.flush - end + env.response.headers["Access-Control-Allow-Origin"] = "*" if only_match?(env) + call_next env end end diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 23ff0da9..6add0237 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -78,15 +78,6 @@ def create_notification_stream(env, topics, connection_channel) video.published = published response = JSON.parse(video.to_json(locale, nil)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -113,15 +104,6 @@ def create_notification_stream(env, topics, connection_channel) Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video| response = JSON.parse(video.to_json(locale)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -155,15 +137,6 @@ def create_notification_stream(env, topics, connection_channel) video.published = Time.unix(published) response = JSON.parse(video.to_json(locale, nil)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -208,3 +181,20 @@ def proxy_file(response, env) IO.copy response.body_io, env.response end end + +# Fetch the playback requests tracker from the statistics endpoint. +# +# Creates a new tracker when unavailable. +def get_playback_statistic + if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]) && tracker.as(Hash).empty? + tracker = { + "totalRequests" => 0_i64, + "successfulRequests" => 0_i64, + "ratio" => 0_f64, + } + + Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"] = tracker + end + + return tracker.as(Hash(String, Int64 | Float64)) +end diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index 76e477a4..1ba3ea61 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -1,8 +1,22 @@ +# Languages requiring a better level of translation (at least 20%) +# to be added to the list below: +# +# "af" => "", # Afrikaans +# "az" => "", # Azerbaijani +# "be" => "", # Belarusian +# "bn_BD" => "", # Bengali (Bangladesh) +# "ia" => "", # Interlingua +# "or" => "", # Odia +# "tk" => "", # Turkmen +# "tok => "", # Toki Pona +# LOCALES_LIST = { "ar" => "العربية", # Arabic + "bg" => "български", # Bulgarian "bn" => "বাংলা", # Bengali "ca" => "Català", # Catalan "cs" => "Čeština", # Czech + "cy" => "Cymraeg", # Welsh "da" => "Dansk", # Danish "de" => "Deutsch", # German "el" => "Ελληνικά", # Greek @@ -23,6 +37,7 @@ LOCALES_LIST = { "it" => "Italiano", # Italian "ja" => "日本語", # Japanese "ko" => "한국어", # Korean + "lmo" => "Lombard", # Lombard "lt" => "Lietuvių", # Lithuanian "nb-NO" => "Norsk bokmål", # Norwegian Bokmål "nl" => "Nederlands", # Dutch @@ -78,7 +93,7 @@ def load_all_locales return locales end -def translate(locale : String?, key : String, text : String | Nil = nil) : String +def translate(locale : String?, key : String, text : String | Hash(String, String) | Nil = nil) : String # Log a warning if "key" doesn't exist in en-US locale and return # that key as the text, so this is more or less transparent to the user. if !LOCALES["en-US"].has_key?(key) @@ -101,10 +116,12 @@ def translate(locale : String?, key : String, text : String | Nil = nil) : Strin match_length = 0 raw_data.as_h.each do |hash_key, value| - if md = text.try &.match(/#{hash_key}/) - if md[0].size >= match_length - translation = value.as_s - match_length = md[0].size + if text.is_a?(String) + if md = text.try &.match(/#{hash_key}/) + if md[0].size >= match_length + translation = value.as_s + match_length = md[0].size + end end end end @@ -114,8 +131,13 @@ def translate(locale : String?, key : String, text : String | Nil = nil) : Strin raise "Invalid translation \"#{raw_data}\"" end - if text + if text.is_a?(String) translation = translation.gsub("`x`", text) + elsif text.is_a?(Hash(String, String)) + # adds support for multi string interpolation. Based on i18next https://www.i18next.com/translation-function/interpolation#basic + text.each_key do |hash_key| + translation = translation.gsub("{{#{hash_key}}}", text[hash_key]) + end end return translation diff --git a/src/invidious/helpers/i18next.cr b/src/invidious/helpers/i18next.cr index e84f88fb..684e6d14 100644 --- a/src/invidious/helpers/i18next.cr +++ b/src/invidious/helpers/i18next.cr @@ -35,27 +35,35 @@ module I18next::Plurals Special_Slovenian = 21 Special_Hebrew = 22 Special_Odia = 23 + + # Mixed v3/v4 rules in Weblate + # `es`, `pt` and `pt-PT` doesn't seem to have been refreshed + # by weblate yet, but I suspect it will happen one day. + # See: https://github.com/translate/translate/issues/4873 + Special_French_Portuguese + Special_Hungarian_Serbian + Special_Spanish_Italian end private PLURAL_SETS = { PluralForms::Single_gt_one => [ - "ach", "ak", "am", "arn", "br", "fil", "fr", "gun", "ln", "mfe", "mg", - "mi", "oc", "pt", "pt-BR", "tg", "tl", "ti", "tr", "uz", "wa", + "ach", "ak", "am", "arn", "br", "fa", "fil", "gun", "ln", "mfe", "mg", + "mi", "oc", "pt-PT", "tg", "tl", "ti", "tr", "uz", "wa", ], PluralForms::Single_not_one => [ "af", "an", "ast", "az", "bg", "bn", "ca", "da", "de", "dev", "el", "en", - "eo", "es", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi", - "hu", "hy", "ia", "it", "kk", "kn", "ku", "lb", "mai", "ml", "mn", "mr", + "eo", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi", + "hu", "hy", "ia", "kk", "kn", "ku", "lb", "mai", "ml", "mn", "mr", "nah", "nap", "nb", "ne", "nl", "nn", "no", "nso", "pa", "pap", "pms", - "ps", "pt-PT", "rm", "sco", "se", "si", "so", "son", "sq", "sv", "sw", + "ps", "rm", "sco", "se", "si", "so", "son", "sq", "sv", "sw", "ta", "te", "tk", "ur", "yo", ], PluralForms::None => [ - "ay", "bo", "cgg", "fa", "ht", "id", "ja", "jbo", "ka", "km", "ko", "ky", + "ay", "bo", "cgg", "ht", "id", "ja", "jbo", "ka", "km", "ko", "ky", "lo", "ms", "sah", "su", "th", "tt", "ug", "vi", "wo", "zh", ], PluralForms::Dual_Slavic => [ - "be", "bs", "cnr", "dz", "hr", "ru", "sr", "uk", + "be", "bs", "cnr", "dz", "ru", "uk", ], } @@ -81,6 +89,13 @@ module I18next::Plurals "ro" => PluralForms::Special_Romanian, "sk" => PluralForms::Special_Czech_Slovak, "sl" => PluralForms::Special_Slovenian, + # Mixed v3/v4 rules + "es" => PluralForms::Special_Spanish_Italian, + "fr" => PluralForms::Special_French_Portuguese, + "hr" => PluralForms::Special_Hungarian_Serbian, + "it" => PluralForms::Special_Spanish_Italian, + "pt" => PluralForms::Special_French_Portuguese, + "sr" => PluralForms::Special_Hungarian_Serbian, } # These are the v1 and v2 compatible suffixes. @@ -150,9 +165,8 @@ module I18next::Plurals end def get_plural_form(locale : String) : PluralForms - # Extract the ISO 639-1 or 639-2 code from an RFC 5646 language code, - # except for pt-BR and pt-PT which needs to be kept as-is. - if !locale.matches?(/^pt-(BR|PT)$/) + # Extract the ISO 639-1 or 639-2 code from an RFC 5646 language code + if !locale.matches?(/^pt-PT$/) locale = locale.split('-')[0] end @@ -174,7 +188,7 @@ module I18next::Plurals # Emulate the `rule.numbers.size == 2 && rule.numbers[0] == 1` check # from original i18next code - private def is_simple_plural(form : PluralForms) : Bool + private def simple_plural?(form : PluralForms) : Bool case form when .single_gt_one? then return true when .single_not_one? then return true @@ -196,7 +210,7 @@ module I18next::Plurals idx = SuffixIndex.get_index(plural_form, count) # Simple plurals are handled differently in all versions (but v4) - if @simplify_plural_suffix && is_simple_plural(plural_form) + if @simplify_plural_suffix && simple_plural?(plural_form) return (idx == 1) ? "_plural" : "" end @@ -246,6 +260,10 @@ module I18next::Plurals when .special_slovenian? then return special_slovenian(count) when .special_hebrew? then return special_hebrew(count) when .special_odia? then return special_odia(count) + # Mixed v3/v4 forms + when .special_spanish_italian? then return special_cldr_spanish_italian(count) + when .special_french_portuguese? then return special_cldr_french_portuguese(count) + when .special_hungarian_serbian? then return special_cldr_hungarian_serbian(count) else # default, if nothing matched above return 0_u8 @@ -507,5 +525,42 @@ module I18next::Plurals def self.special_odia(count : Int) : UInt8 return (count == 1) ? 0_u8 : 1_u8 end + + # ------------------- + # "v3.5" rules + # ------------------- + + # Plural form for Spanish & Italian languages + # + # This rule is mostly compliant to CLDR v42 + # + def self.special_cldr_spanish_italian(count : Int) : UInt8 + return 0_u8 if (count == 1) # one + return 1_u8 if (count != 0 && count % 1_000_000 == 0) # many + return 2_u8 # other + end + + # Plural form for French and Portuguese + # + # This rule is mostly compliant to CLDR v42 + # + def self.special_cldr_french_portuguese(count : Int) : UInt8 + return 0_u8 if (count == 0 || count == 1) # one + return 1_u8 if (count % 1_000_000 == 0) # many + return 2_u8 # other + end + + # Plural form for Hungarian and Serbian + # + # This rule is mostly compliant to CLDR v42 + # + def self.special_cldr_hungarian_serbian(count : Int) : UInt8 + n_mod_10 = count % 10 + n_mod_100 = count % 100 + + return 0_u8 if (n_mod_10 == 1 && n_mod_100 != 11) # one + return 1_u8 if (2 <= n_mod_10 <= 4 && (n_mod_100 < 12 || 14 < n_mod_100)) # few + return 2_u8 # other + end end end diff --git a/src/invidious/helpers/json_filter.cr b/src/invidious/helpers/json_filter.cr deleted file mode 100644 index 3f4080ba..00000000 --- a/src/invidious/helpers/json_filter.cr +++ /dev/null @@ -1,248 +0,0 @@ -module JSONFilter - alias BracketIndex = Hash(Int64, Int64) - - alias GroupedFieldsValue = String | Array(GroupedFieldsValue) - alias GroupedFieldsList = Array(GroupedFieldsValue) - - class FieldsParser - class ParseError < Exception - end - - # Returns the `Regex` pattern used to match nest groups - def self.nest_group_pattern : Regex - # uses a '.' character to match json keys as they are allowed - # to contain any unicode codepoint - /(?:|,)(?<groupname>[^,\n]*?)\(/ - end - - # Returns the `Regex` pattern used to check if there are any empty nest groups - def self.unnamed_nest_group_pattern : Regex - /^\(|\(\(|\/\(/ - end - - def self.parse_fields(fields_text : String, &) : Nil - if fields_text.empty? - raise FieldsParser::ParseError.new "Fields is empty" - end - - opening_bracket_count = fields_text.count('(') - closing_bracket_count = fields_text.count(')') - - if opening_bracket_count != closing_bracket_count - bracket_type = opening_bracket_count > closing_bracket_count ? "opening" : "closing" - raise FieldsParser::ParseError.new "There are too many #{bracket_type} brackets (#{opening_bracket_count}:#{closing_bracket_count})" - elsif match_result = unnamed_nest_group_pattern.match(fields_text) - raise FieldsParser::ParseError.new "Unnamed nest group at position #{match_result.begin}" - end - - # first, handle top-level single nested properties: items/id, playlistItems/snippet, etc - parse_single_nests(fields_text) { |nest_list| yield nest_list } - - # next, handle nest groups: items(id, etag, etc) - parse_nest_groups(fields_text) { |nest_list| yield nest_list } - end - - def self.parse_single_nests(fields_text : String, &) : Nil - single_nests = remove_nest_groups(fields_text) - - if !single_nests.empty? - property_nests = single_nests.split(',') - - property_nests.each do |nest| - nest_list = nest.split('/') - if nest_list.includes? "" - raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list}" - end - yield nest_list - end - # else - # raise FieldsParser::ParseError.new "Empty key in nest list 22: #{fields_text} | #{single_nests}" - end - end - - def self.parse_nest_groups(fields_text : String, &) : Nil - nest_stack = [] of NamedTuple(group_name: String, closing_bracket_index: Int64) - bracket_pairs = get_bracket_pairs(fields_text, true) - - text_index = 0 - regex_index = 0 - - while regex_result = self.nest_group_pattern.match(fields_text, regex_index) - raw_match = regex_result[0] - group_name = regex_result["groupname"] - - text_index = regex_result.begin - regex_index = regex_result.end - - if text_index.nil? || regex_index.nil? - raise FieldsParser::ParseError.new "Received invalid index while parsing nest groups: text_index: #{text_index} | regex_index: #{regex_index}" - end - - offset = raw_match.starts_with?(',') ? 1 : 0 - - opening_bracket_index = (text_index + group_name.size) + offset - closing_bracket_index = bracket_pairs[opening_bracket_index] - content_start = opening_bracket_index + 1 - - content = fields_text[content_start...closing_bracket_index] - - if content.empty? - raise FieldsParser::ParseError.new "Empty nest group at position #{content_start}" - else - content = remove_nest_groups(content) - end - - while nest_stack.size > 0 && closing_bracket_index > nest_stack[nest_stack.size - 1][:closing_bracket_index] - if nest_stack.size - nest_stack.pop - end - end - - group_name.split('/').each do |name| - nest_stack.push({ - group_name: name, - closing_bracket_index: closing_bracket_index, - }) - end - - if !content.empty? - properties = content.split(',') - - properties.each do |prop| - nest_list = nest_stack.map { |nest_prop| nest_prop[:group_name] } - - if !prop.empty? - if prop.includes?('/') - parse_single_nests(prop) { |list| nest_list += list } - else - nest_list.push prop - end - else - raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list << prop}" - end - - yield nest_list - end - end - end - end - - def self.remove_nest_groups(text : String) : String - content_bracket_pairs = get_bracket_pairs(text, false) - - content_bracket_pairs.each_key.to_a.reverse.each do |opening_bracket| - closing_bracket = content_bracket_pairs[opening_bracket] - last_comma = text.rindex(',', opening_bracket) || 0 - - text = text[0...last_comma] + text[closing_bracket + 1...text.size] - end - - return text.starts_with?(',') ? text[1...text.size] : text - end - - def self.get_bracket_pairs(text : String, recursive = true) : BracketIndex - istart = [] of Int64 - bracket_index = BracketIndex.new - - text.each_char_with_index do |char, index| - if char == '(' - istart.push(index.to_i64) - end - - if char == ')' - begin - opening = istart.pop - if recursive || (!recursive && istart.size == 0) - bracket_index[opening] = index.to_i64 - end - rescue - raise FieldsParser::ParseError.new "No matching opening parenthesis at: #{index}" - end - end - end - - if istart.size != 0 - idx = istart.pop - raise FieldsParser::ParseError.new "No matching closing parenthesis at: #{idx}" - end - - return bracket_index - end - end - - class FieldsGrouper - alias SkeletonValue = Hash(String, SkeletonValue) - - def self.create_json_skeleton(fields_text : String) : SkeletonValue - root_hash = {} of String => SkeletonValue - - FieldsParser.parse_fields(fields_text) do |nest_list| - current_item = root_hash - nest_list.each do |key| - if current_item[key]? - current_item = current_item[key] - else - current_item[key] = {} of String => SkeletonValue - current_item = current_item[key] - end - end - end - root_hash - end - - def self.create_grouped_fields_list(json_skeleton : SkeletonValue) : GroupedFieldsList - grouped_fields_list = GroupedFieldsList.new - json_skeleton.each do |key, value| - grouped_fields_list.push key - - nested_keys = create_grouped_fields_list(value) - grouped_fields_list.push nested_keys unless nested_keys.empty? - end - return grouped_fields_list - end - end - - class FilterError < Exception - end - - def self.filter(item : JSON::Any, fields_text : String, in_place : Bool = true) - skeleton = FieldsGrouper.create_json_skeleton(fields_text) - grouped_fields_list = FieldsGrouper.create_grouped_fields_list(skeleton) - filter(item, grouped_fields_list, in_place) - end - - def self.filter(item : JSON::Any, grouped_fields_list : GroupedFieldsList, in_place : Bool = true) : JSON::Any - item = item.clone unless in_place - - if !item.as_h? && !item.as_a? - raise FilterError.new "Can't filter '#{item}' by #{grouped_fields_list}" - end - - top_level_keys = Array(String).new - grouped_fields_list.each do |value| - if value.is_a? String - top_level_keys.push value - elsif value.is_a? Array - if !top_level_keys.empty? - key_to_filter = top_level_keys.last - - if item.as_h? - filter(item[key_to_filter], value, in_place: true) - elsif item.as_a? - item.as_a.each { |arr_item| filter(arr_item[key_to_filter], value, in_place: true) } - end - else - raise FilterError.new "Tried to filter while top level keys list is empty" - end - end - end - - if item.as_h? - item.as_h.select! top_level_keys - elsif item.as_a? - item.as_a.map { |value| filter(value, top_level_keys, in_place: true) } - end - - item - end -end diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr index e2e50905..03349595 100644 --- a/src/invidious/helpers/logger.cr +++ b/src/invidious/helpers/logger.cr @@ -1,3 +1,5 @@ +require "colorize" + enum LogLevel All = 0 Trace = 1 @@ -10,7 +12,9 @@ enum LogLevel end class Invidious::LogHandler < Kemal::BaseLogHandler - def initialize(@io : IO = STDOUT, @level = LogLevel::Debug) + def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, use_color : Bool = true) + Colorize.enabled = use_color + Colorize.on_tty_only! end def call(context : HTTP::Server::Context) @@ -34,28 +38,27 @@ class Invidious::LogHandler < Kemal::BaseLogHandler context end - def puts(message : String) - @io << message << '\n' - @io.flush - end - def write(message : String) @io << message @io.flush end - def set_log_level(level : String) - @level = LogLevel.parse(level) - end - - def set_log_level(level : LogLevel) - @level = level + def color(level) + case level + when LogLevel::Trace then :cyan + when LogLevel::Debug then :green + when LogLevel::Info then :white + when LogLevel::Warn then :yellow + when LogLevel::Error then :red + when LogLevel::Fatal then :magenta + else :default + end end {% for level in %w(trace debug info warn error fatal) %} def {{level.id}}(message : String) if LogLevel::{{level.id.capitalize}} >= @level - puts("#{Time.utc} [{{level.id}}] #{message}") + puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}}))) end end {% end %} diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index e0bd7279..f8e8f187 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -1,3 +1,16 @@ +@[Flags] +enum VideoBadges + LiveNow + Premium + ThreeD + FourK + New + EightK + VR180 + VR360 + ClosedCaptions +end + struct SearchVideo include DB::Serializable @@ -9,10 +22,10 @@ struct SearchVideo property views : Int64 property description_html : String property length_seconds : Int32 - property live_now : Bool - property premium : Bool property premiere_timestamp : Time? property author_verified : Bool + property author_thumbnail : String? + property badges : VideoBadges def to_xml(auto_generated, query_params, xml : XML::Builder) query_params["v"] = self.id @@ -76,6 +89,24 @@ struct SearchVideo json.field "authorUrl", "/channel/#{self.ucid}" json.field "authorVerified", self.author_verified + author_thumbnail = self.author_thumbnail + + if author_thumbnail + json.field "authorThumbnails" do + json.array do + qualities = {32, 48, 76, 100, 176, 512} + + qualities.each do |quality| + json.object do + json.field "url", author_thumbnail.gsub(/=s\d+/, "=s#{quality}") + json.field "width", quality + json.field "height", quality + end + end + end + end + end + json.field "videoThumbnails" do Invidious::JSONify::APIv1.thumbnails(json, self.id) end @@ -88,13 +119,20 @@ struct SearchVideo json.field "published", self.published.to_unix json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) json.field "lengthSeconds", self.length_seconds - json.field "liveNow", self.live_now - json.field "premium", self.premium - json.field "isUpcoming", self.is_upcoming + json.field "liveNow", self.badges.live_now? + json.field "premium", self.badges.premium? + json.field "isUpcoming", self.upcoming? if self.premiere_timestamp json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix end + json.field "isNew", self.badges.new? + json.field "is4k", self.badges.four_k? + json.field "is8k", self.badges.eight_k? + json.field "isVr180", self.badges.vr180? + json.field "isVr360", self.badges.vr360? + json.field "is3d", self.badges.three_d? + json.field "hasCaptions", self.badges.closed_captions? end end @@ -109,7 +147,7 @@ struct SearchVideo to_json(nil, json) end - def is_upcoming + def upcoming? premiere_timestamp ? true : false end end @@ -186,6 +224,7 @@ struct SearchChannel property author_thumbnail : String property subscriber_count : Int32 property video_count : Int32 + property channel_handle : String? property description_html : String property auto_generated : Bool property author_verified : Bool @@ -203,7 +242,7 @@ struct SearchChannel qualities.each do |quality| json.object do - json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}") + json.field "url", self.author_thumbnail.gsub(/=s\d+/, "=s#{quality}") json.field "width", quality json.field "height", quality end @@ -214,6 +253,7 @@ struct SearchChannel json.field "autoGenerated", self.auto_generated json.field "subCount", self.subscriber_count json.field "videoCount", self.video_count + json.field "channelHandle", self.channel_handle json.field "description", html_to_content(self.description_html) json.field "descriptionHtml", self.description_html diff --git a/src/invidious/helpers/sig_helper.cr b/src/invidious/helpers/sig_helper.cr new file mode 100644 index 00000000..6d198a42 --- /dev/null +++ b/src/invidious/helpers/sig_helper.cr @@ -0,0 +1,349 @@ +require "uri" +require "socket" +require "socket/tcp_socket" +require "socket/unix_socket" + +{% if flag?(:advanced_debug) %} + require "io/hexdump" +{% end %} + +private alias NetworkEndian = IO::ByteFormat::NetworkEndian + +module Invidious::SigHelper + enum UpdateStatus + Updated + UpdateNotRequired + Error + end + + # ------------------- + # Payload types + # ------------------- + + abstract struct Payload + end + + struct StringPayload < Payload + getter string : String + + def initialize(str : String) + raise Exception.new("SigHelper: String can't be empty") if str.empty? + @string = str + end + + def self.from_bytes(slice : Bytes) + size = IO::ByteFormat::NetworkEndian.decode(UInt16, slice) + if size == 0 # Error code + raise Exception.new("SigHelper: Server encountered an error") + end + + if (slice.bytesize - 2) != size + raise Exception.new("SigHelper: String size mismatch") + end + + if str = String.new(slice[2..]) + return self.new(str) + else + raise Exception.new("SigHelper: Can't read string from socket") + end + end + + def to_io(io) + # `.to_u16` raises if there is an overflow during the conversion + io.write_bytes(@string.bytesize.to_u16, NetworkEndian) + io.write(@string.to_slice) + end + end + + private enum Opcode + FORCE_UPDATE = 0 + DECRYPT_N_SIGNATURE = 1 + DECRYPT_SIGNATURE = 2 + GET_SIGNATURE_TIMESTAMP = 3 + GET_PLAYER_STATUS = 4 + PLAYER_UPDATE_TIMESTAMP = 5 + end + + private record Request, + opcode : Opcode, + payload : Payload? + + # ---------------------- + # High-level functions + # ---------------------- + + class Client + @mux : Multiplexor + + def initialize(uri_or_path) + @mux = Multiplexor.new(uri_or_path) + end + + # Forces the server to re-fetch the YouTube player, and extract the necessary + # components from it (nsig function code, sig function code, signature timestamp). + def force_update : UpdateStatus + request = Request.new(Opcode::FORCE_UPDATE, nil) + + value = send_request(request) do |bytes| + IO::ByteFormat::NetworkEndian.decode(UInt16, bytes) + end + + case value + when 0x0000 then return UpdateStatus::Error + when 0xFFFF then return UpdateStatus::UpdateNotRequired + when 0xF44F then return UpdateStatus::Updated + else + code = value.nil? ? "nil" : value.to_s(base: 16) + raise Exception.new("SigHelper: Invalid status code received #{code}") + end + end + + # Decrypt a provided n signature using the server's current nsig function + # code, and return the result (or an error). + def decrypt_n_param(n : String) : String? + request = Request.new(Opcode::DECRYPT_N_SIGNATURE, StringPayload.new(n)) + + n_dec = self.send_request(request) do |bytes| + StringPayload.from_bytes(bytes).string + end + + return n_dec + end + + # Decrypt a provided s signature using the server's current sig function + # code, and return the result (or an error). + def decrypt_sig(sig : String) : String? + request = Request.new(Opcode::DECRYPT_SIGNATURE, StringPayload.new(sig)) + + sig_dec = self.send_request(request) do |bytes| + StringPayload.from_bytes(bytes).string + end + + return sig_dec + end + + # Return the signature timestamp from the server's current player + def get_signature_timestamp : UInt64? + request = Request.new(Opcode::GET_SIGNATURE_TIMESTAMP, nil) + + return self.send_request(request) do |bytes| + IO::ByteFormat::NetworkEndian.decode(UInt64, bytes) + end + end + + # Return the current player's version + def get_player : UInt32? + request = Request.new(Opcode::GET_PLAYER_STATUS, nil) + + return self.send_request(request) do |bytes| + has_player = (bytes[0] == 0xFF) + player_version = IO::ByteFormat::NetworkEndian.decode(UInt32, bytes[1..4]) + has_player ? player_version : nil + end + end + + # Return when the player was last updated + def get_player_timestamp : UInt64? + request = Request.new(Opcode::PLAYER_UPDATE_TIMESTAMP, nil) + + return self.send_request(request) do |bytes| + IO::ByteFormat::NetworkEndian.decode(UInt64, bytes) + end + end + + private def send_request(request : Request, &) + channel = @mux.send(request) + slice = channel.receive + return yield slice + rescue ex + LOGGER.debug("SigHelper: Error when sending a request") + LOGGER.trace(ex.inspect_with_backtrace) + return nil + end + end + + # --------------------- + # Low level functions + # --------------------- + + class Multiplexor + alias TransactionID = UInt32 + record Transaction, channel = ::Channel(Bytes).new + + @prng = Random.new + @mutex = Mutex.new + @queue = {} of TransactionID => Transaction + + @conn : Connection + @uri_or_path : String + + def initialize(@uri_or_path) + @conn = Connection.new(uri_or_path) + listen + end + + def listen : Nil + raise "Socket is closed" if @conn.closed? + + LOGGER.debug("SigHelper: Multiplexor listening") + + spawn do + loop do + begin + receive_data + rescue ex + LOGGER.info("SigHelper: Connection to helper died with '#{ex.message}' trying to reconnect...") + # We close the socket because for some reason is not closed. + @conn.close + loop do + begin + @conn = Connection.new(@uri_or_path) + LOGGER.info("SigHelper: Reconnected to SigHelper!") + rescue ex + LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}'. Retrying") + sleep 500.milliseconds + next + end + break if !@conn.closed? + end + end + Fiber.yield + end + end + end + + def send(request : Request) + transaction = Transaction.new + transaction_id = @prng.rand(TransactionID) + + # Add transaction to queue + @mutex.synchronize do + # On a 32-bits random integer, this should never happen. Though, just in case, ... + if @queue[transaction_id]? + raise Exception.new("SigHelper: Duplicate transaction ID! You got a shiny pokemon!") + end + + @queue[transaction_id] = transaction + end + + write_packet(transaction_id, request) + + return transaction.channel + end + + def receive_data + transaction_id, slice = read_packet + + @mutex.synchronize do + if transaction = @queue.delete(transaction_id) + # Remove transaction from queue and send data to the channel + transaction.channel.send(slice) + LOGGER.trace("SigHelper: Transaction unqueued and data sent to channel") + else + raise Exception.new("SigHelper: Received transaction was not in queue") + end + end + end + + # Read a single packet from the socket + private def read_packet : {TransactionID, Bytes} + # Header + transaction_id = @conn.read_bytes(UInt32, NetworkEndian) + length = @conn.read_bytes(UInt32, NetworkEndian) + + LOGGER.trace("SigHelper: Recv transaction 0x#{transaction_id.to_s(base: 16)} / length #{length}") + + if length > 67_000 + raise Exception.new("SigHelper: Packet longer than expected (#{length})") + end + + # Payload + slice = Bytes.new(length) + @conn.read(slice) if length > 0 + + LOGGER.trace("SigHelper: payload = #{slice}") + LOGGER.trace("SigHelper: Recv transaction 0x#{transaction_id.to_s(base: 16)} - Done") + + return transaction_id, slice + end + + # Write a single packet to the socket + private def write_packet(transaction_id : TransactionID, request : Request) + LOGGER.trace("SigHelper: Send transaction 0x#{transaction_id.to_s(base: 16)} / opcode #{request.opcode}") + + io = IO::Memory.new(1024) + io.write_bytes(request.opcode.to_u8, NetworkEndian) + io.write_bytes(transaction_id, NetworkEndian) + + if payload = request.payload + payload.to_io(io) + end + + @conn.send(io) + @conn.flush + + LOGGER.trace("SigHelper: Send transaction 0x#{transaction_id.to_s(base: 16)} - Done") + end + end + + class Connection + @socket : UNIXSocket | TCPSocket + + {% if flag?(:advanced_debug) %} + @io : IO::Hexdump + {% end %} + + def initialize(host_or_path : String) + case host_or_path + when .starts_with?('/') + # Make sure that the file exists + if File.exists?(host_or_path) + @socket = UNIXSocket.new(host_or_path) + else + raise Exception.new("SigHelper: '#{host_or_path}' no such file") + end + when .starts_with?("tcp://") + uri = URI.parse(host_or_path) + @socket = TCPSocket.new(uri.host.not_nil!, uri.port.not_nil!) + else + uri = URI.parse("tcp://#{host_or_path}") + @socket = TCPSocket.new(uri.host.not_nil!, uri.port.not_nil!) + end + LOGGER.info("SigHelper: Using helper at '#{host_or_path}'") + + {% if flag?(:advanced_debug) %} + @io = IO::Hexdump.new(@socket, output: STDERR, read: true, write: true) + {% end %} + + @socket.sync = false + @socket.blocking = false + end + + def closed? : Bool + return @socket.closed? + end + + def close : Nil + @socket.close if !@socket.closed? + end + + def flush(*args, **options) + @socket.flush(*args, **options) + end + + def send(*args, **options) + @socket.send(*args, **options) + end + + # Wrap IO functions, with added debug tooling if needed + {% for function in %w(read read_bytes write write_bytes) %} + def {{function.id}}(*args, **options) + {% if flag?(:advanced_debug) %} + @io.{{function.id}}(*args, **options) + {% else %} + @socket.{{function.id}}(*args, **options) + {% end %} + end + {% end %} + end +end diff --git a/src/invidious/helpers/signatures.cr b/src/invidious/helpers/signatures.cr index ee09415b..82a28fc0 100644 --- a/src/invidious/helpers/signatures.cr +++ b/src/invidious/helpers/signatures.cr @@ -1,73 +1,53 @@ -alias SigProc = Proc(Array(String), Int32, Array(String)) +require "http/params" +require "./sig_helper" -struct DecryptFunction - @decrypt_function = [] of {SigProc, Int32} - @decrypt_time = Time.monotonic +class Invidious::DecryptFunction + @last_update : Time = Time.utc - 42.days - def initialize(@use_polling = true) + def initialize(uri_or_path) + @client = SigHelper::Client.new(uri_or_path) + self.check_update end - def update_decrypt_function - @decrypt_function = fetch_decrypt_function - end - - private def fetch_decrypt_function(id = "CvFH_6DNRCY") - document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en").body - url = document.match(/src="(?<url>\/s\/player\/[^\/]+\/player_ias[^\/]+\/en_US\/base.js)"/).not_nil!["url"] - player = YT_POOL.client &.get(url).body - - function_name = player.match(/^(?<name>[^=]+)=function\(\w\){\w=\w\.split\(""\);[^\. ]+\.[^( ]+/m).not_nil!["name"] - function_body = player.match(/^#{Regex.escape(function_name)}=function\(\w\){(?<body>[^}]+)}/m).not_nil!["body"] - function_body = function_body.split(";")[1..-2] - - var_name = function_body[0][0, 2] - var_body = player.delete("\n").match(/var #{Regex.escape(var_name)}={(?<body>(.*?))};/).not_nil!["body"] - - operations = {} of String => SigProc - var_body.split("},").each do |operation| - op_name = operation.match(/^[^:]+/).not_nil![0] - op_body = operation.match(/\{[^}]+/).not_nil![0] - - case op_body - when "{a.reverse()" - operations[op_name] = ->(a : Array(String), _b : Int32) { a.reverse } - when "{a.splice(0,b)" - operations[op_name] = ->(a : Array(String), b : Int32) { a.delete_at(0..(b - 1)); a } - else - operations[op_name] = ->(a : Array(String), b : Int32) { c = a[0]; a[0] = a[b % a.size]; a[b % a.size] = c; a } - end - end + def check_update + # If we have updated in the last 5 minutes, do nothing + return if (Time.utc - @last_update) < 5.minutes - decrypt_function = [] of {SigProc, Int32} - function_body.each do |function| - function = function.lchop(var_name).delete("[].") + # Get the amount of time elapsed since when the player was updated, in the + # event where multiple invidious processes are run in parallel. + update_time_elapsed = (@client.get_player_timestamp || 301).seconds - op_name = function.match(/[^\(]+/).not_nil![0] - value = function.match(/\(\w,(?<value>[\d]+)\)/).not_nil!["value"].to_i - - decrypt_function << {operations[op_name], value} + if update_time_elapsed > 5.minutes + LOGGER.debug("Signature: Player might be outdated, updating") + @client.force_update + @last_update = Time.utc end - - return decrypt_function end - def decrypt_signature(fmt : Hash(String, JSON::Any)) - return "" if !fmt["s"]? || !fmt["sp"]? - - sp = fmt["sp"].as_s - sig = fmt["s"].as_s.split("") - if !@use_polling - now = Time.monotonic - if now - @decrypt_time > 60.seconds || @decrypt_function.size == 0 - @decrypt_function = fetch_decrypt_function - @decrypt_time = Time.monotonic - end - end + def decrypt_nsig(n : String) : String? + self.check_update + return @client.decrypt_n_param(n) + rescue ex + LOGGER.debug(ex.message || "Signature: Unknown error") + LOGGER.trace(ex.inspect_with_backtrace) + return nil + end - @decrypt_function.each do |proc, value| - sig = proc.call(sig, value) - end + def decrypt_signature(str : String) : String? + self.check_update + return @client.decrypt_sig(str) + rescue ex + LOGGER.debug(ex.message || "Signature: Unknown error") + LOGGER.trace(ex.inspect_with_backtrace) + return nil + end - return "&#{sp}=#{sig.join("")}" + def get_sts : UInt64? + self.check_update + return @client.get_signature_timestamp + rescue ex + LOGGER.debug(ex.message || "Signature: Unknown error") + LOGGER.trace(ex.inspect_with_backtrace) + return nil end end diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index a006d602..4d9bb28d 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -52,9 +52,9 @@ def recode_length_seconds(time) end def decode_interval(string : String) : Time::Span - rawMinutes = string.try &.to_i32? + raw_minutes = string.try &.to_i32? - if !rawMinutes + if !raw_minutes hours = /(?<hours>\d+)h/.match(string).try &.["hours"].try &.to_i32 hours ||= 0 @@ -63,7 +63,7 @@ def decode_interval(string : String) : Time::Span time = Time::Span.new(hours: hours, minutes: minutes) else - time = Time::Span.new(minutes: rawMinutes) + time = Time::Span.new(minutes: raw_minutes) end return time @@ -262,7 +262,7 @@ def get_referer(env, fallback = "/", unroll = true) end referer = referer.request_target - referer = "/" + referer.gsub(/[^\/?@&%=\-_.:,0-9a-zA-Z]/, "").lstrip("/\\") + referer = "/" + referer.gsub(/[^\/?@&%=\-_.:,*0-9a-zA-Z]/, "").lstrip("/\\") if referer == env.request.path referer = fallback @@ -323,68 +323,6 @@ def parse_range(range) return 0_i64, nil end -def fetch_random_instance - begin - instance_api_client = make_client(URI.parse("https://api.invidious.io")) - - # Timeouts - instance_api_client.connect_timeout = 10.seconds - instance_api_client.dns_timeout = 10.seconds - - instance_list = JSON.parse(instance_api_client.get("/instances.json").body).as_a - instance_api_client.close - rescue Socket::ConnectError | IO::TimeoutError | JSON::ParseException - instance_list = [] of JSON::Any - end - - filtered_instance_list = [] of String - - instance_list.each do |data| - # TODO Check if current URL is onion instance and use .onion types if so. - if data[1]["type"] == "https" - # Instances can have statistics disabled, which is an requirement of version validation. - # as_nil? doesn't exist. Thus we'll have to handle the error raised if as_nil fails. - begin - data[1]["stats"].as_nil - next - rescue TypeCastError - end - - # stats endpoint could also lack the software dict. - next if data[1]["stats"]["software"]?.nil? - - # Makes sure the instance isn't too outdated. - if remote_version = data[1]["stats"]?.try &.["software"]?.try &.["version"] - remote_commit_date = remote_version.as_s.match(/\d{4}\.\d{2}\.\d{2}/) - next if !remote_commit_date - - remote_commit_date = Time.parse(remote_commit_date[0], "%Y.%m.%d", Time::Location::UTC) - local_commit_date = Time.parse(CURRENT_VERSION, "%Y.%m.%d", Time::Location::UTC) - - next if (remote_commit_date - local_commit_date).abs.days > 30 - - begin - data[1]["monitor"].as_nil - health = data[1]["monitor"].as_h["dailyRatios"][0].as_h["ratio"] - filtered_instance_list << data[0].as_s if health.to_s.to_f > 90 - rescue TypeCastError - # We can't check the health if the monitoring is broken. Thus we'll just add it to the list - # and move on. Ideally we'll ignore any instance that has broken health monitoring but due to the fact that - # it's an error that often occurs with all the instances at the same time, we have to just skip the check. - filtered_instance_list << data[0].as_s - end - end - end - end - - # If for some reason no instances managed to get fetched successfully then we'll just redirect to redirect.invidious.io - if filtered_instance_list.size == 0 - return "redirect.invidious.io" - end - - return filtered_instance_list.sample(1)[0] -end - def reduce_uri(uri : URI | String, max_length : Int32 = 50, suffix : String = "…") : String str = uri.to_s.sub(/^https?:\/\//, "") if str.size > max_length diff --git a/src/invidious/helpers/webvtt.cr b/src/invidious/helpers/webvtt.cr new file mode 100644 index 00000000..260d250f --- /dev/null +++ b/src/invidious/helpers/webvtt.cr @@ -0,0 +1,81 @@ +# Namespace for logic relating to generating WebVTT files +# +# Probably not compliant to WebVTT's specs but it is enough for Invidious. +module WebVTT + # A WebVTT builder generates WebVTT files + private class Builder + # See https://developer.mozilla.org/en-US/docs/Web/API/WebVTT_API#cue_payload + private ESCAPE_SUBSTITUTIONS = { + '&' => "&", + '<' => "<", + '>' => ">", + '\u200E' => "‎", + '\u200F' => "‏", + '\u00A0' => " ", + } + + def initialize(@io : IO) + end + + # Writes an vtt cue with the specified time stamp and contents + def cue(start_time : Time::Span, end_time : Time::Span, text : String) + timestamp(start_time, end_time) + @io << self.escape(text) + @io << "\n\n" + end + + private def timestamp(start_time : Time::Span, end_time : Time::Span) + timestamp_component(start_time) + @io << " --> " + timestamp_component(end_time) + + @io << '\n' + end + + private def timestamp_component(timestamp : Time::Span) + @io << timestamp.hours.to_s.rjust(2, '0') + @io << ':' << timestamp.minutes.to_s.rjust(2, '0') + @io << ':' << timestamp.seconds.to_s.rjust(2, '0') + @io << '.' << timestamp.milliseconds.to_s.rjust(3, '0') + end + + private def escape(text : String) : String + return text.gsub(ESCAPE_SUBSTITUTIONS) + end + + def document(setting_fields : Hash(String, String)? = nil, &) + @io << "WEBVTT\n" + + if setting_fields + setting_fields.each do |name, value| + @io << name << ": " << value << '\n' + end + end + + @io << '\n' + + yield + end + end + + # Returns the resulting `String` of writing WebVTT to the yielded `WebVTT::Builder` + # + # ``` + # string = WebVTT.build do |vtt| + # vtt.cue(Time::Span.new(seconds: 1), Time::Span.new(seconds: 2), "Line 1") + # vtt.cue(Time::Span.new(seconds: 2), Time::Span.new(seconds: 3), "Line 2") + # end + # + # string # => "WEBVTT\n\n00:00:01.000 --> 00:00:02.000\nLine 1\n\n00:00:02.000 --> 00:00:03.000\nLine 2\n\n" + # ``` + # + # Accepts an optional settings fields hash to add settings attribute to the resulting vtt file. + def self.build(setting_fields : Hash(String, String)? = nil, &) + String.build do |str| + builder = Builder.new(str) + builder.document(setting_fields) do + yield builder + end + end + end +end diff --git a/src/invidious/http_server/utils.cr b/src/invidious/http_server/utils.cr index 222dfc4a..623a9177 100644 --- a/src/invidious/http_server/utils.cr +++ b/src/invidious/http_server/utils.cr @@ -11,11 +11,12 @@ module Invidious::HttpServer params = url.query_params params["host"] = url.host.not_nil! # Should never be nil, in theory params["region"] = region if !region.nil? + url.query_params = params if absolute - return "#{HOST_URL}#{url.request_target}?#{params}" + return "#{HOST_URL}#{url.request_target}" else - return "#{url.request_target}?#{params}" + return url.request_target end end diff --git a/src/invidious/jobs/bypass_captcha_job.cr b/src/invidious/jobs/bypass_captcha_job.cr deleted file mode 100644 index 71f8a938..00000000 --- a/src/invidious/jobs/bypass_captcha_job.cr +++ /dev/null @@ -1,135 +0,0 @@ -class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob - def begin - loop do - begin - random_video = PG_DB.query_one?("select id, ucid from (select id, ucid from channel_videos limit 1000) as s ORDER BY RANDOM() LIMIT 1", as: {id: String, ucid: String}) - if !random_video - random_video = {id: "zj82_v2R6ts", ucid: "UCK87Lox575O_HCHBWaBSyGA"} - end - {"/watch?v=#{random_video["id"]}&gl=US&hl=en&has_verified=1&bpctr=9999999999", produce_channel_videos_url(ucid: random_video["ucid"])}.each do |path| - response = YT_POOL.client &.get(path) - if response.body.includes?("To continue with your YouTube experience, please fill out the form below.") - html = XML.parse_html(response.body) - form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil! - site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"] - s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"] - - inputs = {} of String => String - form.xpath_nodes(%(.//input[@name])).map do |node| - inputs[node["name"]] = node["value"] - end - - headers = response.cookies.add_request_headers(HTTP::Headers.new) - - response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/createTask", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "task" => { - "type" => "NoCaptchaTaskProxyless", - "websiteURL" => "https://www.youtube.com#{path}", - "websiteKey" => site_key, - "recaptchaDataSValue" => s_value, - }, - }.to_json).body) - - raise response["error"].as_s if response["error"]? - task_id = response["taskId"].as_i - - loop do - sleep 10.seconds - - response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/getTaskResult", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "taskId" => task_id, - }.to_json).body) - - if response["status"]?.try &.== "ready" - break - elsif response["errorId"]?.try &.as_i != 0 - raise response["errorDescription"].as_s - end - end - - inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s - headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || "" - response = YT_POOL.client &.post("/das_captcha", headers, form: inputs) - - response.cookies - .select { |cookie| cookie.name != "PREF" } - .each { |cookie| CONFIG.cookies << cookie } - - # Persist cookies between runs - File.write("config/config.yml", CONFIG.to_yaml) - elsif response.headers["Location"]?.try &.includes?("/sorry/index") - location = response.headers["Location"].try { |u| URI.parse(u) } - headers = HTTP::Headers{":authority" => location.host.not_nil!} - response = YT_POOL.client &.get(location.request_target, headers) - - html = XML.parse_html(response.body) - form = html.xpath_node(%(//form[@action="index"])).not_nil! - site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"] - s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"] - - inputs = {} of String => String - form.xpath_nodes(%(.//input[@name])).map do |node| - inputs[node["name"]] = node["value"] - end - - captcha_client = HTTPClient.new(URI.parse(CONFIG.captcha_api_url)) - captcha_client.family = CONFIG.force_resolve || Socket::Family::INET - response = JSON.parse(captcha_client.post("/createTask", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "task" => { - "type" => "NoCaptchaTaskProxyless", - "websiteURL" => location.to_s, - "websiteKey" => site_key, - "recaptchaDataSValue" => s_value, - }, - }.to_json).body) - - captcha_client.close - - raise response["error"].as_s if response["error"]? - task_id = response["taskId"].as_i - - loop do - sleep 10.seconds - - response = JSON.parse(captcha_client.post("/getTaskResult", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "taskId" => task_id, - }.to_json).body) - - if response["status"]?.try &.== "ready" - break - elsif response["errorId"]?.try &.as_i != 0 - raise response["errorDescription"].as_s - end - end - - inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s - headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || "" - response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs) - headers = HTTP::Headers{ - "Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0], - } - cookies = HTTP::Cookies.from_client_headers(headers) - - cookies.each { |cookie| CONFIG.cookies << cookie } - - # Persist cookies between runs - File.write("config/config.yml", CONFIG.to_yaml) - end - end - rescue ex - LOGGER.error("BypassCaptchaJob: #{ex.message}") - ensure - sleep 1.minute - Fiber.yield - end - end - end -end diff --git a/src/invidious/jobs/instance_refresh_job.cr b/src/invidious/jobs/instance_refresh_job.cr new file mode 100644 index 00000000..cb4280b9 --- /dev/null +++ b/src/invidious/jobs/instance_refresh_job.cr @@ -0,0 +1,97 @@ +class Invidious::Jobs::InstanceListRefreshJob < Invidious::Jobs::BaseJob + # We update the internals of a constant as so it can be accessed from anywhere + # within the codebase + # + # "INSTANCES" => Array(Tuple(String, String)) # region, instance + + INSTANCES = {"INSTANCES" => [] of Tuple(String, String)} + + def initialize + end + + def begin + loop do + refresh_instances + LOGGER.info("InstanceListRefreshJob: Done, sleeping for 30 minutes") + sleep 30.minute + Fiber.yield + end + end + + # Refreshes the list of instances used for redirects. + # + # Does the following three checks for each instance + # - Is it a clear-net instance? + # - Is it an instance with a good uptime? + # - Is it an updated instance? + private def refresh_instances + raw_instance_list = self.fetch_instances + filtered_instance_list = [] of Tuple(String, String) + + raw_instance_list.each do |instance_data| + # TODO allow Tor hidden service instances when the current instance + # is also a hidden service. Same for i2p and any other non-clearnet instances. + begin + domain = instance_data[0] + info = instance_data[1] + stats = info["stats"] + + next unless info["type"] == "https" + next if bad_uptime?(info["monitor"]) + next if outdated?(stats["software"]["version"]) + + filtered_instance_list << {info["region"].as_s, domain.as_s} + rescue ex + if domain + LOGGER.info("InstanceListRefreshJob: failed to parse information from '#{domain}' because \"#{ex}\"\n\"#{ex.backtrace.join('\n')}\" ") + else + LOGGER.info("InstanceListRefreshJob: failed to parse information from an instance because \"#{ex}\"\n\"#{ex.backtrace.join('\n')}\" ") + end + end + end + + if !filtered_instance_list.empty? + INSTANCES["INSTANCES"] = filtered_instance_list + end + end + + # Fetches information regarding instances from api.invidious.io or an otherwise configured URL + private def fetch_instances : Array(JSON::Any) + begin + # We directly call the stdlib HTTP::Client here as it allows us to negate the effects + # of the force_resolve config option. This is needed as api.invidious.io does not support ipv6 + # and as such the following request raises if we were to use force_resolve with the ipv6 value. + instance_api_client = HTTP::Client.new(URI.parse("https://api.invidious.io")) + + # Timeouts + instance_api_client.connect_timeout = 10.seconds + instance_api_client.dns_timeout = 10.seconds + + raw_instance_list = JSON.parse(instance_api_client.get("/instances.json").body).as_a + instance_api_client.close + rescue ex : Socket::ConnectError | IO::TimeoutError | JSON::ParseException + raw_instance_list = [] of JSON::Any + end + + return raw_instance_list + end + + # Checks if the given target instance is outdated + private def outdated?(target_instance_version) : Bool + remote_commit_date = target_instance_version.as_s.match(/\d{4}\.\d{2}\.\d{2}/) + return false if !remote_commit_date + + remote_commit_date = Time.parse(remote_commit_date[0], "%Y.%m.%d", Time::Location::UTC) + local_commit_date = Time.parse(CURRENT_VERSION, "%Y.%m.%d", Time::Location::UTC) + + return (remote_commit_date - local_commit_date).abs.days > 30 + end + + # Checks if the uptime of the target instance is greater than 90% over a 30 day period + private def bad_uptime?(target_instance_health_monitor) : Bool + return true if !target_instance_health_monitor["down"].as_bool == false + return true if target_instance_health_monitor["uptime"].as_f < 90 + + return false + end +end diff --git a/src/invidious/jobs/statistics_refresh_job.cr b/src/invidious/jobs/statistics_refresh_job.cr index a113bd77..66c91ad5 100644 --- a/src/invidious/jobs/statistics_refresh_job.cr +++ b/src/invidious/jobs/statistics_refresh_job.cr @@ -18,6 +18,13 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob "updatedAt" => Time.utc.to_unix, "lastChannelRefreshedAt" => 0_i64, }, + + # + # "totalRequests" => 0_i64, + # "successfulRequests" => 0_i64 + # "ratio" => 0_i64 + # + "playback" => {} of String => Int64 | Float64, } private getter db : DB::Database @@ -30,7 +37,7 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob loop do refresh_stats - sleep 1.minute + sleep 10.minute Fiber.yield end end @@ -49,12 +56,15 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob users = STATISTICS.dig("usage", "users").as(Hash(String, Int64)) users["total"] = Invidious::Database::Statistics.count_users_total - users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_1m - users["activeMonth"] = Invidious::Database::Statistics.count_users_active_6m + users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_6m + users["activeMonth"] = Invidious::Database::Statistics.count_users_active_1m STATISTICS["metadata"] = { "updatedAt" => Time.utc.to_unix, "lastChannelRefreshedAt" => Invidious::Database::Statistics.channel_last_update.try &.to_unix || 0_i64, } + + # Reset playback requests tracker + STATISTICS["playback"] = {} of String => Int64 | Float64 end end diff --git a/src/invidious/jobs/update_decrypt_function_job.cr b/src/invidious/jobs/update_decrypt_function_job.cr deleted file mode 100644 index 6fa0ae1b..00000000 --- a/src/invidious/jobs/update_decrypt_function_job.cr +++ /dev/null @@ -1,14 +0,0 @@ -class Invidious::Jobs::UpdateDecryptFunctionJob < Invidious::Jobs::BaseJob - def begin - loop do - begin - DECRYPT_FUNCTION.update_decrypt_function - rescue ex - LOGGER.error("UpdateDecryptFunctionJob : #{ex.message}") - ensure - sleep 1.minute - Fiber.yield - end - end - end -end diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index fe4b5223..3439ae60 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -39,6 +39,7 @@ module Invidious::JSONify::APIv1 json.field "author", video.author json.field "authorId", video.ucid json.field "authorUrl", "/channel/#{video.ucid}" + json.field "authorVerified", video.author_verified json.field "authorThumbnails" do json.array do @@ -61,7 +62,8 @@ module Invidious::JSONify::APIv1 json.field "rating", 0_i64 json.field "isListed", video.is_listed json.field "liveNow", video.live_now - json.field "isUpcoming", video.is_upcoming + json.field "isPostLiveDvr", video.post_live_dvr + json.field "isUpcoming", video.upcoming? if video.premiere_timestamp json.field "premiereTimestamp", video.premiere_timestamp.try &.to_unix @@ -107,30 +109,36 @@ module Invidious::JSONify::APIv1 # On livestreams, it's not present, so always fall back to the # current unix timestamp (up to mS precision) for compatibility. last_modified = fmt["lastModified"]? - last_modified ||= "#{Time.utc.to_unix_ms.to_s}000" + last_modified ||= "#{Time.utc.to_unix_ms}000" json.field "lmt", last_modified json.field "projectionType", fmt["projectionType"] - if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) - fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 + height = fmt["height"]?.try &.as_i + width = fmt["width"]?.try &.as_i + + fps = fmt["fps"]?.try &.as_i + + if fps json.field "fps", fps - json.field "container", fmt_info["ext"] - json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] + end - if fmt_info["height"]? - json.field "resolution", "#{fmt_info["height"]}p" + if height && width + json.field "size", "#{width}x#{height}" + json.field "resolution", "#{height}p" - quality_label = "#{fmt_info["height"]}p" - if fps > 30 - quality_label += "60" - end - json.field "qualityLabel", quality_label + quality_label = "#{width > height ? height : width}p" - if fmt_info["width"]? - json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" - end + if fps && fps > 30 + quality_label += fps.to_s end + + json.field "qualityLabel", quality_label + end + + if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) + json.field "container", fmt_info["ext"] + json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] end # Livestream chunk infos @@ -154,31 +162,44 @@ module Invidious::JSONify::APIv1 json.array do video.fmt_stream.each do |fmt| json.object do - json.field "url", fmt["url"] + if proxy + json.field "url", Invidious::HttpServer::Utils.proxy_video_url( + fmt["url"].to_s, absolute: true + ) + else + json.field "url", fmt["url"] + end json.field "itag", fmt["itag"].as_i.to_s json.field "type", fmt["mimeType"] json.field "quality", fmt["quality"] - fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) - if fmt_info - fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 + json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]? + + height = fmt["height"]?.try &.as_i + width = fmt["width"]?.try &.as_i + + fps = fmt["fps"]?.try &.as_i + + if fps json.field "fps", fps - json.field "container", fmt_info["ext"] - json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] + end - if fmt_info["height"]? - json.field "resolution", "#{fmt_info["height"]}p" + if height && width + json.field "size", "#{width}x#{height}" + json.field "resolution", "#{height}p" - quality_label = "#{fmt_info["height"]}p" - if fps > 30 - quality_label += "60" - end - json.field "qualityLabel", quality_label + quality_label = "#{width > height ? height : width}p" - if fmt_info["width"]? - json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" - end + if fps && fps > 30 + quality_label += fps.to_s end + + json.field "qualityLabel", quality_label + end + + if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) + json.field "container", fmt_info["ext"] + json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] end end end @@ -226,6 +247,7 @@ module Invidious::JSONify::APIv1 json.field "author", rv["author"] json.field "authorUrl", "/channel/#{rv["ucid"]?}" json.field "authorId", rv["ucid"]? + json.field "authorVerified", rv["author_verified"] == "true" if rv["author_thumbnail"]? json.field "authorThumbnails" do json.array do @@ -245,6 +267,12 @@ module Invidious::JSONify::APIv1 json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i json.field "viewCountText", rv["short_view_count"]? json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 + json.field "published", rv["published"]? + if !rv["published"]?.nil? + json.field "publishedText", translate(locale, "`x` ago", recode_date(Time.parse_rfc3339(rv["published"].to_s), locale)) + else + json.field "publishedText", "" + end end end end @@ -255,17 +283,17 @@ module Invidious::JSONify::APIv1 def storyboards(json, id, storyboards) json.array do - storyboards.each do |storyboard| + storyboards.each do |sb| json.object do - json.field "url", "/api/v1/storyboards/#{id}?width=#{storyboard[:width]}&height=#{storyboard[:height]}" - json.field "templateUrl", storyboard[:url] - json.field "width", storyboard[:width] - json.field "height", storyboard[:height] - json.field "count", storyboard[:count] - json.field "interval", storyboard[:interval] - json.field "storyboardWidth", storyboard[:storyboard_width] - json.field "storyboardHeight", storyboard[:storyboard_height] - json.field "storyboardCount", storyboard[:storyboard_count] + json.field "url", "/api/v1/storyboards/#{id}?width=#{sb.width}&height=#{sb.height}" + json.field "templateUrl", sb.url.to_s + json.field "width", sb.width + json.field "height", sb.height + json.field "count", sb.count + json.field "interval", sb.interval + json.field "storyboardWidth", sb.columns + json.field "storyboardHeight", sb.rows + json.field "storyboardCount", sb.images_count end end end diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr index 823ca85b..28ff0ff6 100644 --- a/src/invidious/mixes.cr +++ b/src/invidious/mixes.cr @@ -81,7 +81,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil) }) end -def template_mix(mix) +def template_mix(mix, listen) html = <<-END_HTML <h3> <a href="/mix?list=#{mix["mixId"]}"> @@ -95,7 +95,7 @@ def template_mix(mix) mix["videos"].as_a.each do |video| html += <<-END_HTML <li class="pure-menu-item"> - <a href="/watch?v=#{video["videoId"]}&list=#{mix["mixId"]}"> + <a href="/watch?v=#{video["videoId"]}&list=#{mix["mixId"]}#{listen ? "&listen=1" : ""}"> <div class="thumbnail"> <img loading="lazy" class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg" alt="" /> <p class="length">#{recode_length_seconds(video["lengthSeconds"].as_i)}</p> diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 955e0855..b670c009 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -46,8 +46,14 @@ struct PlaylistVideo XML.build { |xml| to_xml(xml) } end + def to_json(locale : String?, json : JSON::Builder) + to_json(json) + end + def to_json(json : JSON::Builder, index : Int32? = nil) json.object do + json.field "type", "video" + json.field "title", self.title json.field "videoId", self.id @@ -67,6 +73,7 @@ struct PlaylistVideo end json.field "lengthSeconds", self.length_seconds + json.field "liveNow", self.live_now end end @@ -263,7 +270,7 @@ end def subscribe_playlist(user, playlist) playlist = InvidiousPlaylist.new({ - title: playlist.title.byte_slice(0, 150), + title: playlist.title[..150], id: playlist.id, author: user.email, description: "", # Max 5000 characters @@ -366,6 +373,8 @@ def fetch_playlist(plid : String) if text.includes? "video" video_count = text.gsub(/\D/, "").to_i? || 0 + elsif text.includes? "episode" + video_count = text.gsub(/\D/, "").to_i? || 0 elsif text.includes? "view" views = text.gsub(/\D/, "").to_i64? || 0_i64 else @@ -496,7 +505,7 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) return videos end -def template_playlist(playlist) +def template_playlist(playlist, listen) html = <<-END_HTML <h3> <a href="/playlist?list=#{playlist["playlistId"]}"> @@ -510,7 +519,7 @@ def template_playlist(playlist) playlist["videos"].as_a.each do |video| html += <<-END_HTML <li class="pure-menu-item" id="#{video["videoId"]}"> - <a href="/watch?v=#{video["videoId"]}&list=#{playlist["playlistId"]}&index=#{video["index"]}"> + <a href="/watch?v=#{video["videoId"]}&list=#{playlist["playlistId"]}&index=#{video["index"]}#{listen ? "&listen=1" : ""}"> <div class="thumbnail"> <img loading="lazy" class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg" alt="" /> <p class="length">#{recode_length_seconds(video["lengthSeconds"].as_i)}</p> diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr index 9d930841..c8db207c 100644 --- a/src/invidious/routes/account.cr +++ b/src/invidious/routes/account.cr @@ -53,7 +53,7 @@ module Invidious::Routes::Account return error_template(401, "Password is a required field") end - new_passwords = env.params.body.select { |k, v| k.match(/^new_password\[\d+\]$/) }.map { |k, v| v } + new_passwords = env.params.body.select { |k, _| k.match(/^new_password\[\d+\]$/) }.map { |_, v| v } if new_passwords.size <= 1 || new_passwords.uniq.size != 1 return error_template(400, "New passwords must match") @@ -240,7 +240,7 @@ module Invidious::Routes::Account return error_template(400, ex) end - scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v } + scopes = env.params.body.select { |k, _| k.match(/^scopes\[\d+\]$/) }.map { |_, v| v } callback_url = env.params.body["callbackUrl"]? expire = env.params.body["expire"]?.try &.to_i? @@ -328,17 +328,9 @@ module Invidious::Routes::Account end end - if env.params.query["action_revoke_token"]? - action = "action_revoke_token" - else - return env.redirect referer - end - - session = env.params.query["session"]? - session ||= "" - - case action - when .starts_with? "action_revoke_token" + case action = env.params.query["action"]? + when "revoke_token" + session = env.params.query["session"] Invidious::Database::SessionIDs.delete(sid: session, email: user.email) else return error_json(400, "Unsupported action #{action}") diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index 662d1002..6c4225e5 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -21,28 +21,27 @@ module Invidious::Routes::API::Manifest end if dashmpd = video.dash_manifest_url - manifest = YT_POOL.client &.get(URI.parse(dashmpd).request_target).body + response = YT_POOL.client &.get(URI.parse(dashmpd).request_target) - manifest = manifest.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl| - url = baseurl.lchop("<BaseURL>") - url = url.rchop("</BaseURL>") - - if local - uri = URI.parse(url) - url = "#{HOST_URL}#{uri.request_target}host/#{uri.host}/" - end + if response.status_code != 200 + haltf env, status_code: response.status_code + end + # Proxy URLs for video playback on invidious. + # Other API clients can get the original URLs by omiting `local=true`. + manifest = response.body.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl| + url = baseurl.lchop("<BaseURL>").rchop("</BaseURL>") + url = HttpServer::Utils.proxy_video_url(url, absolute: true) if local "<BaseURL>#{url}</BaseURL>" end return manifest end - adaptive_fmts = video.adaptive_fmts - + # Ditto, only proxify URLs if `local=true` is used if local - adaptive_fmts.each do |fmt| - fmt["url"] = JSON::Any.new("#{HOST_URL}#{URI.parse(fmt["url"].as_s).request_target}") + video.adaptive_fmts.each do |fmt| + fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s, absolute: true)) end end @@ -64,17 +63,23 @@ module Invidious::Routes::API::Manifest # OTF streams aren't supported yet (See https://github.com/TeamNewPipe/NewPipe/issues/2415) next if !(fmt.has_key?("indexRange") && fmt.has_key?("initRange")) + audio_track = fmt["audioTrack"]?.try &.as_h? || {} of String => JSON::Any + lang = audio_track["id"]?.try &.as_s.split('.')[0] || "und" + is_default = audio_track.has_key?("audioIsDefault") ? audio_track["audioIsDefault"].as_bool : i == 0 + displayname = audio_track["displayName"]?.try &.as_s || "Unknown" + bitrate = fmt["bitrate"] + # Different representations of the same audio should be groupped into one AdaptationSet. # However, most players don't support auto quality switching, so we have to trick them # into providing a quality selector. # See https://github.com/iv-org/invidious/issues/3074 for more details. - xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, label: fmt["bitrate"].to_s + "k") do + xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, label: "#{displayname} [#{bitrate}k]", lang: lang) do codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"') bandwidth = fmt["bitrate"].as_i itag = fmt["itag"].as_i url = fmt["url"].as_s - xml.element("Role", schemeIdUri: "urn:mpeg:dash:role:2011", value: i == 0 ? "main" : "alternate") + xml.element("Role", schemeIdUri: "urn:mpeg:dash:role:2011", value: is_default ? "main" : "alternate") xml.element("Representation", id: fmt["itag"], codecs: codecs, bandwidth: bandwidth) do xml.element("AudioChannelConfiguration", schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011", @@ -171,8 +176,9 @@ module Invidious::Routes::API::Manifest manifest = response.body if local - manifest = manifest.gsub(/^https:\/\/\w+---.{11}\.c\.youtube\.com[^\n]*/m) do |match| - path = URI.parse(match).path + manifest = manifest.gsub(/https:\/\/[^\n"]*/m) do |match| + uri = URI.parse(match) + path = uri.path path = path.lchop("/videoplayback/") path = path.rchop("/") @@ -201,7 +207,7 @@ module Invidious::Routes::API::Manifest raw_params["fvip"] = fvip["fvip"] end - raw_params["local"] = "true" + raw_params["host"] = uri.host.not_nil! "#{HOST_URL}/videoplayback?#{raw_params}" end diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index adf05d30..588bbc2a 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -27,10 +27,21 @@ module Invidious::Routes::API::V1::Channels # Retrieve "sort by" setting from URL parameters sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" - begin - videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by) - rescue ex - return error_json(500, ex) + if channel.is_age_gated + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULF")) + videos = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + videos = [] of PlaylistVideo + end + next_continuation = nil + else + begin + videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by) + rescue ex + return error_json(500, ex) + end end JSON.build do |json| @@ -84,12 +95,14 @@ module Invidious::Routes::API::V1::Channels json.field "joined", channel.joined.to_unix json.field "autoGenerated", channel.auto_generated + json.field "ageGated", channel.is_age_gated json.field "isFamilyFriendly", channel.is_family_friendly json.field "description", html_to_content(channel.description_html) json.field "descriptionHtml", channel.description_html json.field "allowedRegions", channel.allowed_regions json.field "tabs", channel.tabs + json.field "tags", channel.tags json.field "authorVerified", channel.verified json.field "latestVideos" do @@ -141,12 +154,23 @@ module Invidious::Routes::API::V1::Channels sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" continuation = env.params.query["continuation"]? - begin - videos, next_continuation = Channel::Tabs.get_60_videos( - channel, continuation: continuation, sort_by: sort_by - ) - rescue ex - return error_json(500, ex) + if channel.is_age_gated + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULF")) + videos = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + videos = [] of PlaylistVideo + end + next_continuation = nil + else + begin + videos, next_continuation = Channel::Tabs.get_60_videos( + channel, continuation: continuation, sort_by: sort_by + ) + rescue ex + return error_json(500, ex) + end end return JSON.build do |json| @@ -173,14 +197,26 @@ module Invidious::Routes::API::V1::Channels get_channel() # Retrieve continuation from URL parameters + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" continuation = env.params.query["continuation"]? - begin - videos, next_continuation = Channel::Tabs.get_shorts( - channel, continuation: continuation - ) - rescue ex - return error_json(500, ex) + if channel.is_age_gated + begin + playlist = get_playlist(channel.ucid.sub("UC", "UUSH")) + videos = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + videos = [] of PlaylistVideo + end + next_continuation = nil + else + begin + videos, next_continuation = Channel::Tabs.get_shorts( + channel, continuation: continuation, sort_by: sort_by + ) + rescue ex + return error_json(500, ex) + end end return JSON.build do |json| @@ -207,14 +243,26 @@ module Invidious::Routes::API::V1::Channels get_channel() # Retrieve continuation from URL parameters + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" continuation = env.params.query["continuation"]? - begin - videos, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation - ) - rescue ex - return error_json(500, ex) + if channel.is_age_gated + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULV")) + videos = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + videos = [] of PlaylistVideo + end + next_continuation = nil + else + begin + videos, next_continuation = Channel::Tabs.get_60_livestreams( + channel, continuation: continuation, sort_by: sort_by + ) + rescue ex + return error_json(500, ex) + end end return JSON.build do |json| @@ -343,6 +391,59 @@ module Invidious::Routes::API::V1::Channels end end + def self.post(env) + locale = env.get("preferences").as(Preferences).locale + + env.response.content_type = "application/json" + id = env.params.url["id"].to_s + ucid = env.params.query["ucid"]? + + thin_mode = env.params.query["thin_mode"]? + thin_mode = thin_mode == "true" + + format = env.params.query["format"]? + format ||= "json" + + if ucid.nil? + response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}") + return error_json(400, "Invalid post ID") if response["error"]? + ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s + else + ucid = ucid.to_s + end + + begin + fetch_channel_community_post(ucid, id, locale, format, thin_mode) + rescue ex + return error_json(500, ex) + end + end + + def self.post_comments(env) + locale = env.get("preferences").as(Preferences).locale + + env.response.content_type = "application/json" + + id = env.params.url["id"] + + thin_mode = env.params.query["thin_mode"]? + thin_mode = thin_mode == "true" + + format = env.params.query["format"]? + format ||= "json" + + continuation = env.params.query["continuation"]? + + case continuation + when nil, "" + ucid = env.params.query["ucid"] + comments = Comments.fetch_community_post_comments(ucid, id) + else + comments = YoutubeAPI.browse(continuation: continuation) + end + return Comments.parse_youtube(id, comments, format, locale, thin_mode, is_post: true) + end + def self.channels(env) locale = env.get("preferences").as(Preferences).locale ucid = env.params.url["ucid"] diff --git a/src/invidious/routes/api/v1/feeds.cr b/src/invidious/routes/api/v1/feeds.cr index 41865f34..fea2993c 100644 --- a/src/invidious/routes/api/v1/feeds.cr +++ b/src/invidious/routes/api/v1/feeds.cr @@ -31,7 +31,7 @@ module Invidious::Routes::API::V1::Feeds if !CONFIG.popular_enabled error_message = {"error" => "Administrator has disabled this endpoint."}.to_json - haltf env, 400, error_message + haltf env, 403, error_message end JSON.build do |json| diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index e499f4d6..4f5b58da 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -6,6 +6,22 @@ module Invidious::Routes::API::V1::Misc if !CONFIG.statistics_enabled return {"software" => SOFTWARE}.to_json else + # Calculate playback success rate + if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]?) + tracker = tracker.as(Hash(String, Int64 | Float64)) + + if !tracker.empty? + total_requests = tracker["totalRequests"] + success_count = tracker["successfulRequests"] + + if total_requests.zero? + tracker["ratio"] = 1_i64 + else + tracker["ratio"] = (success_count / (total_requests)).round(2) + end + end + end + return Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json end end @@ -26,6 +42,9 @@ module Invidious::Routes::API::V1::Misc format = env.params.query["format"]? format ||= "json" + listen_param = env.params.query["listen"]? + listen = (listen_param == "true" || listen_param == "1") + if plid.starts_with? "RD" return env.redirect "/api/v1/mixes/#{plid}" end @@ -58,7 +77,9 @@ module Invidious::Routes::API::V1::Misc response = playlist.to_json(offset, video_id: video_id) json_response = JSON.parse(response) - if json_response["videos"].as_a[0]["index"] != offset + if json_response["videos"].as_a.empty? + json_response = JSON.parse(response) + elsif json_response["videos"].as_a[0]["index"] != offset offset = json_response["videos"].as_a[0]["index"].as_i lookback = offset < 50 ? offset : 50 response = playlist.to_json(offset - lookback) @@ -67,7 +88,7 @@ module Invidious::Routes::API::V1::Misc end if format == "html" - playlist_html = template_playlist(json_response) + playlist_html = template_playlist(json_response, listen) index, next_video = json_response["videos"].as_a.skip(1 + lookback).select { |video| !video["author"].as_s.empty? }[0]?.try { |v| {v["index"], v["videoId"]} } || {nil, nil} response = { @@ -93,6 +114,9 @@ module Invidious::Routes::API::V1::Misc format = env.params.query["format"]? format ||= "json" + listen_param = env.params.query["listen"]? + listen = (listen_param == "true" || listen_param == "1") + begin mix = fetch_mix(rdid, continuation, locale: locale) @@ -123,9 +147,7 @@ module Invidious::Routes::API::V1::Misc json.field "authorUrl", "/channel/#{video.ucid}" json.field "videoThumbnails" do - json.array do - Invidious::JSONify::APIv1.thumbnails(json, video.id) - end + Invidious::JSONify::APIv1.thumbnails(json, video.id) end json.field "index", video.index @@ -139,7 +161,7 @@ module Invidious::Routes::API::V1::Misc if format == "html" response = JSON.parse(response) - playlist_html = template_mix(response) + playlist_html = template_mix(response, listen) next_video = response["videos"].as_a.select { |video| !video["author"].as_s.empty? }[0]?.try &.["videoId"] response = { @@ -161,19 +183,24 @@ module Invidious::Routes::API::V1::Misc begin resolved_url = YoutubeAPI.resolve_url(url.as(String)) endpoint = resolved_url["endpoint"] - pageType = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || "" - if resolved_ucid = endpoint.dig?("watchEndpoint", "videoId") - elsif resolved_ucid = endpoint.dig?("browseEndpoint", "browseId") - elsif pageType == "WEB_PAGE_TYPE_UNKNOWN" + page_type = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || "" + if page_type == "WEB_PAGE_TYPE_UNKNOWN" return error_json(400, "Unknown url") end + + sub_endpoint = endpoint["watchEndpoint"]? || endpoint["browseEndpoint"]? || endpoint + params = sub_endpoint.try &.dig?("params") rescue ex return error_json(500, ex) end JSON.build do |json| json.object do - json.field "ucid", resolved_ucid.try &.as_s || "" - json.field "pageType", pageType + json.field "ucid", sub_endpoint["browseId"].as_s if sub_endpoint["browseId"]? + json.field "videoId", sub_endpoint["videoId"].as_s if sub_endpoint["videoId"]? + json.field "playlistId", sub_endpoint["playlistId"].as_s if sub_endpoint["playlistId"]? + json.field "startTimeSeconds", sub_endpoint["startTimeSeconds"].as_i if sub_endpoint["startTimeSeconds"]? + json.field "params", params.try &.as_s + json.field "pageType", page_type end end end diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index 9fb283c2..59a30745 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -31,12 +31,13 @@ module Invidious::Routes::API::V1::Search query = env.params.query["q"]? || "" begin - client = HTTP::Client.new("suggestqueries-clients6.youtube.com") - url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&xssi=t&gs_ri=youtube&ds=yt" + client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers: true) + url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt" response = client.get(url).body + client.close - body = JSON.parse(response[5..-1]).as_a + body = JSON.parse(response[19..-2]).as_a suggestions = body[1].as_a[0..-2] JSON.build do |json| diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 25e766d2..368304ac 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -1,3 +1,5 @@ +require "html" + module Invidious::Routes::API::V1::Videos def self.videos(env) locale = env.get("preferences").as(Preferences).locale @@ -89,9 +91,14 @@ module Invidious::Routes::API::V1::Videos if CONFIG.use_innertube_for_captions params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated) - initial_data = YoutubeAPI.get_transcript(params) - webvtt = Invidious::Videos::Transcript.convert_transcripts_to_vtt(initial_data, caption.language_code) + transcript = Invidious::Videos::Transcript.from_raw( + YoutubeAPI.get_transcript(params), + caption.language_code, + caption.auto_generated + ) + + webvtt = transcript.to_vtt else # Timedtext API handling url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target @@ -101,20 +108,17 @@ module Invidious::Routes::API::V1::Videos if caption.name.includes? "auto-generated" caption_xml = YT_POOL.client &.get(url).body + settings_field = { + "Kind" => "captions", + "Language" => "#{tlang || caption.language_code}", + } + if caption_xml.starts_with?("<?xml") webvtt = caption.timedtext_to_vtt(caption_xml, tlang) else caption_xml = XML.parse(caption_xml) - webvtt = String.build do |str| - str << <<-END_VTT - WEBVTT - Kind: captions - Language: #{tlang || caption.language_code} - - - END_VTT - + webvtt = WebVTT.build(settings_field) do |builder| caption_nodes = caption_xml.xpath_nodes("//transcript/text") caption_nodes.each_with_index do |node, i| start_time = node["start"].to_f.seconds @@ -127,9 +131,6 @@ module Invidious::Routes::API::V1::Videos end_time = start_time + duration end - start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}" - end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}" - text = HTML.unescape(node.content) text = text.gsub(/<font color="#[a-fA-F0-9]{6}">/, "") text = text.gsub(/<\/font>/, "") @@ -137,27 +138,26 @@ module Invidious::Routes::API::V1::Videos text = "<v #{md["name"]}>#{md["text"]}</v>" end - str << <<-END_CUE - #{start_time} --> #{end_time} - #{text} - - - END_CUE + builder.cue(start_time, end_time, text) end end end else - # Some captions have "align:[start/end]" and "position:[num]%" - # attributes. Those are causing issues with VideoJS, which is unable - # to properly align the captions on the video, so we remove them. - # - # See: https://github.com/iv-org/invidious/issues/2391 - webvtt = YT_POOL.client &.get("#{url}&format=vtt").body + uri = URI.parse(url) + query_params = uri.query_params + query_params["fmt"] = "vtt" + uri.query_params = query_params + webvtt = YT_POOL.client &.get(uri.request_target).body + if webvtt.starts_with?("<?xml") webvtt = caption.timedtext_to_vtt(webvtt) else - webvtt = YT_POOL.client &.get("#{url}&format=vtt").body - .gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1") + # Some captions have "align:[start/end]" and "position:[num]%" + # attributes. Those are causing issues with VideoJS, which is unable + # to properly align the captions on the video, so we remove them. + # + # See: https://github.com/iv-org/invidious/issues/2391 + webvtt = webvtt.gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1") end end end @@ -189,15 +189,14 @@ module Invidious::Routes::API::V1::Videos haltf env, 500 end - storyboards = video.storyboards - width = env.params.query["width"]? - height = env.params.query["height"]? + width = env.params.query["width"]?.try &.to_i + height = env.params.query["height"]?.try &.to_i if !width && !height response = JSON.build do |json| json.object do json.field "storyboards" do - Invidious::JSONify::APIv1.storyboards(json, id, storyboards) + Invidious::JSONify::APIv1.storyboards(json, id, video.storyboards) end end end @@ -207,43 +206,48 @@ module Invidious::Routes::API::V1::Videos env.response.content_type = "text/vtt" - storyboard = storyboards.select { |sb| width == "#{sb[:width]}" || height == "#{sb[:height]}" } + # Select a storyboard matching the user's provided width/height + storyboard = video.storyboards.select { |x| x.width == width || x.height == height } + haltf env, 404 if storyboard.empty? - if storyboard.empty? - haltf env, 404 - else - storyboard = storyboard[0] - end + # Alias variable, to make the code below esaier to read + sb = storyboard[0] - String.build do |str| - str << <<-END_VTT - WEBVTT - END_VTT + # Some base URL segments that we'll use to craft the final URLs + work_url = sb.proxied_url.dup + template_path = sb.proxied_url.path - start_time = 0.milliseconds - end_time = storyboard[:interval].milliseconds + # Initialize cue timing variables + # NOTE: videojs-vtt-thumbnails gets lost when the cue times don't overlap + # (i.e: if cue[n] end time is 1:06:25.000, cue[n+1] start time should be 1:06:25.000) + time_delta = sb.interval.milliseconds + start_time = 0.milliseconds + end_time = time_delta - storyboard[:storyboard_count].times do |i| - url = storyboard[:url] - authority = /(i\d?).ytimg.com/.match(url).not_nil![1]? - url = url.gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "") - url = "#{HOST_URL}/sb/#{authority}/#{url}" + # Build a VTT file for VideoJS-vtt plugin + vtt_file = WebVTT.build do |vtt| + sb.images_count.times do |i| + # Replace the variable component part of the path + work_url.path = template_path.sub("$M", i) - storyboard[:storyboard_height].times do |j| - storyboard[:storyboard_width].times do |k| - str << <<-END_CUE - #{start_time}.000 --> #{end_time}.000 - #{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]} + sb.rows.times do |j| + sb.columns.times do |k| + # The URL fragment represents the offset of the thumbnail inside the storyboard image + work_url.fragment = "xywh=#{sb.width * k},#{sb.height * j},#{sb.width - 2},#{sb.height}" + vtt.cue(start_time, end_time, work_url.to_s) - END_CUE - - start_time += storyboard[:interval].milliseconds - end_time += storyboard[:interval].milliseconds + start_time += time_delta + end_time += time_delta end end end end + + # videojs-vtt-thumbnails is not compliant to the VTT specification, it + # doesn't unescape the HTML entities, so we have to do it here: + # TODO: remove this when we migrate to VideoJS 8 + return HTML.unescape(vtt_file) end def self.annotations(env) @@ -264,7 +268,7 @@ module Invidious::Routes::API::V1::Videos if CONFIG.cache_annotations && (cached_annotation = Invidious::Database::Annotations.select(id)) annotations = cached_annotation.annotations else - index = CHARS_SAFE.index(id[0]).not_nil!.to_s.rjust(2, '0') + index = CHARS_SAFE.index!(id[0]).to_s.rjust(2, '0') # IA doesn't handle leading hyphens, # so we use https://archive.org/details/youtubeannotations_64 @@ -382,4 +386,47 @@ module Invidious::Routes::API::V1::Videos end end end + + def self.clips(env) + locale = env.get("preferences").as(Preferences).locale + + env.response.content_type = "application/json" + + clip_id = env.params.url["id"] + region = env.params.query["region"]? + proxy = {"1", "true"}.any? &.== env.params.query["local"]? + + response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}") + return error_json(400, "Invalid clip ID") if response["error"]? + + video_id = response.dig?("endpoint", "watchEndpoint", "videoId").try &.as_s + return error_json(400, "Invalid clip ID") if video_id.nil? + + start_time = nil + end_time = nil + clip_title = nil + + if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s + start_time, end_time, clip_title = parse_clip_parameters(params) + end + + begin + video = get_video(video_id, region: region) + rescue ex : NotFoundException + return error_json(404, ex) + rescue ex + return error_json(500, ex) + end + + return JSON.build do |json| + json.object do + json.field "startTime", start_time + json.field "endTime", end_time + json.field "clipTitle", clip_title + json.field "video" do + Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy) + end + end + end + end end diff --git a/src/invidious/routes/before_all.cr b/src/invidious/routes/before_all.cr index 396840a4..5695dee9 100644 --- a/src/invidious/routes/before_all.cr +++ b/src/invidious/routes/before_all.cr @@ -30,7 +30,7 @@ module Invidious::Routes::BeforeAll # Only allow the pages at /embed/* to be embedded if env.request.resource.starts_with?("/embed") - frame_ancestors = "'self' http: https:" + frame_ancestors = "'self' file: http: https:" else frame_ancestors = "'none'" end diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 9892ae2a..7d634cbb 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -1,6 +1,12 @@ {% skip_file if flag?(:api_only) %} module Invidious::Routes::Channels + # Redirection for unsupported routes ("tabs") + def self.redirect_home(env) + ucid = env.params.url["ucid"] + return env.redirect "/channel/#{URI.encode_www_form(ucid)}" + end + def self.home(env) self.videos(env) end @@ -14,10 +20,11 @@ module Invidious::Routes::Channels sort_by = env.params.query["sort_by"]?.try &.downcase if channel.auto_generated + sort_by ||= "last" sort_options = {"last", "oldest", "newest"} items, next_continuation = fetch_channel_playlists( - channel.ucid, channel.author, continuation, (sort_by || "last") + channel.ucid, channel.author, continuation, sort_by ) items.uniq! do |item| @@ -30,12 +37,26 @@ module Invidious::Routes::Channels items = items.select(SearchPlaylist) items.each(&.author = "") else - sort_options = {"newest", "oldest", "popular"} - # Fetch items and continuation token - items, next_continuation = Channel::Tabs.get_videos( - channel, continuation: continuation, sort_by: (sort_by || "newest") - ) + if channel.is_age_gated + sort_by = "" + sort_options = [] of String + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULF")) + items = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + items = [] of PlaylistVideo + end + next_continuation = nil + else + sort_by ||= "newest" + sort_options = {"newest", "oldest", "popular"} + + items, next_continuation = Channel::Tabs.get_60_videos( + channel, continuation: continuation, sort_by: sort_by + ) + end end selected_tab = Frontend::ChannelPage::TabsAvailable::Videos @@ -52,14 +73,26 @@ module Invidious::Routes::Channels return env.redirect "/channel/#{channel.ucid}" end - # TODO: support sort option for shorts - sort_by = "" - sort_options = [] of String + if channel.is_age_gated + sort_by = "" + sort_options = [] of String + begin + playlist = get_playlist(channel.ucid.sub("UC", "UUSH")) + items = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + items = [] of PlaylistVideo + end + next_continuation = nil + else + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" + sort_options = {"newest", "oldest", "popular"} - # Fetch items and continuation token - items, next_continuation = Channel::Tabs.get_shorts( - channel, continuation: continuation - ) + # Fetch items and continuation token + items, next_continuation = Channel::Tabs.get_shorts( + channel, continuation: continuation, sort_by: sort_by + ) + end selected_tab = Frontend::ChannelPage::TabsAvailable::Shorts templated "channel" @@ -75,14 +108,26 @@ module Invidious::Routes::Channels return env.redirect "/channel/#{channel.ucid}" end - # TODO: support sort option for livestreams - sort_by = "" - sort_options = [] of String + if channel.is_age_gated + sort_by = "" + sort_options = [] of String + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULV")) + items = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + items = [] of PlaylistVideo + end + next_continuation = nil + else + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" + sort_options = {"newest", "oldest", "popular"} - # Fetch items and continuation token - items, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation - ) + # Fetch items and continuation token + items, next_continuation = Channel::Tabs.get_60_livestreams( + channel, continuation: continuation, sort_by: sort_by + ) + end selected_tab = Frontend::ChannelPage::TabsAvailable::Streams templated "channel" @@ -159,6 +204,11 @@ module Invidious::Routes::Channels end locale, user, subscriptions, continuation, ucid, channel = data + # redirect to post page + if lb = env.params.query["lb"]? + env.redirect "/post/#{URI.encode_www_form(lb)}?ucid=#{URI.encode_www_form(ucid)}" + end + thin_mode = env.params.query["thin_mode"]? || env.get("preferences").as(Preferences).thin_mode thin_mode = thin_mode == "true" @@ -187,6 +237,44 @@ module Invidious::Routes::Channels templated "community" end + def self.post(env) + # /post/{postId} + id = env.params.url["id"] + ucid = env.params.query["ucid"]? + + prefs = env.get("preferences").as(Preferences) + + locale = prefs.locale + + thin_mode = env.params.query["thin_mode"]? || prefs.thin_mode + thin_mode = thin_mode == "true" + + nojs = env.params.query["nojs"]? + + nojs ||= "0" + nojs = nojs == "1" + + if !ucid.nil? + ucid = ucid.to_s + post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode) + else + # resolve the url to get the author's UCID + response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}") + return error_template(400, "Invalid post ID") if response["error"]? + + ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s + post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode) + end + + post_response = JSON.parse(post_response) + + if nojs + comments = Comments.fetch_community_post_comments(ucid, id) + comment_html = JSON.parse(Comments.parse_youtube(id, comments, "html", locale, thin_mode, is_post: true))["contentHtml"] + end + templated "post" + end + def self.channels(env) data = self.fetch_basic_information(env) return data if !data.is_a?(Tuple) @@ -217,6 +305,11 @@ module Invidious::Routes::Channels env.redirect "/channel/#{ucid}" end + private KNOWN_TABS = { + "home", "videos", "shorts", "streams", "podcasts", + "releases", "playlists", "community", "channels", "about", + } + # Redirects brand url channels to a normal /channel/:ucid route def self.brand_redirect(env) locale = env.get("preferences").as(Preferences).locale @@ -227,7 +320,10 @@ module Invidious::Routes::Channels yt_url_params = URI::Params.encode(env.params.query.to_h.select(["a", "u", "user"])) # Retrieves URL params that only Invidious uses - invidious_url_params = URI::Params.encode(env.params.query.to_h.select!(["a", "u", "user"])) + invidious_url_params = env.params.query.dup + invidious_url_params.delete_all("a") + invidious_url_params.delete_all("u") + invidious_url_params.delete_all("user") begin resolved_url = YoutubeAPI.resolve_url("https://youtube.com#{env.request.path}#{yt_url_params.size > 0 ? "?#{yt_url_params}" : ""}") @@ -236,14 +332,17 @@ module Invidious::Routes::Channels return error_template(404, translate(locale, "This channel does not exist.")) end - selected_tab = env.request.path.split("/")[-1] - if {"home", "videos", "shorts", "streams", "playlists", "community", "channels", "about"}.includes? selected_tab + selected_tab = env.params.url["tab"]? + + if KNOWN_TABS.includes? selected_tab url = "/channel/#{ucid}/#{selected_tab}" else url = "/channel/#{ucid}" end - env.redirect url + url += "?#{invidious_url_params}" if !invidious_url_params.empty? + + return env.redirect url end # Handles redirects for the /profile endpoint diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 266f7ba4..00f24159 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -157,10 +157,12 @@ module Invidious::Routes::Embed adaptive_fmts = video.adaptive_fmts if params.local - fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) } - adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) } + fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) } end + # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback + adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) } + video_streams = video.video_streams audio_streams = video.audio_streams diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 40bca008..82c04994 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -192,11 +192,10 @@ module Invidious::Routes::Feeds views: views, description_html: description_html, length_seconds: 0, - live_now: false, - paid: false, - premium: false, premiere_timestamp: nil, author_verified: false, + author_thumbnail: nil, + badges: VideoBadges::None, }) end @@ -407,14 +406,23 @@ module Invidious::Routes::Feeds end spawn do - rss = XML.parse_html(body) - rss.xpath_nodes("//feed/entry").each do |entry| - id = entry.xpath_node("videoid").not_nil!.content - author = entry.xpath_node("author/name").not_nil!.content - published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) - updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) - - video = get_video(id, force_refresh: true) + # TODO: unify this with the other almost identical looking parts in this and channels.cr somehow? + namespaces = { + "yt" => "http://www.youtube.com/xml/schemas/2015", + "default" => "http://www.w3.org/2005/Atom", + } + rss = XML.parse(body) + rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry| + id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content + author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content + published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content) + updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content) + + begin + video = get_video(id, force_refresh: true) + rescue + next # skip this video since it raised an exception (e.g. it is a scheduled live event) + end if CONFIG.enable_user_notifications # Deliver notifications to `/api/v1/auth/notifications` diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index b6a2e110..639697db 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -11,29 +11,9 @@ module Invidious::Routes::Images end end - # We're encapsulating this into a proc in order to easily reuse this - # portion of the code for each request block below. - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - env.response.headers.delete("Transfer-Encoding") - return - end - - proxy_file(response, env) - } - begin - HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| - return request_proc.call(resp) + GGPHT_POOL.client &.get(url, headers) do |resp| + return self.proxy_image(env, resp) end rescue ex end @@ -61,27 +41,10 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Connection"] = "close" - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - return env.response.headers.delete("Transfer-Encoding") - end - - proxy_file(response, env) - } - begin - HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| - return request_proc.call(resp) + get_ytimg_pool(authority).client &.get(url, headers) do |resp| + env.response.headers["Connection"] = "close" + return self.proxy_image(env, resp) end rescue ex end @@ -101,26 +64,9 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - return env.response.headers.delete("Transfer-Encoding") - end - - proxy_file(response, env) - } - begin - HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| - return request_proc.call(resp) + get_ytimg_pool("i9").client &.get(url, headers) do |resp| + return self.proxy_image(env, resp) end rescue ex end @@ -165,8 +111,7 @@ module Invidious::Routes::Images if name == "maxres.jpg" build_thumbnails(id).each do |thumb| thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" - # This can likely be optimized into a (small) pool sometime in the future. - if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 + if get_ytimg_pool("i9").client &.head(thumbnail_resource_path, headers).status_code == 200 name = thumb[:url] + ".jpg" break end @@ -181,29 +126,28 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end + begin + get_ytimg_pool("i").client &.get(url, headers) do |resp| + return self.proxy_image(env, resp) end + rescue ex + end + end - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - return env.response.headers.delete("Transfer-Encoding") + private def self.proxy_image(env, response) + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value end + end - proxy_file(response, env) - } + env.response.headers["Access-Control-Allow-Origin"] = "*" - begin - # This can likely be optimized into a (small) pool sometime in the future. - HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end - rescue ex + if response.status_code >= 300 + return env.response.headers.delete("Transfer-Encoding") end + + return proxy_file(response, env) end end diff --git a/src/invidious/routes/misc.cr b/src/invidious/routes/misc.cr index d6bd9571..8b620d63 100644 --- a/src/invidious/routes/misc.cr +++ b/src/invidious/routes/misc.cr @@ -40,7 +40,16 @@ module Invidious::Routes::Misc def self.cross_instance_redirect(env) referer = get_referer(env) - instance_url = fetch_random_instance + + instance_list = Invidious::Jobs::InstanceListRefreshJob::INSTANCES["INSTANCES"] + if instance_list.empty? + instance_url = "redirect.invidious.io" + else + # Sample returns an array + # Instances are packaged as {region, domain} in the instance list + instance_url = instance_list.sample(1)[0][1] + end + env.redirect "https://#{instance_url}#{referer}" end end diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index 9c6843e9..f2213da4 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -304,23 +304,6 @@ module Invidious::Routes::Playlists end end - if env.params.query["action_create_playlist"]? - action = "action_create_playlist" - elsif env.params.query["action_delete_playlist"]? - action = "action_delete_playlist" - elsif env.params.query["action_edit_playlist"]? - action = "action_edit_playlist" - elsif env.params.query["action_add_video"]? - action = "action_add_video" - video_id = env.params.query["video_id"] - elsif env.params.query["action_remove_video"]? - action = "action_remove_video" - elsif env.params.query["action_move_video_before"]? - action = "action_move_video_before" - else - return env.redirect referer - end - begin playlist_id = env.params.query["playlist_id"] playlist = get_playlist(playlist_id).as(InvidiousPlaylist) @@ -335,12 +318,8 @@ module Invidious::Routes::Playlists end end - email = user.email - - case action - when "action_edit_playlist" - # TODO: Playlist stub - when "action_add_video" + case action = env.params.query["action"]? + when "add_video" if playlist.index.size >= CONFIG.playlist_length_limit if redirect return error_template(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos") @@ -377,12 +356,14 @@ module Invidious::Routes::Playlists Invidious::Database::PlaylistVideos.insert(playlist_video) Invidious::Database::Playlists.update_video_added(playlist_id, playlist_video.index) - when "action_remove_video" + when "remove_video" index = env.params.query["set_video_id"] Invidious::Database::PlaylistVideos.delete(index) Invidious::Database::Playlists.update_video_removed(playlist_id, index) - when "action_move_video_before" + when "move_video_before" # TODO: Playlist stub + when nil + return error_json(400, "Missing action") else return error_json(400, "Unsupported action #{action}") end diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index abe0f34e..39ca77c0 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -27,6 +27,10 @@ module Invidious::Routes::PreferencesRoute annotations_subscribed ||= "off" annotations_subscribed = annotations_subscribed == "on" + preload = env.params.body["preload"]?.try &.as(String) + preload ||= "off" + preload = preload == "on" + autoplay = env.params.body["autoplay"]?.try &.as(String) autoplay ||= "off" autoplay = autoplay == "on" @@ -144,6 +148,7 @@ module Invidious::Routes::PreferencesRoute preferences = Preferences.from_json({ annotations: annotations, annotations_subscribed: annotations_subscribed, + preload: preload, autoplay: autoplay, captions: captions, comments: comments, @@ -214,7 +219,7 @@ module Invidious::Routes::PreferencesRoute statistics_enabled ||= "off" CONFIG.statistics_enabled = statistics_enabled == "on" - CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.try &.as(String) + CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.presence File.write("config/config.yml", CONFIG.to_yaml) end @@ -319,6 +324,15 @@ module Invidious::Routes::PreferencesRoute response: error_template(415, "Invalid playlist file uploaded") ) end + when "import_youtube_wh" + filename = part.filename || "" + success = Invidious::User::Import.from_youtube_wh(user, body, filename, type) + + if !success + haltf(env, status_code: 415, + response: error_template(415, "Invalid watch history file uploaded") + ) + end when "import_freetube" Invidious::User::Import.from_freetube(user, body) when "import_newpipe_subscriptions" diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index 5be33533..44970922 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -51,6 +51,12 @@ module Invidious::Routes::Search else user = env.get? "user" + # An URL was copy/pasted in the search box. + # Redirect the user to the appropriate page. + if query.url? + return env.redirect UrlSanitizer.process(query.text).to_s + end + begin items = query.process rescue ex : ChannelSearchException diff --git a/src/invidious/routes/subscriptions.cr b/src/invidious/routes/subscriptions.cr index 7f9ec592..1de655d2 100644 --- a/src/invidious/routes/subscriptions.cr +++ b/src/invidious/routes/subscriptions.cr @@ -32,24 +32,16 @@ module Invidious::Routes::Subscriptions end end - if env.params.query["action_create_subscription_to_channel"]?.try &.to_i?.try &.== 1 - action = "action_create_subscription_to_channel" - elsif env.params.query["action_remove_subscriptions"]?.try &.to_i?.try &.== 1 - action = "action_remove_subscriptions" - else - return env.redirect referer - end - channel_id = env.params.query["c"]? channel_id ||= "" - case action - when "action_create_subscription_to_channel" + case action = env.params.query["action"]? + when "create_subscription_to_channel" if !user.subscriptions.includes? channel_id get_channel(channel_id) Invidious::Database::Users.subscribe_channel(user, channel_id) end - when "action_remove_subscriptions" + when "remove_subscriptions" Invidious::Database::Users.unsubscribe_channel(user, channel_id) else return error_json(400, "Unsupported action #{action}") diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 9641e01a..a8f9f665 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -42,7 +42,7 @@ module Invidious::Routes::VideoPlayback headers["Range"] = "bytes=#{range_for_head}" end - client = make_client(URI.parse(host), region) + client = make_client(URI.parse(host), region, force_resolve: true) response = HTTP::Client::Response.new(500) error = "" 5.times do @@ -57,7 +57,7 @@ module Invidious::Routes::VideoPlayback if new_host != host host = new_host client.close - client = make_client(URI.parse(new_host), region) + client = make_client(URI.parse(new_host), region, force_resolve: true) end url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" @@ -71,7 +71,7 @@ module Invidious::Routes::VideoPlayback fvip = "3" host = "https://r#{fvip}---#{mn}.googlevideo.com" - client = make_client(URI.parse(host), region) + client = make_client(URI.parse(host), region, force_resolve: true) rescue ex error = ex.message end @@ -80,9 +80,14 @@ module Invidious::Routes::VideoPlayback # Remove the Range header added previously. headers.delete("Range") if range_header.nil? + playback_statistics = get_playback_statistic() + playback_statistics["totalRequests"] += 1 + if response.status_code >= 400 env.response.content_type = "text/plain" haltf env, response.status_code + else + playback_statistics["successfulRequests"] += 1 end if url.includes? "&file=seg.ts" @@ -126,7 +131,7 @@ module Invidious::Routes::VideoPlayback end # TODO: Record bytes written so we can restart after a chunk fails - while true + loop do if !range_end && content_length range_end = content_length end @@ -159,10 +164,13 @@ module Invidious::Routes::VideoPlayback env.response.headers["Access-Control-Allow-Origin"] = "*" if location = resp.headers["Location"]? - location = URI.parse(location) - location = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" + url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region) + + if title = query_params["title"]? + url = "#{url}&title=#{URI.encode_www_form(title)}" + end - env.redirect location + env.redirect url break end @@ -191,7 +199,7 @@ module Invidious::Routes::VideoPlayback break else client.close - client = make_client(URI.parse(host), region) + client = make_client(URI.parse(host), region, force_resolve: true) end end diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index e5cf3716..1f384546 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -30,14 +30,6 @@ module Invidious::Routes::Watch return env.redirect "/" end - embed_link = "/embed/#{id}" - if env.params.query.size > 1 - embed_params = HTTP::Params.parse(env.params.query.to_s) - embed_params.delete_all("v") - embed_link += "?" - embed_link += embed_params.to_s - end - plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") continuation = process_continuation(env.params.query, plid, id) @@ -129,10 +121,12 @@ module Invidious::Routes::Watch adaptive_fmts = video.adaptive_fmts if params.local - fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) } - adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) } + fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) } end + # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback + adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) } + video_streams = video.video_streams audio_streams = video.audio_streams @@ -249,18 +243,10 @@ module Invidious::Routes::Watch end end - if env.params.query["action_mark_watched"]? - action = "action_mark_watched" - elsif env.params.query["action_mark_unwatched"]? - action = "action_mark_unwatched" - else - return env.redirect referer - end - - case action - when "action_mark_watched" + case action = env.params.query["action"]? + when "mark_watched" Invidious::Database::Users.mark_watched(user, id) - when "action_mark_unwatched" + when "mark_unwatched" Invidious::Database::Users.mark_unwatched(user, id) else return error_json(400, "Unsupported action #{action}") @@ -283,6 +269,12 @@ module Invidious::Routes::Watch return error_template(400, "Invalid clip ID") if response["error"]? if video_id = response.dig?("endpoint", "watchEndpoint", "videoId") + if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s + start_time, end_time, _ = parse_clip_parameters(params) + env.params.query["start"] = start_time.to_s if start_time != nil + env.params.query["end"] = end_time.to_s if end_time != nil + end + return env.redirect "/watch?v=#{video_id}&#{env.params.query}" else return error_template(404, "The requested clip doesn't exist") diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index 9c43171c..9009062f 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -124,28 +124,42 @@ module Invidious::Routing get "/channel/:ucid/community", Routes::Channels, :community get "/channel/:ucid/channels", Routes::Channels, :channels get "/channel/:ucid/about", Routes::Channels, :about + get "/channel/:ucid/live", Routes::Channels, :live get "/user/:user/live", Routes::Channels, :live get "/c/:user/live", Routes::Channels, :live + get "/post/:id", Routes::Channels, :post - {"", "/videos", "/shorts", "/streams", "/playlists", "/community", "/about"}.each do |path| - # /c/LinusTechTips - get "/c/:user#{path}", Routes::Channels, :brand_redirect - # /user/linustechtips | Not always the same as /c/ - get "/user/:user#{path}", Routes::Channels, :brand_redirect - # /@LinusTechTips | Handle - get "/@:user#{path}", Routes::Channels, :brand_redirect - # /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow - get "/attribution_link#{path}", Routes::Channels, :brand_redirect - # /profile?user=linustechtips - get "/profile/#{path}", Routes::Channels, :profile - end + # Channel catch-all, to redirect future routes to the channel's home + # NOTE: defined last in order to be processed after the other routes + get "/channel/:ucid/*", Routes::Channels, :redirect_home + + # /c/LinusTechTips + get "/c/:user", Routes::Channels, :brand_redirect + get "/c/:user/:tab", Routes::Channels, :brand_redirect + + # /user/linustechtips (Not always the same as /c/) + get "/user/:user", Routes::Channels, :brand_redirect + get "/user/:user/:tab", Routes::Channels, :brand_redirect + + # /@LinusTechTips (Handle) + get "/@:user", Routes::Channels, :brand_redirect + get "/@:user/:tab", Routes::Channels, :brand_redirect + + # /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow + get "/attribution_link", Routes::Channels, :brand_redirect + get "/attribution_link/:tab", Routes::Channels, :brand_redirect + + # /profile?user=linustechtips + get "/profile", Routes::Channels, :profile + get "/profile/*", Routes::Channels, :profile end def register_watch_routes get "/watch", Routes::Watch, :handle post "/watch_ajax", Routes::Watch, :mark_watched get "/watch/:id", Routes::Watch, :redirect + get "/live/:id", Routes::Watch, :redirect get "/shorts/:id", Routes::Watch, :redirect get "/clip/:clip", Routes::Watch, :clip get "/w/:id", Routes::Watch, :redirect @@ -221,6 +235,7 @@ module Invidious::Routing get "/api/v1/captions/:id", {{namespace}}::Videos, :captions get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations get "/api/v1/comments/:id", {{namespace}}::Videos, :comments + get "/api/v1/clips/:id", {{namespace}}::Videos, :clips # Feeds get "/api/v1/trending", {{namespace}}::Feeds, :trending @@ -228,17 +243,20 @@ module Invidious::Routing # Channels get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home + get "/api/v1/channels/:ucid/latest", {{namespace}}::Channels, :latest + get "/api/v1/channels/:ucid/videos", {{namespace}}::Channels, :videos get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams get "/api/v1/channels/:ucid/podcasts", {{namespace}}::Channels, :podcasts get "/api/v1/channels/:ucid/releases", {{namespace}}::Channels, :releases - + get "/api/v1/channels/:ucid/playlists", {{namespace}}::Channels, :playlists + get "/api/v1/channels/:ucid/community", {{namespace}}::Channels, :community get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels + get "/api/v1/channels/:ucid/search", {{namespace}}::Channels, :search - {% for route in {"videos", "latest", "playlists", "community", "search"} %} - get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}} - get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}} - {% end %} + # Posts + get "/api/v1/post/:id", {{namespace}}::Channels, :post + get "/api/v1/post/:id/comments", {{namespace}}::Channels, :post_comments # 301 redirects to new /api/v1/channels/community/:ucid and /:ucid/community get "/api/v1/channels/comments/:ucid", {{namespace}}::Channels, :channel_comments_redirect @@ -249,12 +267,8 @@ module Invidious::Routing get "/api/v1/search/suggestions", {{namespace}}::Search, :search_suggestions get "/api/v1/hashtag/:hashtag", {{namespace}}::Search, :hashtag - # Authenticated - # The notification APIs cannot be extracted yet! They require the *local* notifications constant defined in invidious.cr - # - # Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications - # Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + # Authenticated get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr index c2b5c758..bc2715cf 100644 --- a/src/invidious/search/filters.cr +++ b/src/invidious/search/filters.cr @@ -75,7 +75,7 @@ module Invidious::Search @type : Type = Type::All, @duration : Duration = Duration::None, @features : Features = Features::None, - @sort : Sort = Sort::Relevance + @sort : Sort = Sort::Relevance, ) end @@ -300,9 +300,9 @@ module Invidious::Search object["9:varint"] = ((page - 1) * 20).to_i64 end - # If the object is empty, return an empty string, - # otherwise encode to protobuf then to base64 - return "" if object.empty? + # Prevent censoring of self harm topics + # See https://github.com/iv-org/invidious/issues/4398 + object["30:varint"] = 1.to_i64 return object .try { |i| Protodec::Any.cast_json(i) } diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr index e38845d9..94a92e23 100644 --- a/src/invidious/search/query.cr +++ b/src/invidious/search/query.cr @@ -20,6 +20,9 @@ module Invidious::Search property region : String? property channel : String = "" + # Flag that indicates if the smart search features have been disabled. + @inhibit_ssf : Bool = false + # Return true if @raw_query is either `nil` or empty private def empty_raw_query? return @raw_query.empty? @@ -44,14 +47,22 @@ module Invidious::Search def initialize( params : HTTP::Params, @type : Type = Type::Regular, - @region : String? = nil + @region : String? = nil, ) # Get the raw search query string (common to all search types). In # Regular search mode, also look for the `search_query` URL parameter - if @type.regular? - @raw_query = params["q"]? || params["search_query"]? || "" - else - @raw_query = params["q"]? || "" + _raw_query = params["q"]? + _raw_query ||= params["search_query"]? if @type.regular? + _raw_query ||= "" + + # Remove surrounding whitespaces. Mostly useful for copy/pasted URLs. + @raw_query = _raw_query.strip + + # Check for smart features (ex: URL search) inhibitor (backslash). + # If inhibitor is present, remove it. + if @raw_query.starts_with?('\\') + @inhibit_ssf = true + @raw_query = @raw_query[1..] end # Get the page number (also common to all search types) @@ -85,7 +96,7 @@ module Invidious::Search @filters = Filters.from_iv_params(params) @channel = params["channel"]? || "" - if @filters.default? && @raw_query.includes?(':') + if @filters.default? && @raw_query.index(/\w:\w/) # Parse legacy filters from query @filters, @channel, @query, subs = Filters.from_legacy_filters(@raw_query) else @@ -136,5 +147,22 @@ module Invidious::Search return params end + + # Checks if the query is a standalone URL + def url? : Bool + # If the smart features have been inhibited, don't go further. + return false if @inhibit_ssf + + # Only supported in regular search mode + return false if !@type.regular? + + # If filters are present, that's a regular search + return false if !@filters.default? + + # Simple heuristics: domain name + return @raw_query.starts_with?( + /(https?:\/\/)?(www\.)?(m\.)?youtu(\.be|be\.com)\// + ) + end end end diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index 2d9f8a83..107d148d 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr @@ -22,12 +22,14 @@ def fetch_trending(trending_type, region, locale) extracted = [] of SearchItem + deduplicate = items.size > 1 + items.each do |itm| if itm.is_a?(Category) # Ignore the smaller categories, as they generally contain a sponsored # channel, which brings a lot of noise on the trending page. # See: https://github.com/iv-org/invidious/issues/2989 - next if itm.contents.size < 24 + next if (itm.contents.size < 24 && deduplicate) extracted.concat extract_category(itm) else diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 86d0ce6e..007eb666 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -115,7 +115,7 @@ struct Invidious::User playlists.each do |item| title = item["title"]?.try &.as_s?.try &.delete("<>") description = item["description"]?.try &.as_s?.try &.delete("\r") - privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy } + privacy = item["privacy"]?.try &.as_s?.try { |raw_pl_privacy_state| PlaylistPrivacy.parse? raw_pl_privacy_state } next if !title next if !description @@ -124,7 +124,7 @@ struct Invidious::User playlist = create_playlist(title, privacy, user) Invidious::Database::Playlists.update_description(playlist.id, description) - videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| + item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| if idx > CONFIG.playlist_length_limit raise InfoException.new("Playlist cannot have more than #{CONFIG.playlist_length_limit} videos") end @@ -161,7 +161,7 @@ struct Invidious::User # Youtube # ------------------- - private def is_opml?(mimetype : String, extension : String) + private def opml?(mimetype : String, extension : String) opml_mimetypes = [ "application/xml", "text/xml", @@ -179,10 +179,10 @@ struct Invidious::User def from_youtube(user : User, body : String, filename : String, type : String) : Bool extension = filename.split(".").last - if is_opml?(type, extension) + if opml?(type, extension) subscriptions = XML.parse(body) user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| - channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] + channel["xmlUrl"].match!(/UC[a-zA-Z0-9_-]{22}/)[0] end elsif extension == "json" || type == "application/json" subscriptions = JSON.parse(body) @@ -218,6 +218,26 @@ struct Invidious::User end end + def from_youtube_wh(user : User, body : String, filename : String, type : String) : Bool + extension = filename.split(".").last + + if extension == "json" || type == "application/json" + data = JSON.parse(body) + watched = data.as_a.compact_map do |item| + next unless url = item["titleUrl"]? + next unless match = url.as_s.match(/\?v=(?<video_id>[a-zA-Z0-9_-]+)$/) + match["video_id"] + end + watched.reverse! # YouTube have newest first + user.watched += watched + user.watched.uniq! + Invidious::Database::Users.update_watch_history(user) + return true + else + return false + end + end + # ------------------- # Freetube # ------------------- @@ -228,8 +248,12 @@ struct Invidious::User subs = matches.map(&.["channel_id"]) if subs.empty? - data = JSON.parse(body)["subscriptions"] - subs = data.as_a.map(&.["id"].as_s) + profiles = body.split('\n', remove_empty: true) + profiles.each do |profile| + if data = JSON.parse(profile)["subscriptions"]? + subs += data.as_a.map(&.["id"].as_s) + end + end end user.subscriptions += subs @@ -266,42 +290,39 @@ struct Invidious::User end def from_newpipe(user : User, body : String) : Bool - io = IO::Memory.new(body) - - Compress::Zip::File.open(io) do |file| - file.entries.each do |entry| - entry.open do |file_io| - # Ensure max size of 4MB - io_sized = IO::Sized.new(file_io, 0x400000) - - next if entry.filename != "newpipe.db" - - tempfile = File.tempfile(".db") - - begin - File.write(tempfile.path, io_sized.gets_to_end) - rescue - return false - end - - db = DB.open("sqlite3://" + tempfile.path) + Compress::Zip::File.open(IO::Memory.new(body), true) do |file| + entry = file.entries.find { |file_entry| file_entry.filename == "newpipe.db" } + return false if entry.nil? + entry.open do |file_io| + # Ensure max size of 4MB + io_sized = IO::Sized.new(file_io, 0x400000) - user.watched += db.query_all("SELECT url FROM streams", as: String) - .map(&.lchop("https://www.youtube.com/watch?v=")) + begin + temp = File.tempfile(".db") do |tempfile| + begin + File.write(tempfile.path, io_sized.gets_to_end) + rescue + return false + end - user.watched.uniq! - Invidious::Database::Users.update_watch_history(user) + DB.open("sqlite3://" + tempfile.path) do |db| + user.watched += db.query_all("SELECT url FROM streams", as: String) + .map(&.lchop("https://www.youtube.com/watch?v=")) - user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String) - .map(&.lchop("https://www.youtube.com/channel/")) + user.watched.uniq! + Invidious::Database::Users.update_watch_history(user) - user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions) + user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String) + .map(&.lchop("https://www.youtube.com/channel/")) - Invidious::Database::Users.update_subscriptions(user) + user.subscriptions.uniq! + user.subscriptions = get_batch_channels(user.subscriptions) - db.close - tempfile.delete + Invidious::Database::Users.update_subscriptions(user) + end + end + ensure + temp.delete if !temp.nil? end end end diff --git a/src/invidious/user/preferences.cr b/src/invidious/user/preferences.cr index b3059403..0a8525f3 100644 --- a/src/invidious/user/preferences.cr +++ b/src/invidious/user/preferences.cr @@ -4,6 +4,7 @@ struct Preferences property annotations : Bool = CONFIG.default_user_preferences.annotations property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed + property preload : Bool = CONFIG.default_user_preferences.preload property autoplay : Bool = CONFIG.default_user_preferences.autoplay property automatic_instance_redirect : Bool = CONFIG.default_user_preferences.automatic_instance_redirect diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 9fbd1374..962f87bd 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -27,12 +27,6 @@ struct Video @captions = [] of Invidious::Videos::Captions::Metadata @[DB::Field(ignore: true)] - property adaptive_fmts : Array(Hash(String, JSON::Any))? - - @[DB::Field(ignore: true)] - property fmt_stream : Array(Hash(String, JSON::Any))? - - @[DB::Field(ignore: true)] property description : String? module JSONConverter @@ -82,6 +76,10 @@ struct Video return (self.video_type == VideoType::Livestream) end + def post_live_dvr + return info["isPostLiveDvr"].as_bool + end + def premiere_timestamp : Time? info .dig?("microformat", "playerMicroformatRenderer", "liveBroadcastDetails", "startTimestamp") @@ -94,45 +92,24 @@ struct Video # Methods for parsing streaming data - def fmt_stream - return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream - - fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any) - fmt_stream.each do |fmt| - if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) } - s.each do |k, v| - fmt[k] = JSON::Any.new(v) - end - fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}") - end - - fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}") - fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]? + def fmt_stream : Array(Hash(String, JSON::Any)) + if formats = info.dig?("streamingData", "formats") + return formats + .as_a.map(&.as_h) + .sort_by! { |f| f["width"]?.try &.as_i || 0 } + else + return [] of Hash(String, JSON::Any) end - - fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 } - @fmt_stream = fmt_stream - return @fmt_stream.as(Array(Hash(String, JSON::Any))) end - def adaptive_fmts - return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts - fmt_stream = info["streamingData"]?.try &.["adaptiveFormats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any) - fmt_stream.each do |fmt| - if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) } - s.each do |k, v| - fmt[k] = JSON::Any.new(v) - end - fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}") - end - - fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}") - fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]? + def adaptive_fmts : Array(Hash(String, JSON::Any)) + if formats = info.dig?("streamingData", "adaptiveFormats") + return formats + .as_a.map(&.as_h) + .sort_by! { |f| f["width"]?.try &.as_i || f["audioTrack"]?.try { |a| a["audioIsDefault"]?.try { |v| v.as_bool ? -1 : 0 } } || 0 } + else + return [] of Hash(String, JSON::Any) end - - fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 } - @adaptive_fmts = fmt_stream - return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) end def video_streams @@ -146,65 +123,8 @@ struct Video # Misc. methods def storyboards - storyboards = info.dig?("storyboards", "playerStoryboardSpecRenderer", "spec") - .try &.as_s.split("|") - - if !storyboards - if storyboard = info.dig?("storyboards", "playerLiveStoryboardSpecRenderer", "spec").try &.as_s - return [{ - url: storyboard.split("#")[0], - width: 106, - height: 60, - count: -1, - interval: 5000, - storyboard_width: 3, - storyboard_height: 3, - storyboard_count: -1, - }] - end - end - - items = [] of NamedTuple( - url: String, - width: Int32, - height: Int32, - count: Int32, - interval: Int32, - storyboard_width: Int32, - storyboard_height: Int32, - storyboard_count: Int32) - - return items if !storyboards - - url = URI.parse(storyboards.shift) - params = HTTP::Params.parse(url.query || "") - - storyboards.each_with_index do |sb, i| - width, height, count, storyboard_width, storyboard_height, interval, _, sigh = sb.split("#") - params["sigh"] = sigh - url.query = params.to_s - - width = width.to_i - height = height.to_i - count = count.to_i - interval = interval.to_i - storyboard_width = storyboard_width.to_i - storyboard_height = storyboard_height.to_i - storyboard_count = (count / (storyboard_width * storyboard_height)).ceil.to_i - - items << { - url: url.to_s.sub("$L", i).sub("$N", "M$M"), - width: width, - height: height, - count: count, - interval: interval, - storyboard_width: storyboard_width, - storyboard_height: storyboard_height, - storyboard_count: storyboard_count, - } - end - - items + container = info.dig?("storyboards") || JSON::Any.new("{}") + return IV::Videos::Storyboard.from_yt_json(container, self.length_seconds) end def paid @@ -227,15 +147,29 @@ struct Video info.dig?("streamingData", "hlsManifestUrl").try &.as_s end - def dash_manifest_url - info.dig?("streamingData", "dashManifestUrl").try &.as_s + def dash_manifest_url : String? + raw_dash_url = info.dig?("streamingData", "dashManifestUrl").try &.as_s + return nil if raw_dash_url.nil? + + # Use manifest v5 parameter to reduce file size + # See https://github.com/iv-org/invidious/issues/4186 + dash_url = URI.parse(raw_dash_url) + dash_query = dash_url.query || "" + + if dash_query.empty? + dash_url.path = "#{dash_url.path}/mpd_version/5" + else + dash_url.query = "#{dash_query}&mpd_version=5" + end + + return dash_url.to_s end def genre_url : String? - info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil + info["genreUcid"].try &.as_s? ? "/channel/#{info["genreUcid"]}" : nil end - def is_vr : Bool? + def vr? : Bool? return {"EQUIRECTANGULAR", "MESH"}.includes? self.projection_type end @@ -316,6 +250,21 @@ struct Video {% if flag?(:debug_macros) %} {{debug}} {% end %} end + # Macro to generate ? and = accessor methods for attributes in `info` + private macro predicate_bool(method_name, name) + # Return {{name.stringify}} from `info` + def {{method_name.id.underscore}}? : Bool + return info[{{name.stringify}}]?.try &.as_bool || false + end + + # Update {{name.stringify}} into `info` + def {{method_name.id.underscore}}=(value : Bool) + info[{{name.stringify}}] = JSON::Any.new(value) + end + + {% if flag?(:debug_macros) %} {{debug}} {% end %} + end + # Method definitions, using the macros above getset_string author @@ -337,11 +286,12 @@ struct Video getset_i64 likes getset_i64 views + # TODO: Make predicate_bool the default as to adhere to Crystal conventions getset_bool allowRatings getset_bool authorVerified getset_bool isFamilyFriendly getset_bool isListed - getset_bool isUpcoming + predicate_bool upcoming, isUpcoming end def get_video(id, refresh = true, region = nil, force_refresh = false) @@ -376,21 +326,6 @@ end def fetch_video(id, region) info = extract_video_info(video_id: id) - allowed_regions = info - .dig?("microformat", "playerMicroformatRenderer", "availableCountries") - .try &.as_a.map &.as_s || [] of String - - # Check for region-blocks - if info["reason"]?.try &.as_s.includes?("your country") - bypass_regions = PROXY_LIST.keys & allowed_regions - if !bypass_regions.empty? - region = bypass_regions[rand(bypass_regions.size)] - region_info = extract_video_info(video_id: id, proxy_region: region) - region_info["region"] = JSON::Any.new(region) if region - info = region_info if !region_info["reason"]? - end - end - if reason = info["reason"]? if reason == "Video unavailable" raise NotFoundException.new(reason.as_s || "") diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr index 256dfcc0..c811cfe1 100644 --- a/src/invidious/videos/caption.cr +++ b/src/invidious/videos/caption.cr @@ -52,17 +52,13 @@ module Invidious::Videos break end end - result = String.build do |result| - result << <<-END_VTT - WEBVTT - Kind: captions - Language: #{tlang || @language_code} + settings_field = { + "Kind" => "captions", + "Language" => "#{tlang || @language_code}", + } - END_VTT - - result << "\n\n" - + result = WebVTT.build(settings_field) do |vtt| cues.each_with_index do |node, i| start_time = node["t"].to_f.milliseconds @@ -76,29 +72,16 @@ module Invidious::Videos end_time = start_time + duration end - # start_time - result << start_time.hours.to_s.rjust(2, '0') - result << ':' << start_time.minutes.to_s.rjust(2, '0') - result << ':' << start_time.seconds.to_s.rjust(2, '0') - result << '.' << start_time.milliseconds.to_s.rjust(3, '0') - - result << " --> " - - # end_time - result << end_time.hours.to_s.rjust(2, '0') - result << ':' << end_time.minutes.to_s.rjust(2, '0') - result << ':' << end_time.seconds.to_s.rjust(2, '0') - result << '.' << end_time.milliseconds.to_s.rjust(3, '0') - - result << "\n" - - node.children.each do |s| - result << s.content + text = String.build do |io| + node.children.each do |s| + io << s.content + end end - result << "\n" - result << "\n" + + vtt.cue(start_time, end_time, text) end end + return result end end @@ -140,6 +123,7 @@ module Invidious::Videos "Esperanto", "Estonian", "Filipino", + "Filipino (auto-generated)", "Finnish", "French", "French (auto-generated)", diff --git a/src/invidious/videos/clip.cr b/src/invidious/videos/clip.cr new file mode 100644 index 00000000..29c57182 --- /dev/null +++ b/src/invidious/videos/clip.cr @@ -0,0 +1,22 @@ +require "json" + +# returns start_time, end_time and clip_title +def parse_clip_parameters(params) : {Float64?, Float64?, String?} + decoded_protobuf = params.try { |i| URI.decode_www_form(i) } + .try { |i| Base64.decode(i) } + .try { |i| IO::Memory.new(i) } + .try { |i| Protodec::Any.parse(i) } + + start_time = decoded_protobuf + .try(&.["50:0:embedded"]["2:1:varint"].as_i64) + .try { |i| i/1000 } + + end_time = decoded_protobuf + .try(&.["50:0:embedded"]["3:2:varint"].as_i64) + .try { |i| i/1000 } + + clip_title = decoded_protobuf + .try(&.["50:0:embedded"]["4:3:string"].as_s) + + return start_time, end_time, clip_title +end diff --git a/src/invidious/videos/description.cr b/src/invidious/videos/description.cr index 542cb416..1371bebb 100644 --- a/src/invidious/videos/description.cr +++ b/src/invidious/videos/description.cr @@ -7,7 +7,19 @@ private def copy_string(str : String::Builder, iter : Iterator, count : Int) : I cp = iter.next break if cp.is_a?(Iterator::Stop) - str << cp.chr + if cp == 0x26 # Ampersand (&) + str << "&" + elsif cp == 0x27 # Single quote (') + str << "'" + elsif cp == 0x22 # Double quote (") + str << """ + elsif cp == 0x3C # Less-than (<) + str << "<" + elsif cp == 0x3E # Greater than (>) + str << ">" + else + str << cp.chr + end # A codepoint from the SMP counts twice copied += 1 if cp > 0xFFFF @@ -24,7 +36,13 @@ def parse_description(desc, video_id : String) : String? return "" if content.empty? commands = desc["commandRuns"]?.try &.as_a - return content if commands.nil? + if commands.nil? + # Slightly faster than HTML.escape, as we're only doing one pass on + # the string instead of five for the standard library + return String.build do |str| + copy_string(str, content.each_codepoint, content.size) + end + end # Not everything is stored in UTF-8 on youtube's side. The SMP codepoints # (0x10000 and above) are encoded as UTF-16 surrogate pairs, which are diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 06ff96b1..5ca4bdb2 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -36,6 +36,13 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container") + if published_time_text = related["publishedTimeText"]? + decoded_time = decode_date(published_time_text["simpleText"].to_s) + published = decoded_time.to_rfc3339.to_s + else + published = nil + end + # TODO: when refactoring video types, make a struct for related videos # or reuse an existing type, if that fits. return { @@ -47,15 +54,16 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? "view_count" => JSON::Any.new(view_count || "0"), "short_view_count" => JSON::Any.new(short_view_count || "0"), "author_verified" => JSON::Any.new(author_verified), + "published" => JSON::Any.new(published || ""), } end -def extract_video_info(video_id : String, proxy_region : String? = nil) +def extract_video_info(video_id : String) # Init client config for the API - client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region) + client_config = YoutubeAPI::ClientConfig.new # Fetch data from the player endpoint - player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) + player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config) playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s @@ -78,6 +86,11 @@ def extract_video_info(video_id : String, proxy_region : String? = nil) # YouTube may return a different video player response than expected. # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 # Line to be reverted if one day we solve the video not available issue. + + # Although technically not a call to /videoplayback the fact that YouTube is returning the + # wrong video means that we should count it as a failure. + get_playback_statistic()["totalRequests"] += 1 + return { "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), "reason" => JSON::Any.new("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. <a href=\"https://github.com/iv-org/invidious/issues/3822\">Click here for more info about the issue.</a>"), @@ -97,38 +110,42 @@ def extract_video_info(video_id : String, proxy_region : String? = nil) new_player_response = nil - if reason.nil? + # Don't use Android test suite client if po_token is passed because po_token doesn't + # work for Android test suite client. + if reason.nil? && CONFIG.po_token.nil? # Fetch the video streams using an Android client in order to get the # decrypted URLs and maybe fix throttling issues (#2194). See the # following issue for an explanation about decrypted URLs: # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 - client_config.client_type = YoutubeAPI::ClientType::Android - new_player_response = try_fetch_streaming_data(video_id, client_config) - elsif !reason.includes?("your country") # Handled separately - # The Android embedded client could help here - client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed - new_player_response = try_fetch_streaming_data(video_id, client_config) - end - - # Last hope - if new_player_response.nil? - client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed + client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite new_player_response = try_fetch_streaming_data(video_id, client_config) end # Replace player response and reset reason if !new_player_response.nil? - # Preserve storyboard data before replacement + # Preserve captions & storyboard data before replacement new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]? + new_player_response["captions"] = player_response["captions"] if player_response["captions"]? player_response = new_player_response params.delete("reason") end - {"captions", "playabilityStatus", "playerConfig", "storyboards", "streamingData"}.each do |f| + {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f| params[f] = player_response[f] if player_response[f]? end + # Convert URLs, if those are present + if streaming_data = player_response["streamingData"]? + %w[formats adaptiveFormats].each do |key| + streaming_data.as_h[key]?.try &.as_a.each do |format| + format.as_h["url"] = JSON::Any.new(convert_url(format)) + end + end + + params["streamingData"] = streaming_data + end + # Data structure version, for cache control params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64) @@ -137,9 +154,7 @@ end def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)? LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.") - # CgIQBg is a workaround for streaming URLs that returns a 403. - # See https://github.com/iv-org/invidious/issues/4027#issuecomment-1666944520 - response = YoutubeAPI.player(video_id: id, params: "CgIQBg", client_config: client_config) + response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config) playability_status = response["playabilityStatus"]["status"] LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.") @@ -147,7 +162,7 @@ def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConf if id != response.dig("videoDetails", "videoId") # YouTube may return a different video player response than expected. # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 - raise VideoNotAvailableException.new( + raise InfoException.new( "The video returned by YouTube isn't the requested one. (#{client_config.client_type} client)" ) elsif playability_status == "OK" @@ -180,10 +195,11 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any end video_details = player_response.dig?("videoDetails") - microformat = player_response.dig?("microformat", "playerMicroformatRenderer") + if !(microformat = player_response.dig?("microformat", "playerMicroformatRenderer")) + microformat = {} of String => JSON::Any + end raise BrokenTubeException.new("videoDetails") if !video_details - raise BrokenTubeException.new("microformat") if !microformat # Basic video infos @@ -208,7 +224,19 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp") .try { |t| Time.parse_rfc3339(t.as_s) } + premiere_timestamp ||= player_response.dig?( + "playabilityStatus", "liveStreamability", + "liveStreamabilityRenderer", "offlineSlate", + "liveStreamOfflineSlateRenderer", "scheduledStartTime" + ) + .try &.as_s.to_i64 + .try { |t| Time.unix(t) } + live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow") + .try &.as_bool + live_now ||= video_details.dig?("isLive").try &.as_bool || false + + post_live_dvr = video_details.dig?("isPostLiveDvr") .try &.as_bool || false # Extra video infos @@ -217,7 +245,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any .try &.as_a.map &.as_s || [] of String allow_ratings = video_details["allowRatings"]?.try &.as_bool - family_friendly = microformat["isFamilySafe"].try &.as_bool + family_friendly = microformat["isFamilySafe"]?.try &.as_bool is_listed = video_details["isCrawlable"]?.try &.as_bool is_upcoming = video_details["isUpcoming"]?.try &.as_bool @@ -262,7 +290,18 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any .try &.dig?("videoActions", "menuRenderer", "topLevelButtons") if toplevel_buttons - likes_button = toplevel_buttons.try &.as_a + # New Format as of december 2023 + likes_button = toplevel_buttons.dig?(0, + "segmentedLikeDislikeButtonViewModel", + "likeButtonViewModel", + "likeButtonViewModel", + "toggleButtonViewModel", + "toggleButtonViewModel", + "defaultButtonViewModel", + "buttonViewModel" + ) + + likes_button ||= toplevel_buttons.try &.as_a .find(&.dig?("toggleButtonRenderer", "defaultIcon", "iconType").=== "LIKE") .try &.["toggleButtonRenderer"] @@ -275,9 +314,10 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any ) if likes_button + likes_txt = likes_button.dig?("accessibilityText") # Note: The like count from `toggledText` is off by one, as it would # represent the new like count in the event where the user clicks on "like". - likes_txt = (likes_button["defaultText"]? || likes_button["toggledText"]?) + likes_txt ||= (likes_button["defaultText"]? || likes_button["toggledText"]?) .try &.dig?("accessibility", "accessibilityData", "label") likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt @@ -400,6 +440,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any "isListed" => JSON::Any.new(is_listed || false), "isUpcoming" => JSON::Any.new(is_upcoming || false), "keywords" => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }), + "isPostLiveDvr" => JSON::Any.new(post_live_dvr), # Related videos "relatedVideos" => JSON::Any.new(related), # Description @@ -408,7 +449,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil), # Video metadata "genre" => JSON::Any.new(genre.try &.as_s || ""), - "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""), + "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s?), "license" => JSON::Any.new(license.try &.as_s || ""), # Music section "music" => JSON.parse(music_list.to_json), @@ -422,3 +463,35 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any return params end + +private def convert_url(fmt) + if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) } + sp = cfr["sp"] + url = URI.parse(cfr["url"]) + params = url.query_params + + LOGGER.debug("convert_url: Decoding '#{cfr}'") + + unsig = DECRYPT_FUNCTION.try &.decrypt_signature(cfr["s"]) + params[sp] = unsig if unsig + else + url = URI.parse(fmt["url"].as_s) + params = url.query_params + end + + n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"]) + params["n"] = n if n + + if token = CONFIG.po_token + params["pot"] = token + end + + url.query_params = params + LOGGER.trace("convert_url: new url is '#{url}'") + + return url.to_s +rescue ex + LOGGER.debug("convert_url: Error when parsing video URL") + LOGGER.trace(ex.inspect_with_backtrace) + return "" +end diff --git a/src/invidious/videos/storyboard.cr b/src/invidious/videos/storyboard.cr new file mode 100644 index 00000000..bd0eef59 --- /dev/null +++ b/src/invidious/videos/storyboard.cr @@ -0,0 +1,122 @@ +require "uri" +require "http/params" + +module Invidious::Videos + struct Storyboard + # Template URL + getter url : URI + getter proxied_url : URI + + # Thumbnail parameters + getter width : Int32 + getter height : Int32 + getter count : Int32 + getter interval : Int32 + + # Image (storyboard) parameters + getter rows : Int32 + getter columns : Int32 + getter images_count : Int32 + + def initialize( + *, @url, @width, @height, @count, @interval, + @rows, @columns, @images_count, + ) + authority = /(i\d?).ytimg.com/.match!(@url.host.not_nil!)[1]? + + @proxied_url = URI.parse(HOST_URL) + @proxied_url.path = "/sb/#{authority}/#{@url.path.lchop("/sb/")}" + @proxied_url.query = @url.query + end + + # Parse the JSON structure from Youtube + def self.from_yt_json(container : JSON::Any, length_seconds : Int32) : Array(Storyboard) + # Livestream storyboards are a bit different + # TODO: document exactly how + if storyboard = container.dig?("playerLiveStoryboardSpecRenderer", "spec").try &.as_s + return [Storyboard.new( + url: URI.parse(storyboard.split("#")[0]), + width: 106, + height: 60, + count: -1, + interval: 5000, + rows: 3, + columns: 3, + images_count: -1 + )] + end + + # Split the storyboard string into chunks + # + # General format (whitespaces added for legibility): + # https://i.ytimg.com/sb/<video_id>/storyboard3_L$L/$N.jpg?sqp=<sig0> + # | 48 # 27 # 100 # 10 # 10 # 0 # default # rs$<sig1> + # | 80 # 45 # 95 # 10 # 10 # 10000 # M$M # rs$<sig2> + # | 160 # 90 # 95 # 5 # 5 # 10000 # M$M # rs$<sig3> + # + storyboards = container.dig?("playerStoryboardSpecRenderer", "spec") + .try &.as_s.split("|") + + return [] of Storyboard if !storyboards + + # The base URL is the first chunk + base_url = URI.parse(storyboards.shift) + + return storyboards.map_with_index do |sb, i| + # Separate the different storyboard parameters: + # width/height: respective dimensions, in pixels, of a single thumbnail + # count: how many thumbnails are displayed across the full video + # columns/rows: maximum amount of thumbnails that can be stuffed in a + # single image, horizontally and vertically. + # interval: interval between two thumbnails, in milliseconds + # name: storyboard filename. Usually "M$M" or "default" + # sigh: URL cryptographic signature + width, height, count, columns, rows, interval, name, sigh = sb.split("#") + + width = width.to_i + height = height.to_i + count = count.to_i + interval = interval.to_i + columns = columns.to_i + rows = rows.to_i + + # Copy base URL object, so that we can modify it + url = base_url.dup + + # Add the signature to the URL + params = url.query_params + params["sigh"] = sigh + url.query_params = params + + # Replace the template parts with what we have + url.path = url.path.sub("$L", i).sub("$N", name) + + # This value represents the maximum amount of thumbnails that can fit + # in a single image. The last image (or the only one for short videos) + # will contain less thumbnails than that. + thumbnails_per_image = columns * rows + + # This value represents the total amount of storyboards required to + # hold all of the thumbnails. It can't be less than 1. + images_count = (count / thumbnails_per_image).ceil.to_i + + # Compute the interval when needed (in general, that's only required + # for the first "default" storyboard). + if interval == 0 + interval = ((length_seconds / count) * 1_000).to_i + end + + Storyboard.new( + url: url, + width: width, + height: height, + count: count, + interval: interval, + rows: rows, + columns: columns, + images_count: images_count, + ) + end + end + end +end diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr index f3360a52..4bd9f820 100644 --- a/src/invidious/videos/transcript.cr +++ b/src/invidious/videos/transcript.cr @@ -1,8 +1,26 @@ module Invidious::Videos - # Namespace for methods primarily relating to Transcripts - module Transcript - record TranscriptLine, start_ms : Time::Span, end_ms : Time::Span, line : String + # A `Transcripts` struct encapsulates a sequence of lines that together forms the whole transcript for a given YouTube video. + # These lines can be categorized into two types: section headings and regular lines representing content from the video. + struct Transcript + # Types + record HeadingLine, start_ms : Time::Span, end_ms : Time::Span, line : String + record RegularLine, start_ms : Time::Span, end_ms : Time::Span, line : String + alias TranscriptLine = HeadingLine | RegularLine + property lines : Array(TranscriptLine) + + property language_code : String + property auto_generated : Bool + + # User friendly label for the current transcript. + # Example: "English (auto-generated)" + property label : String + + # Initializes a new Transcript struct with the contents and associated metadata describing it + def initialize(@lines : Array(TranscriptLine), @language_code : String, @auto_generated : Bool, @label : String) + end + + # Generates a protobuf string to fetch the requested transcript from YouTube def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String kind = auto_generated ? "asr" : "" @@ -30,74 +48,79 @@ module Invidious::Videos return params end - def self.convert_transcripts_to_vtt(initial_data : Hash(String, JSON::Any), target_language : String) : String - # Convert into array of TranscriptLine - lines = self.parse(initial_data) + # Constructs a Transcripts struct from the initial YouTube response + def self.from_raw(initial_data : Hash(String, JSON::Any), language_code : String, auto_generated : Bool) + transcript_panel = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", + "content", "transcriptSearchPanelRenderer") - # Taken from Invidious::Videos::Captions::Metadata.timedtext_to_vtt() - vtt = String.build do |vtt| - vtt << <<-END_VTT - WEBVTT - Kind: captions - Language: #{target_language} + segment_list = transcript_panel.dig("body", "transcriptSegmentListRenderer") + if !segment_list["initialSegments"]? + raise NotFoundException.new("Requested transcript does not exist") + end - END_VTT - - vtt << "\n\n" + # Extract user-friendly label for the current transcript - lines.each do |line| - start_time = line.start_ms - end_time = line.end_ms + footer_language_menu = transcript_panel.dig?( + "footer", "transcriptFooterRenderer", "languageMenu", "sortFilterSubMenuRenderer", "subMenuItems" + ) - # start_time - vtt << start_time.hours.to_s.rjust(2, '0') - vtt << ':' << start_time.minutes.to_s.rjust(2, '0') - vtt << ':' << start_time.seconds.to_s.rjust(2, '0') - vtt << '.' << start_time.milliseconds.to_s.rjust(3, '0') + if footer_language_menu + label = footer_language_menu.as_a.select(&.["selected"].as_bool)[0]["title"].as_s + else + label = language_code + end - vtt << " --> " + # Extract transcript lines - # end_time - vtt << end_time.hours.to_s.rjust(2, '0') - vtt << ':' << end_time.minutes.to_s.rjust(2, '0') - vtt << ':' << end_time.seconds.to_s.rjust(2, '0') - vtt << '.' << end_time.milliseconds.to_s.rjust(3, '0') + initial_segments = segment_list["initialSegments"].as_a - vtt << "\n" - vtt << line.line + lines = [] of TranscriptLine - vtt << "\n" - vtt << "\n" + initial_segments.each do |line| + if unpacked_line = line["transcriptSectionHeaderRenderer"]? + line_type = HeadingLine + else + unpacked_line = line["transcriptSegmentRenderer"] + line_type = RegularLine end - end - return vtt - end - - private def self.parse(initial_data : Hash(String, JSON::Any)) - body = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", - "content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer", - "initialSegments").as_a + start_ms = unpacked_line["startMs"].as_s.to_i.millisecond + end_ms = unpacked_line["endMs"].as_s.to_i.millisecond + text = extract_text(unpacked_line["snippet"]) || "" - lines = [] of TranscriptLine - body.each do |line| - # Transcript section headers. They are not apart of the captions and as such we can safely skip them. - if line.as_h.has_key?("transcriptSectionHeaderRenderer") - next - end + lines << line_type.new(start_ms, end_ms, text) + end - line = line["transcriptSegmentRenderer"] + return Transcript.new( + lines: lines, + language_code: language_code, + auto_generated: auto_generated, + label: label + ) + end - start_ms = line["startMs"].as_s.to_i.millisecond - end_ms = line["endMs"].as_s.to_i.millisecond + # Converts transcript lines to a WebVTT file + # + # This is used within Invidious to replace subtitles + # as to workaround YouTube's rate-limited timedtext endpoint. + def to_vtt + settings_field = { + "Kind" => "captions", + "Language" => @language_code, + } - text = extract_text(line["snippet"]) || "" + vtt = WebVTT.build(settings_field) do |builder| + @lines.each do |line| + # Section headers are excluded from the VTT conversion as to + # match the regular captions returned from YouTube as much as possible + next if line.is_a? HeadingLine - lines << TranscriptLine.new(start_ms, end_ms, text) + builder.cue(line.start_ms, line.end_ms, line.line) + end end - return lines + return vtt end end end diff --git a/src/invidious/videos/video_preferences.cr b/src/invidious/videos/video_preferences.cr index 34cf7ff0..48177bd8 100644 --- a/src/invidious/videos/video_preferences.cr +++ b/src/invidious/videos/video_preferences.cr @@ -2,6 +2,7 @@ struct VideoPreferences include JSON::Serializable property annotations : Bool + property preload : Bool property autoplay : Bool property comments : Array(String) property continue : Bool @@ -28,6 +29,7 @@ end def process_video_params(query, preferences) annotations = query["iv_load_policy"]?.try &.to_i? + preload = query["preload"]?.try { |q| (q == "true" || q == "1").to_unsafe } autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } comments = query["comments"]?.try &.split(",").map(&.downcase) continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe } @@ -50,6 +52,7 @@ def process_video_params(query, preferences) if preferences # region ||= preferences.region annotations ||= preferences.annotations.to_unsafe + preload ||= preferences.preload.to_unsafe autoplay ||= preferences.autoplay.to_unsafe comments ||= preferences.comments continue ||= preferences.continue.to_unsafe @@ -70,6 +73,7 @@ def process_video_params(query, preferences) end annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe + preload ||= CONFIG.default_user_preferences.preload.to_unsafe autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe comments ||= CONFIG.default_user_preferences.comments continue ||= CONFIG.default_user_preferences.continue.to_unsafe @@ -89,6 +93,7 @@ def process_video_params(query, preferences) save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe annotations = annotations == 1 + preload = preload == 1 autoplay = autoplay == 1 continue = continue == 1 continue_autoplay = continue_autoplay == 1 @@ -128,6 +133,7 @@ def process_video_params(query, preferences) params = VideoPreferences.new({ annotations: annotations, + preload: preload, autoplay: autoplay, comments: comments, continue: continue, diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr index fbb43358..1fe8ab7e 100644 --- a/src/invidious/views/channel.ecr +++ b/src/invidious/views/channel.ecr @@ -32,13 +32,13 @@ <meta property="og:site_name" content="Invidious"> <meta property="og:url" content="<%= HOST_URL %>/channel/<%= ucid %>"> <meta property="og:title" content="<%= author %>"> -<meta property="og:image" content="/ggpht<%= channel_profile_pic %>"> +<meta property="og:image" content="<%= HOST_URL %>/ggpht<%= channel_profile_pic %>"> <meta property="og:description" content="<%= channel.description %>"> <meta name="twitter:card" content="summary"> <meta name="twitter:url" content="<%= HOST_URL %>/channel/<%= ucid %>"> <meta name="twitter:title" content="<%= author %>"> <meta name="twitter:description" content="<%= channel.description %>"> -<meta name="twitter:image" content="/ggpht<%= channel_profile_pic %>"> +<meta name="twitter:image" content="<%= HOST_URL %>/ggpht<%= channel_profile_pic %>"> <link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/channel/<%= ucid %>" /> <%- end -%> diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr index 24efc34e..d2a305d3 100644 --- a/src/invidious/views/community.ecr +++ b/src/invidious/views/community.ecr @@ -26,7 +26,7 @@ <p><%= error_message %></p> </div> <% else %> - <div class="h-box pure-g" id="comments"> + <div class="h-box pure-g comments" id="comments"> <%= IV::Frontend::Comments.template_youtube(items.not_nil!, locale, thin_mode) %> </div> <% end %> diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index c29ec47b..c966a926 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -26,8 +26,9 @@ </a></div> </div> + <% if !item.channel_handle.nil? %><p class="channel-name" dir="auto"><%= item.channel_handle %></p><% end %> <p><%= translate_count(locale, "generic_subscribers_count", item.subscriber_count, NumberFormatting::Separator) %></p> - <% if !item.auto_generated %><p><%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %></p><% end %> + <% if !item.auto_generated && item.channel_handle.nil? %><p><%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %></p><% end %> <h5><%= item.description_html %></h5> <% when SearchHashtag %> <% if !thin_mode %> @@ -81,11 +82,19 @@ </div> <div class="video-card-row flexible"> - <div class="flex-left"><a href="/channel/<%= item.ucid %>"> - <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> - <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> - </p> - </a></div> + <div class="flex-left"> + <% if !item.ucid.to_s.empty? %> + <a href="/channel/<%= item.ucid %>"> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + </a> + <% else %> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + <% end %> + </div> </div> <% when Category %> <% else %> @@ -119,7 +128,7 @@ <div class="top-left-overlay"> <%- if env.get? "show_watched" -%> - <form data-onsubmit="return_false" action="/watch_ajax?action_mark_watched=1&id=<%= item.id %>&referer=<%= env.get("current_page") %>" method="post"> + <form data-onsubmit="return_false" action="/watch_ajax?action=mark_watched&id=<%= item.id %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>"> <button type="submit" class="pure-button pure-button-secondary low-profile" data-onclick="mark_watched" data-id="<%= item.id %>"> @@ -129,14 +138,14 @@ <%- end -%> <%- if plid_form = env.get?("add_playlist_items") -%> - <%- form_parameters = "action_add_video=1&video_id=#{item.id}&playlist_id=#{plid_form}&referer=#{env.get("current_page")}" -%> + <%- form_parameters = "action=add_video&video_id=#{item.id}&playlist_id=#{plid_form}&referer=#{env.get("current_page")}" -%> <form data-onsubmit="return_false" action="/playlist_ajax?<%= form_parameters %>" method="post"> <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>"> <button type="submit" class="pure-button pure-button-secondary low-profile" data-onclick="add_playlist_item" data-id="<%= item.id %>" data-plid="<%= plid_form %>"><i class="icon ion-md-add"></i></button> </form> <%- elsif item.is_a?(PlaylistVideo) && (plid_form = env.get?("remove_playlist_items")) -%> - <%- form_parameters = "action_remove_video=1&set_video_id=#{item.index}&playlist_id=#{plid_form}&referer=#{env.get("current_page")}" -%> + <%- form_parameters = "action=remove_video&set_video_id=#{item.index}&playlist_id=#{plid_form}&referer=#{env.get("current_page")}" -%> <form data-onsubmit="return_false" action="/playlist_ajax?<%= form_parameters %>" method="post"> <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>"> <button type="submit" class="pure-button pure-button-secondary low-profile" @@ -159,11 +168,19 @@ </div> <div class="video-card-row flexible"> - <div class="flex-left"><a href="/channel/<%= item.ucid %>"> - <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> - <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> - </p> - </a></div> + <div class="flex-left"> + <% if !item.ucid.to_s.empty? %> + <a href="/channel/<%= item.ucid %>"> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + </a> + <% else %> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + <% end %> + </div> <%= rendered "components/video-context-buttons" %> </div> diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index c3c02df0..5c28358b 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -1,5 +1,6 @@ <video style="outline:none;width:100%;background-color:#000" playsinline poster="<%= thumbnail %>" id="player" class="on-video_player video-js player-style-<%= params.player_style %>" + preload="<% if params.preload %>auto<% else %>none<% end %>" <% if params.autoplay %>autoplay<% end %> <% if params.video_loop %>loop<% end %> <% if params.controls %>controls<% end %>> diff --git a/src/invidious/views/components/search_box.ecr b/src/invidious/views/components/search_box.ecr index a03785d1..29da2c52 100644 --- a/src/invidious/views/components/search_box.ecr +++ b/src/invidious/views/components/search_box.ecr @@ -6,4 +6,7 @@ title="<%= translate(locale, "search") %>" value="<%= env.get?("search").try {|x| HTML.escape(x.as(String)) } %>"> </fieldset> + <button type="submit" id="searchbutton" aria-label="<%= translate(locale, "search") %>"> + <i class="icon ion-ios-search"></i> + </button> </form> diff --git a/src/invidious/views/components/subscribe_widget.ecr b/src/invidious/views/components/subscribe_widget.ecr index 05e4e253..3cfcb0eb 100644 --- a/src/invidious/views/components/subscribe_widget.ecr +++ b/src/invidious/views/components/subscribe_widget.ecr @@ -1,13 +1,13 @@ <% if user %> <% if subscriptions.includes? ucid %> - <form action="/subscription_ajax?action_remove_subscriptions=1&c=<%= ucid %>&referer=<%= env.get("current_page") %>" method="post"> + <form action="/subscription_ajax?action=remove_subscriptions&c=<%= ucid %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>"> <button data-type="unsubscribe" id="subscribe" class="pure-button pure-button-primary"> <b><input style="all:unset" type="submit" value="<%= translate(locale, "Unsubscribe") %> | <%= sub_count_text %>"></b> </button> </form> <% else %> - <form action="/subscription_ajax?action_create_subscription_to_channel=1&c=<%= ucid %>&referer=<%= env.get("current_page") %>" method="post"> + <form action="/subscription_ajax?action=create_subscription_to_channel&c=<%= ucid %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>"> <button data-type="subscribe" id="subscribe" class="pure-button pure-button-primary"> <b><input style="all:unset" type="submit" value="<%= translate(locale, "Subscribe") %> | <%= sub_count_text %>"></b> diff --git a/src/invidious/views/components/video-context-buttons.ecr b/src/invidious/views/components/video-context-buttons.ecr index 385ed6b3..22458a03 100644 --- a/src/invidious/views/components/video-context-buttons.ecr +++ b/src/invidious/views/components/video-context-buttons.ecr @@ -1,6 +1,6 @@ <div class="flex-right flexible"> <div class="icon-buttons"> - <a title="<%=translate(locale, "videoinfo_watch_on_youTube")%>" href="https://www.youtube.com/watch<%=endpoint_params%>"> + <a title="<%=translate(locale, "videoinfo_watch_on_youTube")%>" rel="noreferrer noopener" href="https://www.youtube.com/watch<%=endpoint_params%>"> <i class="icon ion-logo-youtube"></i> </a> <a title="<%=translate(locale, "Audio mode")%>" href="/watch<%=endpoint_params%>&listen=1"> diff --git a/src/invidious/views/feeds/history.ecr b/src/invidious/views/feeds/history.ecr index bda4e1f3..13fe4147 100644 --- a/src/invidious/views/feeds/history.ecr +++ b/src/invidious/views/feeds/history.ecr @@ -37,7 +37,7 @@ </a> <div class="top-left-overlay"><div class="watched"> - <form data-onsubmit="return_false" action="/watch_ajax?action_mark_unwatched=1&id=<%= item %>&referer=<%= env.get("current_page") %>" method="post"> + <form data-onsubmit="return_false" action="/watch_ajax?action=mark_unwatched&id=<%= item %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> <button type="submit" class="pure-button pure-button-secondary low-profile" data-onclick="mark_unwatched" data-id="<%= item %>"><i class="icon ion-md-trash"></i></button> diff --git a/src/invidious/views/playlist.ecr b/src/invidious/views/playlist.ecr index 24ba437d..c27ddba6 100644 --- a/src/invidious/views/playlist.ecr +++ b/src/invidious/views/playlist.ecr @@ -83,7 +83,7 @@ <% if !playlist.is_a? InvidiousPlaylist %> <div class="pure-u-2-3"> - <a href="https://www.youtube.com/playlist?list=<%= playlist.id %>"> + <a rel="noreferrer noopener" href="https://www.youtube.com/playlist?list=<%= playlist.id %>"> <%= translate(locale, "View playlist on YouTube") %> </a> <span> | </span> diff --git a/src/invidious/views/post.ecr b/src/invidious/views/post.ecr new file mode 100644 index 00000000..fb03a44c --- /dev/null +++ b/src/invidious/views/post.ecr @@ -0,0 +1,48 @@ +<% content_for "header" do %> +<title>Invidious</title> +<% end %> + +<div> + <div id="post" class="comments post-comments"> + <%= IV::Frontend::Comments.template_youtube(post_response.not_nil!, locale, thin_mode) %> + </div> + + <% if nojs %> + <hr> + <% end %> + <br /> + + <div id="comments" class="comments post-comments"> + <% if nojs %> + <%= comment_html %> + <% else %> + <noscript> + <a href="/post/<%= id %>?ucid=<%= ucid %>&nojs=1"> + <%= translate(locale, "Hi! Looks like you have JavaScript turned off. Click here to view comments, keep in mind they may take a bit longer to load.") %> + </a> + </noscript> + <% end %> + </div> +</div> + +<script id="video_data" type="application/json"> +<%= +{ + "id" => id, + "youtube_comments_text" => HTML.escape(translate(locale, "View YouTube comments")), + "reddit_comments_text" => "", + "reddit_permalink_text" => "", + "comments_text" => HTML.escape(translate(locale, "View `x` comments", "{commentCount}")), + "hide_replies_text" => HTML.escape(translate(locale, "Hide replies")), + "show_replies_text" => HTML.escape(translate(locale, "Show replies")), + "params" => { + "comments": ["youtube"] + }, + "preferences" => prefs, + "base_url" => "/api/v1/post/#{URI.encode_www_form(id)}/comments", + "ucid" => ucid +}.to_pretty_json +%> +</script> +<script src="/js/comments.js?v=<%= ASSET_COMMIT %>"></script> +<script src="/js/post.js?v=<%= ASSET_COMMIT %>"></script>
\ No newline at end of file diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index 77265679..9904b4fc 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -1,5 +1,9 @@ +<% + locale = env.get("preferences").as(Preferences).locale + dark_mode = env.get("preferences").as(Preferences).dark_mode +%> <!DOCTYPE html> -<html lang="<%= env.get("preferences").as(Preferences).locale %>"> +<html lang="<%= locale %>"> <head> <meta charset="utf-8"> @@ -17,19 +21,14 @@ <link rel="stylesheet" href="/css/grids-responsive-min.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/ionicons.min.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/default.css?v=<%= ASSET_COMMIT %>"> + <link rel="stylesheet" href="/css/carousel.css?v=<%= ASSET_COMMIT %>"> <script src="/js/_helpers.js?v=<%= ASSET_COMMIT %>"></script> </head> -<% - locale = env.get("preferences").as(Preferences).locale - dark_mode = env.get("preferences").as(Preferences).dark_mode -%> - <body class="<%= dark_mode.blank? ? "no" : dark_mode %>-theme"> - <span style="display:none" id="dark_mode_pref"><%= env.get("preferences").as(Preferences).dark_mode %></span> + <span style="display:none" id="dark_mode_pref"><%= dark_mode %></span> <div class="pure-g"> - <div class="pure-u-1 pure-u-md-2-24"></div> - <div class="pure-u-1 pure-u-md-20-24" id="contents"> + <div class="pure-u-1 pure-u-xl-20-24" id="contents"> <div class="pure-g navbar h-box"> <% if navbar_search %> <div class="pure-u-1 pure-u-md-4-24"> @@ -43,8 +42,8 @@ <div class="pure-u-1 pure-u-md-8-24 user-field"> <% if env.get? "user" %> <div class="pure-u-1-4"> - <a id="toggle_theme" href="/toggle_theme?referer=<%= env.get?("current_page") %>" class="pure-menu-heading"> - <% if env.get("preferences").as(Preferences).dark_mode == "dark" %> + <a id="toggle_theme" href="/toggle_theme?referer=<%= env.get?("current_page") %>" class="pure-menu-heading" title="<%= translate(locale, "toggle_theme") %>"> + <% if dark_mode == "dark" %> <i class="icon ion-ios-sunny"></i> <% else %> <i class="icon ion-ios-moon"></i> @@ -81,8 +80,8 @@ </div> <% else %> <div class="pure-u-1-3"> - <a id="toggle_theme" href="/toggle_theme?referer=<%= env.get?("current_page") %>" class="pure-menu-heading"> - <% if env.get("preferences").as(Preferences).dark_mode == "dark" %> + <a id="toggle_theme" href="/toggle_theme?referer=<%= env.get?("current_page") %>" class="pure-menu-heading" title="<%= translate(locale, "toggle_theme") %>"> + <% if dark_mode == "dark" %> <i class="icon ion-ios-sunny"></i> <% else %> <i class="icon ion-ios-moon"></i> @@ -156,7 +155,6 @@ </footer> </div> - <div class="pure-u-1 pure-u-md-2-24"></div> </div> <script src="/js/handlers.js?v=<%= ASSET_COMMIT %>"></script> <script src="/js/themes.js?v=<%= ASSET_COMMIT %>"></script> diff --git a/src/invidious/views/user/data_control.ecr b/src/invidious/views/user/data_control.ecr index 27654b40..9ce42c99 100644 --- a/src/invidious/views/user/data_control.ecr +++ b/src/invidious/views/user/data_control.ecr @@ -27,6 +27,11 @@ </div> <div class="pure-control-group"> + <label for="import_youtube_wh"><%= translate(locale, "Import YouTube watch history (.json)") %></label> + <input type="file" id="import_youtube_wh" name="import_youtube_wh"> + </div> + + <div class="pure-control-group"> <label for="import_freetube"><%= translate(locale, "Import FreeTube subscriptions (.db)") %></label> <input type="file" id="import_freetube" name="import_freetube"> </div> diff --git a/src/invidious/views/user/preferences.ecr b/src/invidious/views/user/preferences.ecr index 55349c5a..cf8b5593 100644 --- a/src/invidious/views/user/preferences.ecr +++ b/src/invidious/views/user/preferences.ecr @@ -13,6 +13,11 @@ </div> <div class="pure-control-group"> + <label for="preload"><%= translate(locale, "preferences_preload_label") %></label> + <input name="preload" id="preload" type="checkbox" <% if preferences.preload %>checked<% end %>> + </div> + + <div class="pure-control-group"> <label for="autoplay"><%= translate(locale, "preferences_autoplay_label") %></label> <input name="autoplay" id="autoplay" type="checkbox" <% if preferences.autoplay %>checked<% end %>> </div> @@ -310,7 +315,7 @@ <div class="pure-control-group"> <label for="modified_source_code_url"><%= translate(locale, "adminprefs_modified_source_code_url_label") %></label> - <input name="modified_source_code_url" id="modified_source_code_url" type="input" <% if CONFIG.modified_source_code_url %>checked<% end %>> + <input name="modified_source_code_url" id="modified_source_code_url" type="url" value="<%= CONFIG.modified_source_code_url %>"> </div> <% end %> diff --git a/src/invidious/views/user/subscription_manager.ecr b/src/invidious/views/user/subscription_manager.ecr index c9801f09..d566e228 100644 --- a/src/invidious/views/user/subscription_manager.ecr +++ b/src/invidious/views/user/subscription_manager.ecr @@ -37,7 +37,7 @@ <div class="pure-u-2-5"></div> <div class="pure-u-1-5" style="text-align:right"> <h3 style="padding-right:0.5em"> - <form data-onsubmit="return_false" action="/subscription_ajax?action_remove_subscriptions=1&c=<%= channel.id %>&referer=<%= env.get("current_page") %>" method="post"> + <form data-onsubmit="return_false" action="/subscription_ajax?action=remove_subscriptions&c=<%= channel.id %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>"> <input style="all:unset" type="submit" data-onclick="remove_subscription" data-ucid="<%= channel.id %>" value="<%= translate(locale, "unsubscribe") %>"> </form> diff --git a/src/invidious/views/user/token_manager.ecr b/src/invidious/views/user/token_manager.ecr index a73fa048..8431deb0 100644 --- a/src/invidious/views/user/token_manager.ecr +++ b/src/invidious/views/user/token_manager.ecr @@ -29,7 +29,7 @@ </div> <div class="pure-u-1-5" style="text-align:right"> <h3 style="padding-right:0.5em"> - <form data-onsubmit="return_false" action="/token_ajax?action_revoke_token=1&session=<%= token[:session] %>&referer=<%= env.get("current_page") %>" method="post"> + <form data-onsubmit="return_false" action="/token_ajax?action=revoke_token&session=<%= token[:session] %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>"> <input style="all:unset" type="submit" data-onclick="revoke_token" data-session="<%= token[:session] %>" value="<%= translate(locale, "revoke") %>"> </form> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 498d57a1..6f9ced6f 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -10,7 +10,7 @@ <meta property="og:site_name" content="<%= author %> | Invidious"> <meta property="og:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>"> <meta property="og:title" content="<%= title %>"> -<meta property="og:image" content="/vi/<%= video.id %>/maxres.jpg"> +<meta property="og:image" content="<%= HOST_URL %>/vi/<%= video.id %>/maxres.jpg"> <meta property="og:description" content="<%= HTML.escape(video.short_description) %>"> <meta property="og:type" content="video.other"> <meta property="og:video:url" content="<%= HOST_URL %>/embed/<%= video.id %>"> @@ -62,9 +62,10 @@ we're going to need to do it here in order to allow for translations. "params" => params, "preferences" => preferences, "premiere_timestamp" => video.premiere_timestamp.try &.to_unix, - "vr" => video.is_vr, + "vr" => video.vr?, "projection_type" => video.projection_type, - "local_disabled" => CONFIG.disabled?("local") + "local_disabled" => CONFIG.disabled?("local"), + "support_reddit" => true }.to_pretty_json %> </script> @@ -112,19 +113,36 @@ we're going to need to do it here in order to allow for translations. <div class="pure-u-1 pure-u-lg-1-5"> <div class="h-box"> <span id="watch-on-youtube"> - <a href="https://www.youtube.com/watch?v=<%= video.id %>"><%= translate(locale, "videoinfo_watch_on_youTube") %></a> - (<a href="https://www.youtube.com/embed/<%= video.id %>"><%= translate(locale, "videoinfo_youTube_embed_link") %></a>) + <%- + link_yt_watch = URI.new(scheme: "https", host: "www.youtube.com", path: "/watch", query: "v=#{video.id}") + link_yt_embed = URI.new(scheme: "https", host: "www.youtube.com", path: "/embed/#{video.id}") + + if !plid.nil? && !continuation.nil? + link_yt_param = URI::Params{"list" => [plid], "index" => [continuation.to_s]} + link_yt_watch = IV::HttpServer::Utils.add_params_to_url(link_yt_watch, link_yt_param) + link_yt_embed = IV::HttpServer::Utils.add_params_to_url(link_yt_embed, link_yt_param) + end + -%> + <a id="link-yt-watch" rel="noreferrer noopener" data-base-url="<%= link_yt_watch %>" href="<%= link_yt_watch %>"><%= translate(locale, "videoinfo_watch_on_youTube") %></a> + (<a id="link-yt-embed" rel="noreferrer noopener" data-base-url="<%= link_yt_embed %>" href="<%= link_yt_embed %>"><%= translate(locale, "videoinfo_youTube_embed_link") %></a>) </span> + <p id="watch-on-another-invidious-instance"> - <% if env.get("preferences").as(Preferences).automatic_instance_redirect%> - <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a> - <% else %> - <a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a> - <% end %> + <%- link_iv_other = IV::Frontend::Misc.redirect_url(env) -%> + <a id="link-iv-other" data-base-url="<%= link_iv_other %>" href="<%= link_iv_other %>"><%= translate(locale, "Switch Invidious Instance") %></a> </p> + <p id="embed-link"> - <a href="<%= embed_link %>"><%= translate(locale, "videoinfo_invidious_embed_link") %></a> + <%- + params_iv_embed = env.params.query.dup + params_iv_embed.delete_all("v") + + link_iv_embed = URI.new(path: "/embed/#{id}") + link_iv_embed = IV::HttpServer::Utils.add_params_to_url(link_iv_embed, params_iv_embed) + -%> + <a id="link-iv-embed" data-base-url="<%= link_iv_embed %>" href="<%= link_iv_embed %>"><%= translate(locale, "videoinfo_invidious_embed_link") %></a> </p> + <p id="annotations"> <% if params.annotations %> <a href="/watch?<%= env.params.query %>&iv_load_policy=3"> @@ -140,7 +158,7 @@ we're going to need to do it here in order to allow for translations. <% if user %> <% playlists = Invidious::Database::Playlists.select_user_created_playlists(user.email) %> <% if !playlists.empty? %> - <form data-onsubmit="return_false" class="pure-form pure-form-stacked" action="/playlist_ajax" method="post" target="_blank"> + <form data-onsubmit="return_false" class="pure-form pure-form-stacked" action="/playlist_ajax?action=add_video" method="post" target="_blank"> <div class="pure-control-group"> <label for="playlist_id"><%= translate(locale, "Add to playlist: ") %></label> <select style="width:100%" name="playlist_id" id="playlist_id"> @@ -151,7 +169,6 @@ we're going to need to do it here in order to allow for translations. </div> <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> - <input type="hidden" name="action_add_video" value="1"> <input type="hidden" name="video_id" value="<%= video.id %>"> <button data-onclick="add_playlist_video" data-id="<%= video.id %>" type="submit" class="pure-button pure-button-primary"> <b><%= translate(locale, "Add to playlist") %></b> @@ -270,7 +287,7 @@ we're going to need to do it here in order to allow for translations. <hr> <% end %> - <div id="comments"> + <div id="comments" class="comments"> <% if nojs %> <%= comment_html %> <% else %> @@ -328,7 +345,7 @@ we're going to need to do it here in order to allow for translations. <h5 class="pure-g"> <div class="pure-u-14-24"> - <% if rv["ucid"]? %> + <% if !rv["ucid"].empty? %> <b style="width:100%"><a href="/channel/<%= rv["ucid"] %>"><%= rv["author"]? %><% if rv["author_verified"]? == "true" %> <i class="icon ion ion-md-checkmark-circle"></i><% end %></a></b> <% else %> <b style="width:100%"><%= rv["author"]? %><% if rv["author_verified"]? == "true" %> <i class="icon ion ion-md-checkmark-circle"></i><% end %></b> @@ -352,4 +369,5 @@ we're going to need to do it here in order to allow for translations. </div> <% end %> </div> +<script src="/js/comments.js?v=<%= ASSET_COMMIT %>"></script> <script src="/js/watch.js?v=<%= ASSET_COMMIT %>"></script> diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index e9eb726c..c4a73aa7 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -1,18 +1,6 @@ -def add_yt_headers(request) - if request.headers["User-Agent"] == "Crystal" - request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" - end - - request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" - request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" - request.headers["Accept-Language"] ||= "en-us,en;q=0.5" - - # Preserve original cookies and add new YT consent cookie for EU servers - request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}" - if !CONFIG.cookies.empty? - request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" - end -end +# Mapping of subdomain => YoutubeConnectionPool +# This is needed as we may need to access arbitrary subdomains of ytimg +private YTIMG_POOLS = {} of String => YoutubeConnectionPool struct YoutubeConnectionPool property! url : URI @@ -25,67 +13,104 @@ struct YoutubeConnectionPool @pool = build_pool() end - def client(region = nil, &block) - if region - conn = make_client(url, region) + def client(&) + conn = pool.checkout + # Proxy needs to be reinstated every time we get a client from the pool + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + + begin response = yield conn - else - conn = pool.checkout - begin - response = yield conn - rescue ex - conn.close - conn = HTTP::Client.new(url) - - conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET - conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC - conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" - response = yield conn - ensure - pool.release(conn) - end + rescue ex + conn.close + conn = make_client(url, force_resolve: true) + + response = yield conn + ensure + pool.release(conn) end response end private def build_pool - DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do - conn = HTTP::Client.new(url) - conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET - conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC - conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" - conn + options = DB::Pool::Options.new( + initial_pool_size: 0, + max_pool_size: capacity, + max_idle_pool_size: capacity, + checkout_timeout: timeout + ) + + DB::Pool(HTTP::Client).new(options) do + next make_client(url, force_resolve: true) end end end -def make_client(url : URI, region = nil) - client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure) - client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC - client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" - client.read_timeout = 10.seconds - client.connect_timeout = 10.seconds +def add_yt_headers(request) + request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" + request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" - if region - PROXY_LIST[region]?.try &.sample(40).each do |proxy| - begin - proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) - client.set_proxy(proxy) - break - rescue ex - end - end + request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" + request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + request.headers["Accept-Language"] ||= "en-us,en;q=0.5" + + # Preserve original cookies and add new YT consent cookie for EU servers + request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}" + if !CONFIG.cookies.empty? + request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" + end +end + +def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false) + client = HTTP::Client.new(url) + client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + + # Force the usage of a specific configured IP Family + if force_resolve + client.family = CONFIG.force_resolve + client.family = Socket::Family::INET if client.family == Socket::Family::UNSPEC end + client.before_request { |r| add_yt_headers(r) } if url.host.try &.ends_with?("youtube.com") || force_youtube_headers + client.read_timeout = 10.seconds + client.connect_timeout = 10.seconds + return client end -def make_client(url : URI, region = nil, &block) - client = make_client(url, region) +def make_client(url : URI, region = nil, force_resolve : Bool = false, &) + client = make_client(url, region, force_resolve: force_resolve) begin yield client ensure client.close end end + +def make_configured_http_proxy_client + # This method is only called when configuration for an HTTP proxy are set + config_proxy = CONFIG.http_proxy.not_nil! + + return HTTP::Proxy::Client.new( + config_proxy.host, + config_proxy.port, + + username: config_proxy.user, + password: config_proxy.password, + ) +end + +# Fetches a HTTP pool for the specified subdomain of ytimg.com +# +# Creates a new one when the specified pool for the subdomain does not exist +def get_ytimg_pool(subdomain) + if pool = YTIMG_POOLS[subdomain]? + return pool + else + LOGGER.info("ytimg_pool: Creating a new HTTP pool for \"https://#{subdomain}.ytimg.com\"") + pool = YoutubeConnectionPool.new(URI.parse("https://#{subdomain}.ytimg.com"), capacity: CONFIG.pool_size) + YTIMG_POOLS[subdomain] = pool + + return pool + end +end diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index aaf7772e..edd7bf1b 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -21,6 +21,7 @@ private ITEM_PARSERS = { Parsers::ItemSectionRendererParser, Parsers::ContinuationItemRendererParser, Parsers::HashtagRendererParser, + Parsers::LockupViewModelParser, } private alias InitialData = Hash(String, JSON::Any) @@ -66,6 +67,8 @@ private module Parsers author_id = author_fallback.id end + author_thumbnail = item_contents.dig?("channelThumbnailSupportedRenderers", "channelThumbnailWithLinkRenderer", "thumbnail", "thumbnails", 0, "url").try &.as_s + author_verified = has_verified_badge?(item_contents["ownerBadges"]?) # For live videos (and possibly recently premiered videos) there is no published information. @@ -108,22 +111,30 @@ private module Parsers length_seconds = 0 end - live_now = false - paid = false - premium = false - premiere_timestamp = item_contents.dig?("upcomingEventData", "startTime").try { |t| Time.unix(t.as_s.to_i64) } - + badges = VideoBadges::None item_contents["badges"]?.try &.as_a.each do |badge| b = badge["metadataBadgeRenderer"] case b["label"].as_s - when "LIVE NOW" - live_now = true - when "New", "4K", "CC" - # TODO + when "LIVE" + badges |= VideoBadges::LiveNow + when "New" + badges |= VideoBadges::New + when "4K" + badges |= VideoBadges::FourK + when "8K" + badges |= VideoBadges::EightK + when "VR180" + badges |= VideoBadges::VR180 + when "360°" + badges |= VideoBadges::VR360 + when "3D" + badges |= VideoBadges::ThreeD + when "CC" + badges |= VideoBadges::ClosedCaptions when "Premium" # TODO: Potentially available as item_contents["topStandaloneBadge"]["metadataBadgeRenderer"] - premium = true + badges |= VideoBadges::Premium else nil # Ignore end end @@ -137,10 +148,10 @@ private module Parsers views: view_count, description_html: description_html, length_seconds: length_seconds, - live_now: live_now, - premium: premium, premiere_timestamp: premiere_timestamp, author_verified: author_verified, + author_thumbnail: author_thumbnail, + badges: badges, }) end @@ -175,17 +186,18 @@ private module Parsers # Always simpleText # TODO change default value to nil - subscriber_count = item_contents.dig?("subscriberCountText", "simpleText") + subscriber_count = item_contents.dig?("subscriberCountText", "simpleText").try &.as_s + channel_handle = subscriber_count if (subscriber_count.try &.starts_with? "@") # Since youtube added channel handles, `VideoCountText` holds the number of # subscribers and `subscriberCountText` holds the handle, except when the # channel doesn't have a handle (e.g: some topic music channels). # See https://github.com/iv-org/invidious/issues/3394#issuecomment-1321261688 - if !subscriber_count || !subscriber_count.as_s.includes? " subscriber" - subscriber_count = item_contents.dig?("videoCountText", "simpleText") + if !subscriber_count || !subscriber_count.includes? " subscriber" + subscriber_count = item_contents.dig?("videoCountText", "simpleText").try &.as_s end subscriber_count = subscriber_count - .try { |s| short_text_to_number(s.as_s.split(" ")[0]).to_i32 } || 0 + .try { |s| short_text_to_number(s.split(" ")[0]).to_i32 } || 0 # Auto-generated channels doesn't have videoCountText # Taken from: https://github.com/iv-org/invidious/pull/2228#discussion_r717620922 @@ -200,6 +212,7 @@ private module Parsers author_thumbnail: author_thumbnail, subscriber_count: subscriber_count, video_count: video_count, + channel_handle: channel_handle, description_html: description_html, auto_generated: auto_generated, author_verified: author_verified, @@ -458,9 +471,9 @@ private module Parsers # Parses an InnerTube richItemRenderer into a SearchVideo. # Returns nil when the given object isn't a RichItemRenderer # - # A richItemRenderer seems to be a simple wrapper for a videoRenderer, used - # by the result page for hashtags and for the podcast tab on channels. - # It is located inside a continuationItems container for hashtags. + # A richItemRenderer seems to be a simple wrapper for a various other types, + # used on the hashtags result page and the channel podcast tab. It is located + # itself inside a richGridRenderer container. # module RichItemRendererParser def self.process(item : JSON::Any, author_fallback : AuthorFallback) @@ -473,6 +486,8 @@ private module Parsers child = VideoRendererParser.process(item_contents, author_fallback) child ||= ReelItemRendererParser.process(item_contents, author_fallback) child ||= PlaylistRendererParser.process(item_contents, author_fallback) + child ||= LockupViewModelParser.process(item_contents, author_fallback) + child ||= ShortsLockupViewModelParser.process(item_contents, author_fallback) return child end @@ -487,6 +502,9 @@ private module Parsers # reelItemRenderer items are used in the new (2022) channel layout, # in the "shorts" tab. # + # NOTE: As of 10/2024, it might have been fully replaced by shortsLockupViewModel + # TODO: Confirm that hypothesis + # module ReelItemRendererParser def self.process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item["reelItemRenderer"]? @@ -562,10 +580,140 @@ private module Parsers views: view_count, description_html: "", length_seconds: duration, - live_now: false, - premium: false, premiere_timestamp: Time.unix(0), author_verified: false, + author_thumbnail: nil, + badges: VideoBadges::None, + }) + end + + def self.parser_name + return {{@type.name}} + end + end + + # Parses an InnerTube lockupViewModel into a SearchPlaylist. + # Returns nil when the given object is not a lockupViewModel. + # + # This structure is present since November 2024 on the "podcasts" and + # "playlists" tabs of the channel page. It is usually encapsulated in either + # a richItemRenderer or a richGridRenderer. + # + module LockupViewModelParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["lockupViewModel"]? + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + playlist_id = item_contents["contentId"].as_s + + thumbnail_view_model = item_contents.dig( + "contentImage", "collectionThumbnailViewModel", + "primaryThumbnail", "thumbnailViewModel" + ) + + thumbnail = thumbnail_view_model.dig("image", "sources", 0, "url").as_s + + # This complicated sequences tries to extract the following data structure: + # "overlays": [{ + # "thumbnailOverlayBadgeViewModel": { + # "thumbnailBadges": [{ + # "thumbnailBadgeViewModel": { + # "text": "430 episodes", + # "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT" + # } + # }] + # } + # }] + # + # NOTE: this simplistic `.to_i` conversion might not work on larger + # playlists and hasn't been tested. + video_count = thumbnail_view_model.dig("overlays").as_a + .compact_map(&.dig?("thumbnailOverlayBadgeViewModel", "thumbnailBadges").try &.as_a) + .flatten + .find(nil, &.dig?("thumbnailBadgeViewModel", "text").try { |node| + {"episodes", "videos"}.any? { |str| node.as_s.ends_with?(str) } + }) + .try &.dig("thumbnailBadgeViewModel", "text").as_s.to_i(strict: false) + + metadata = item_contents.dig("metadata", "lockupMetadataViewModel") + title = metadata.dig("title", "content").as_s + + # TODO: Retrieve "updated" info from metadata parts + # rows = metadata.dig("metadata", "contentMetadataViewModel", "metadataRows").as_a + # parts_text = rows.map(&.dig?("metadataParts", "text", "content").try &.as_s) + # One of these parts should contain a string like: "Updated 2 days ago" + + # TODO: Maybe add a button to access the first video of the playlist? + # item_contents.dig("rendererContext", "commandContext", "onTap", "innertubeCommand", "watchEndpoint") + # Available fields: "videoId", "playlistId", "params" + + return SearchPlaylist.new({ + title: title, + id: playlist_id, + author: author_fallback.name, + ucid: author_fallback.id, + video_count: video_count || -1, + videos: [] of SearchPlaylistVideo, + thumbnail: thumbnail, + author_verified: false, + }) + end + + def self.parser_name + return {{@type.name}} + end + end + + # Parses an InnerTube shortsLockupViewModel into a SearchVideo. + # Returns nil when the given object is not a shortsLockupViewModel. + # + # This structure is present since around October 2024 on the "shorts" tab of + # the channel page and likely replaces the reelItemRenderer structure. It is + # usually (always?) encapsulated in a richItemRenderer. + # + module ShortsLockupViewModelParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["shortsLockupViewModel"]? + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + # TODO: Maybe add support for "oardefault.jpg" thumbnails? + # thumbnail = item_contents.dig("thumbnail", "sources", 0, "url").as_s + # Gives: https://i.ytimg.com/vi/{video_id}/oardefault.jpg?... + + video_id = item_contents.dig( + "onTap", "innertubeCommand", "reelWatchEndpoint", "videoId" + ).as_s + + title = item_contents.dig("overlayMetadata", "primaryText", "content").as_s + + view_count = short_text_to_number( + item_contents.dig("overlayMetadata", "secondaryText", "content").as_s + ) + + # Approximate to one minute, as "shorts" generally don't exceed that. + # NOTE: The actual duration is not provided by Youtube anymore. + # TODO: Maybe use -1 as an error value and handle that on the frontend? + duration = 60_i32 + + SearchVideo.new({ + title: title, + id: video_id, + author: author_fallback.name, + ucid: author_fallback.id, + published: Time.unix(0), + views: view_count, + description_html: "", + length_seconds: duration, + premiere_timestamp: Time.unix(0), + author_verified: false, + author_thumbnail: nil, + badges: VideoBadges::None, }) end @@ -820,9 +968,9 @@ module HelperExtractors end # Retrieves the ID required for querying the InnerTube browse endpoint. - # Raises when it's unable to do so + # Returns an empty string when it's unable to do so def self.get_browse_id(container) - return container.dig("navigationEndpoint", "browseEndpoint", "browseId").as_s + return container.dig?("navigationEndpoint", "browseEndpoint", "browseId").try &.as_s || "" end end @@ -854,7 +1002,7 @@ end # # This function yields the container so that items can be parsed separately. # -def extract_items(initial_data : InitialData, &block) +def extract_items(initial_data : InitialData, &) if unpackaged_data = initial_data["contents"]?.try &.as_h elsif unpackaged_data = initial_data["response"]?.try &.as_h elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 1).try &.as_h @@ -881,7 +1029,7 @@ end def extract_items( initial_data : InitialData, author_fallback : String? = nil, - author_id_fallback : String? = nil + author_id_fallback : String? = nil, ) : {Array(SearchItem), String?} items = [] of SearchItem continuation = nil diff --git a/src/invidious/yt_backend/extractors_utils.cr b/src/invidious/yt_backend/extractors_utils.cr index 11d95958..c83a2de5 100644 --- a/src/invidious/yt_backend/extractors_utils.cr +++ b/src/invidious/yt_backend/extractors_utils.cr @@ -83,5 +83,5 @@ end def extract_selected_tab(tabs) # Extract the selected tab from the array of tabs Youtube returns - return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"]?.try &.as_bool)[0]["tabRenderer"] + return tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"]?.try &.as_bool)[0]["tabRenderer"] end diff --git a/src/invidious/yt_backend/proxy.cr b/src/invidious/yt_backend/proxy.cr deleted file mode 100644 index 2d0fd4ba..00000000 --- a/src/invidious/yt_backend/proxy.cr +++ /dev/null @@ -1,316 +0,0 @@ -# See https://github.com/crystal-lang/crystal/issues/2963 -class HTTPProxy - getter proxy_host : String - getter proxy_port : Int32 - getter options : Hash(Symbol, String) - getter tls : OpenSSL::SSL::Context::Client? - - def initialize(@proxy_host, @proxy_port = 80, @options = {} of Symbol => String) - end - - def open(host, port, tls = nil, connection_options = {} of Symbol => Float64 | Nil) - dns_timeout = connection_options.fetch(:dns_timeout, nil) - connect_timeout = connection_options.fetch(:connect_timeout, nil) - read_timeout = connection_options.fetch(:read_timeout, nil) - - socket = TCPSocket.new @proxy_host, @proxy_port, dns_timeout, connect_timeout - socket.read_timeout = read_timeout if read_timeout - socket.sync = true - - socket << "CONNECT #{host}:#{port} HTTP/1.1\r\n" - - if options[:user]? - credentials = Base64.strict_encode("#{options[:user]}:#{options[:password]}") - credentials = "#{credentials}\n".gsub(/\s/, "") - socket << "Proxy-Authorization: Basic #{credentials}\r\n" - end - - socket << "\r\n" - - resp = parse_response(socket) - - if resp[:code]? == 200 - {% if !flag?(:without_openssl) %} - if tls - tls_socket = OpenSSL::SSL::Socket::Client.new(socket, context: tls, sync_close: true, hostname: host) - socket = tls_socket - end - {% end %} - - return socket - else - socket.close - raise IO::Error.new(resp.inspect) - end - end - - private def parse_response(socket) - resp = {} of Symbol => Int32 | String | Hash(String, String) - - begin - version, code, reason = socket.gets.as(String).chomp.split(/ /, 3) - - headers = {} of String => String - - while (line = socket.gets.as(String)) && (line.chomp != "") - name, value = line.split(/:/, 2) - headers[name.strip] = value.strip - end - - resp[:version] = version - resp[:code] = code.to_i - resp[:reason] = reason - resp[:headers] = headers - rescue - end - - return resp - end -end - -class HTTPClient < HTTP::Client - def set_proxy(proxy : HTTPProxy) - begin - @io = proxy.open(host: @host, port: @port, tls: @tls, connection_options: proxy_connection_options) - rescue IO::Error - @io = nil - end - end - - def unset_proxy - @io = nil - end - - def proxy_connection_options - opts = {} of Symbol => Float64 | Nil - - opts[:dns_timeout] = @dns_timeout - opts[:connect_timeout] = @connect_timeout - opts[:read_timeout] = @read_timeout - - return opts - end -end - -def get_proxies(country_code = "US") - # return get_spys_proxies(country_code) - return get_nova_proxies(country_code) -end - -def filter_proxies(proxies) - proxies.select! do |proxy| - begin - client = HTTPClient.new(YT_URL) - client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" - client.read_timeout = 10.seconds - client.connect_timeout = 10.seconds - - proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) - client.set_proxy(proxy) - - status_ok = client.head("/").status_code == 200 - client.close - status_ok - rescue ex - false - end - end - - return proxies -end - -def get_nova_proxies(country_code = "US") - country_code = country_code.downcase - client = HTTP::Client.new(URI.parse("https://www.proxynova.com")) - client.read_timeout = 10.seconds - client.connect_timeout = 10.seconds - - headers = HTTP::Headers.new - headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36" - headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8" - headers["Accept-Language"] = "Accept-Language: en-US,en;q=0.9" - headers["Host"] = "www.proxynova.com" - headers["Origin"] = "https://www.proxynova.com" - headers["Referer"] = "https://www.proxynova.com/proxy-server-list/country-#{country_code}/" - - response = client.get("/proxy-server-list/country-#{country_code}/", headers) - client.close - document = XML.parse_html(response.body) - - proxies = [] of {ip: String, port: Int32, score: Float64} - document.xpath_nodes(%q(//tr[@data-proxy-id])).each do |node| - ip = node.xpath_node(%q(.//td/abbr/script)).not_nil!.content - ip = ip.match(/document\.write\('(?<sub1>[^']+)'.substr\(8\) \+ '(?<sub2>[^']+)'/).not_nil! - ip = "#{ip["sub1"][8..-1]}#{ip["sub2"]}" - port = node.xpath_node(%q(.//td[2])).not_nil!.content.strip.to_i - - anchor = node.xpath_node(%q(.//td[4]/div)).not_nil! - speed = anchor["data-value"].to_f - latency = anchor["title"].to_f - uptime = node.xpath_node(%q(.//td[5]/span)).not_nil!.content.rchop("%").to_f - - # TODO: Tweak me - score = (uptime*4 + speed*2 + latency)/7 - proxies << {ip: ip, port: port, score: score} - end - - # proxies = proxies.sort_by { |proxy| proxy[:score] }.reverse - return proxies -end - -def get_spys_proxies(country_code = "US") - client = HTTP::Client.new(URI.parse("http://spys.one")) - client.read_timeout = 10.seconds - client.connect_timeout = 10.seconds - - headers = HTTP::Headers.new - headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36" - headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8" - headers["Accept-Language"] = "Accept-Language: en-US,en;q=0.9" - headers["Host"] = "spys.one" - headers["Origin"] = "http://spys.one" - headers["Referer"] = "http://spys.one/free-proxy-list/#{country_code}/" - headers["Content-Type"] = "application/x-www-form-urlencoded" - body = { - "xpp" => "5", - "xf1" => "0", - "xf2" => "0", - "xf4" => "0", - "xf5" => "1", - } - - response = client.post("/free-proxy-list/#{country_code}/", headers, form: body) - client.close - 20.times do - if response.status_code == 200 - break - end - response = client.post("/free-proxy-list/#{country_code}/", headers, form: body) - end - - response = XML.parse_html(response.body) - - mapping = response.xpath_node(%q(.//body/script)).not_nil!.content - mapping = mapping.match(/\}\('(?<p>[^']+)',\d+,\d+,'(?<x>[^']+)'/).not_nil! - p = mapping["p"].not_nil! - x = mapping["x"].not_nil! - mapping = decrypt_port(p, x) - - proxies = [] of {ip: String, port: Int32, score: Float64} - response = response.xpath_node(%q(//tr/td/table)).not_nil! - response.xpath_nodes(%q(.//tr)).each do |node| - if !node["onmouseover"]? - next - end - - ip = node.xpath_node(%q(.//td[1]/font[2])).to_s.match(/<font class="spy14">(?<address>[^<]+)</).not_nil!["address"] - encrypted_port = node.xpath_node(%q(.//td[1]/font[2]/script)).not_nil!.content - encrypted_port = encrypted_port.match(/<\\\/font>"\+(?<encrypted_port>[\d\D]+)\)$/).not_nil!["encrypted_port"] - - port = "" - encrypted_port.split("+").each do |number| - number = number.delete("()") - left_side, right_side = number.split("^") - result = mapping[left_side] ^ mapping[right_side] - port = "#{port}#{result}" - end - port = port.to_i - - latency = node.xpath_node(%q(.//td[6])).not_nil!.content.to_f - speed = node.xpath_node(%q(.//td[7]/font/table)).not_nil!["width"].to_f - uptime = node.xpath_node(%q(.//td[8]/font/acronym)).not_nil! - - # Skip proxies that are down - if uptime["title"].ends_with? "?" - next - end - - if md = uptime.content.match(/^\d+/) - uptime = md[0].to_f - else - next - end - - score = (uptime*4 + speed*2 + latency)/7 - - proxies << {ip: ip, port: port, score: score} - end - - proxies = proxies.sort_by!(&.[:score]).reverse! - return proxies -end - -def decrypt_port(p, x) - x = x.split("^") - s = {} of String => String - - 60.times do |i| - if x[i]?.try &.empty? - s[y_func(i)] = y_func(i) - else - s[y_func(i)] = x[i] - end - end - - x = s - p = p.gsub(/\b\w+\b/, x) - - p = p.split(";") - p = p.map(&.split("=")) - - mapping = {} of String => Int32 - p.each do |item| - if item == [""] - next - end - - key = item[0] - value = item[1] - value = value.split("^") - - if value.size == 1 - value = value[0].to_i - else - left_side = value[0].to_i? - left_side ||= mapping[value[0]] - right_side = value[1].to_i? - right_side ||= mapping[value[1]] - - value = left_side ^ right_side - end - - mapping[key] = value - end - - return mapping -end - -def y_func(c) - return (c < 60 ? "" : y_func((c/60).to_i)) + ((c = c % 60) > 35 ? ((c.to_u8 + 29).unsafe_chr) : c.to_s(36)) -end - -PROXY_LIST = { - "GB" => [{ip: "147.135.206.233", port: 3128}, {ip: "167.114.180.102", port: 8080}, {ip: "176.35.250.108", port: 8080}, {ip: "5.148.128.44", port: 80}, {ip: "62.7.85.234", port: 8080}, {ip: "88.150.135.10", port: 36624}], - "DE" => [{ip: "138.201.223.250", port: 31288}, {ip: "138.68.73.59", port: 32574}, {ip: "159.69.211.173", port: 3128}, {ip: "173.249.43.105", port: 3128}, {ip: "212.202.244.90", port: 8080}, {ip: "5.56.18.35", port: 38827}], - "FR" => [{ip: "137.74.254.242", port: 3128}, {ip: "151.80.143.155", port: 53281}, {ip: "178.33.150.97", port: 3128}, {ip: "37.187.2.31", port: 3128}, {ip: "5.135.164.72", port: 3128}, {ip: "5.39.91.73", port: 3128}, {ip: "51.38.162.2", port: 32231}, {ip: "51.38.217.121", port: 808}, {ip: "51.75.109.81", port: 3128}, {ip: "51.75.109.82", port: 3128}, {ip: "51.75.109.83", port: 3128}, {ip: "51.75.109.84", port: 3128}, {ip: "51.75.109.86", port: 3128}, {ip: "51.75.109.88", port: 3128}, {ip: "51.75.109.90", port: 3128}, {ip: "62.210.167.3", port: 3128}, {ip: "90.63.218.232", port: 8080}, {ip: "91.134.165.198", port: 9999}], - "IN" => [{ip: "1.186.151.206", port: 36253}, {ip: "1.186.63.130", port: 39142}, {ip: "103.105.40.1", port: 16538}, {ip: "103.105.40.153", port: 16538}, {ip: "103.106.148.203", port: 60227}, {ip: "103.106.148.207", port: 51451}, {ip: "103.12.246.12", port: 8080}, {ip: "103.14.235.109", port: 8080}, {ip: "103.14.235.26", port: 8080}, {ip: "103.198.172.4", port: 50820}, {ip: "103.205.112.1", port: 23500}, {ip: "103.209.64.19", port: 6666}, {ip: "103.211.76.5", port: 8080}, {ip: "103.216.82.19", port: 6666}, {ip: "103.216.82.190", port: 6666}, {ip: "103.216.82.209", port: 54806}, {ip: "103.216.82.214", port: 6666}, {ip: "103.216.82.37", port: 6666}, {ip: "103.216.82.44", port: 8080}, {ip: "103.216.82.50", port: 53281}, {ip: "103.22.173.230", port: 8080}, {ip: "103.224.38.2", port: 83}, {ip: "103.226.142.90", port: 41386}, {ip: "103.236.114.38", port: 49638}, {ip: "103.240.161.107", port: 6666}, {ip: "103.240.161.108", port: 6666}, {ip: "103.240.161.109", port: 6666}, {ip: "103.240.161.59", port: 48809}, {ip: "103.245.198.101", port: 8080}, {ip: "103.250.148.82", port: 6666}, {ip: "103.251.58.51", port: 61489}, {ip: "103.253.169.115", port: 32731}, {ip: "103.253.211.182", port: 8080}, {ip: "103.253.211.182", port: 80}, {ip: "103.255.234.169", port: 39847}, {ip: "103.42.161.118", port: 8080}, {ip: "103.42.162.30", port: 8080}, {ip: "103.42.162.50", port: 8080}, {ip: "103.42.162.58", port: 8080}, {ip: "103.46.233.12", port: 83}, {ip: "103.46.233.13", port: 83}, {ip: "103.46.233.16", port: 83}, {ip: "103.46.233.17", port: 83}, {ip: "103.46.233.21", port: 83}, {ip: "103.46.233.23", port: 83}, {ip: "103.46.233.29", port: 81}, {ip: "103.46.233.29", port: 83}, {ip: "103.46.233.50", port: 83}, {ip: "103.47.153.87", port: 8080}, {ip: "103.47.66.2", port: 39804}, {ip: "103.49.53.1", port: 81}, {ip: "103.52.220.1", port: 49068}, {ip: "103.56.228.166", port: 53281}, {ip: "103.56.30.128", port: 8080}, {ip: "103.65.193.17", port: 50862}, {ip: "103.65.195.1", port: 33960}, {ip: "103.69.220.14", port: 3128}, {ip: "103.70.128.84", port: 8080}, {ip: "103.70.128.86", port: 8080}, {ip: "103.70.131.74", port: 8080}, {ip: "103.70.146.250", port: 59563}, {ip: "103.72.216.194", port: 38345}, {ip: "103.75.161.38", port: 21776}, {ip: "103.76.253.155", port: 3128}, {ip: "103.87.104.137", port: 8080}, {ip: "110.235.198.3", port: 57660}, {ip: "114.69.229.161", port: 8080}, {ip: "117.196.231.201", port: 37769}, {ip: "117.211.166.214", port: 3128}, {ip: "117.240.175.51", port: 3128}, {ip: "117.240.210.155", port: 53281}, {ip: "117.240.59.115", port: 36127}, {ip: "117.242.154.73", port: 33889}, {ip: "117.244.15.243", port: 3128}, {ip: "119.235.54.3", port: 8080}, {ip: "120.138.117.102", port: 59308}, {ip: "123.108.200.185", port: 83}, {ip: "123.108.200.217", port: 82}, {ip: "123.176.43.218", port: 40524}, {ip: "125.21.43.82", port: 8080}, {ip: "125.62.192.225", port: 82}, {ip: "125.62.192.33", port: 84}, {ip: "125.62.194.1", port: 83}, {ip: "125.62.213.134", port: 82}, {ip: "125.62.213.18", port: 83}, {ip: "125.62.213.201", port: 84}, {ip: "125.62.213.242", port: 83}, {ip: "125.62.214.185", port: 84}, {ip: "139.5.26.27", port: 53281}, {ip: "14.102.67.101", port: 30337}, {ip: "14.142.122.134", port: 8080}, {ip: "150.129.114.194", port: 6666}, {ip: "150.129.151.62", port: 6666}, {ip: "150.129.171.115", port: 6666}, {ip: "150.129.201.30", port: 6666}, {ip: "157.119.207.38", port: 53281}, {ip: "175.100.185.151", port: 53281}, {ip: "182.18.177.114", port: 56173}, {ip: "182.73.194.170", port: 8080}, {ip: "182.74.85.230", port: 51214}, {ip: "183.82.116.56", port: 8080}, {ip: "183.82.32.56", port: 49551}, {ip: "183.87.14.229", port: 53281}, {ip: "183.87.14.250", port: 44915}, {ip: "202.134.160.168", port: 8080}, {ip: "202.134.166.1", port: 8080}, {ip: "202.134.180.50", port: 8080}, {ip: "202.62.84.210", port: 53281}, {ip: "203.192.193.225", port: 8080}, {ip: "203.192.195.14", port: 31062}, {ip: "203.192.217.11", port: 8080}, {ip: "223.196.83.182", port: 53281}, {ip: "27.116.20.169", port: 36630}, {ip: "27.116.20.209", port: 36630}, {ip: "27.116.51.21", port: 36033}, {ip: "43.224.8.114", port: 50333}, {ip: "43.224.8.116", port: 6666}, {ip: "43.224.8.124", port: 6666}, {ip: "43.224.8.86", port: 6666}, {ip: "43.225.20.73", port: 8080}, {ip: "43.225.23.26", port: 8080}, {ip: "43.230.196.98", port: 36569}, {ip: "43.240.5.225", port: 31777}, {ip: "43.241.28.248", port: 8080}, {ip: "43.242.209.201", port: 8080}, {ip: "43.246.139.82", port: 8080}, {ip: "43.248.73.86", port: 53281}, {ip: "43.251.170.145", port: 54059}, {ip: "45.112.57.230", port: 61222}, {ip: "45.115.171.30", port: 47949}, {ip: "45.121.29.254", port: 54858}, {ip: "45.123.26.146", port: 53281}, {ip: "45.125.61.193", port: 32804}, {ip: "45.125.61.209", port: 32804}, {ip: "45.127.121.194", port: 53281}, {ip: "45.250.226.14", port: 3128}, {ip: "45.250.226.38", port: 8080}, {ip: "45.250.226.47", port: 8080}, {ip: "45.250.226.55", port: 8080}, {ip: "49.249.251.86", port: 53281}], - "CN" => [{ip: "182.61.170.45", port: 3128}], - "RU" => [{ip: "109.106.139.225", port: 45689}, {ip: "109.161.48.228", port: 53281}, {ip: "109.167.224.198", port: 51919}, {ip: "109.172.57.250", port: 23500}, {ip: "109.194.2.126", port: 61822}, {ip: "109.195.150.128", port: 37564}, {ip: "109.201.96.171", port: 31773}, {ip: "109.201.97.204", port: 41258}, {ip: "109.201.97.235", port: 39125}, {ip: "109.206.140.74", port: 45991}, {ip: "109.206.148.31", port: 30797}, {ip: "109.69.75.5", port: 46347}, {ip: "109.71.181.170", port: 53983}, {ip: "109.74.132.190", port: 42663}, {ip: "109.74.143.45", port: 36529}, {ip: "109.75.140.158", port: 59916}, {ip: "109.95.84.114", port: 52125}, {ip: "130.255.12.24", port: 31004}, {ip: "134.19.147.72", port: 44812}, {ip: "134.90.181.7", port: 54353}, {ip: "145.255.6.171", port: 31252}, {ip: "146.120.227.3", port: 8080}, {ip: "149.255.112.194", port: 48968}, {ip: "158.46.127.222", port: 52574}, {ip: "158.46.43.144", port: 39120}, {ip: "158.58.130.185", port: 50016}, {ip: "158.58.132.12", port: 56962}, {ip: "158.58.133.106", port: 41258}, {ip: "158.58.133.13", port: 21213}, {ip: "176.101.0.47", port: 34471}, {ip: "176.101.89.226", port: 33470}, {ip: "176.106.12.65", port: 30120}, {ip: "176.107.80.110", port: 58901}, {ip: "176.110.121.9", port: 46322}, {ip: "176.110.121.90", port: 21776}, {ip: "176.111.97.18", port: 8080}, {ip: "176.112.106.230", port: 33996}, {ip: "176.112.110.40", port: 61142}, {ip: "176.113.116.70", port: 55589}, {ip: "176.113.27.192", port: 47337}, {ip: "176.115.197.118", port: 8080}, {ip: "176.117.255.182", port: 53100}, {ip: "176.120.200.69", port: 44331}, {ip: "176.124.123.93", port: 41258}, {ip: "176.192.124.98", port: 60787}, {ip: "176.192.5.238", port: 61227}, {ip: "176.192.8.206", port: 39422}, {ip: "176.193.15.94", port: 8080}, {ip: "176.196.195.170", port: 48129}, {ip: "176.196.198.154", port: 35252}, {ip: "176.196.238.234", port: 44648}, {ip: "176.196.239.46", port: 35656}, {ip: "176.196.246.6", port: 53281}, {ip: "176.196.84.138", port: 51336}, {ip: "176.197.145.246", port: 32649}, {ip: "176.197.99.142", port: 47278}, {ip: "176.215.1.108", port: 60339}, {ip: "176.215.170.147", port: 35604}, {ip: "176.56.23.14", port: 35340}, {ip: "176.62.185.54", port: 53883}, {ip: "176.74.13.110", port: 8080}, {ip: "178.130.29.226", port: 53295}, {ip: "178.170.254.178", port: 46788}, {ip: "178.213.13.136", port: 53281}, {ip: "178.218.104.8", port: 49707}, {ip: "178.219.183.163", port: 8080}, {ip: "178.237.180.34", port: 57307}, {ip: "178.57.101.212", port: 38020}, {ip: "178.57.101.235", port: 31309}, {ip: "178.64.190.133", port: 46688}, {ip: "178.75.1.111", port: 50411}, {ip: "178.75.27.131", port: 41879}, {ip: "185.13.35.178", port: 40654}, {ip: "185.15.189.67", port: 30215}, {ip: "185.175.119.137", port: 41258}, {ip: "185.18.111.194", port: 41258}, {ip: "185.19.176.237", port: 53281}, {ip: "185.190.40.115", port: 31747}, {ip: "185.216.195.134", port: 61287}, {ip: "185.22.172.94", port: 10010}, {ip: "185.22.172.94", port: 1448}, {ip: "185.22.174.65", port: 10010}, {ip: "185.22.174.65", port: 1448}, {ip: "185.23.64.100", port: 3130}, {ip: "185.23.82.39", port: 59248}, {ip: "185.233.94.105", port: 59288}, {ip: "185.233.94.146", port: 57736}, {ip: "185.3.68.54", port: 53500}, {ip: "185.32.120.177", port: 60724}, {ip: "185.34.20.164", port: 53700}, {ip: "185.34.23.43", port: 63238}, {ip: "185.51.60.141", port: 39935}, {ip: "185.61.92.228", port: 33060}, {ip: "185.61.93.67", port: 49107}, {ip: "185.7.233.66", port: 53504}, {ip: "185.72.225.10", port: 56285}, {ip: "185.75.5.158", port: 60819}, {ip: "185.9.86.186", port: 39345}, {ip: "188.133.136.10", port: 47113}, {ip: "188.168.75.254", port: 56899}, {ip: "188.170.41.6", port: 60332}, {ip: "188.187.189.142", port: 38264}, {ip: "188.234.151.103", port: 8080}, {ip: "188.235.11.88", port: 57143}, {ip: "188.235.137.196", port: 23500}, {ip: "188.244.175.2", port: 8080}, {ip: "188.255.82.136", port: 53281}, {ip: "188.43.4.117", port: 60577}, {ip: "188.68.95.166", port: 41258}, {ip: "188.92.242.180", port: 52048}, {ip: "188.93.242.213", port: 49774}, {ip: "192.162.193.243", port: 36910}, {ip: "192.162.214.11", port: 41258}, {ip: "193.106.170.133", port: 38591}, {ip: "193.232.113.244", port: 40412}, {ip: "193.232.234.130", port: 61932}, {ip: "193.242.177.105", port: 53281}, {ip: "193.242.178.50", port: 52376}, {ip: "193.242.178.90", port: 8080}, {ip: "193.33.101.152", port: 34611}, {ip: "194.114.128.149", port: 61213}, {ip: "194.135.15.146", port: 59328}, {ip: "194.135.216.178", port: 56805}, {ip: "194.135.75.74", port: 41258}, {ip: "194.146.201.67", port: 53281}, {ip: "194.186.18.46", port: 56408}, {ip: "194.186.20.62", port: 21231}, {ip: "194.190.171.214", port: 43960}, {ip: "194.9.27.82", port: 42720}, {ip: "195.133.232.58", port: 41733}, {ip: "195.14.114.116", port: 59530}, {ip: "195.14.114.24", port: 56897}, {ip: "195.158.250.97", port: 41582}, {ip: "195.16.48.142", port: 36083}, {ip: "195.191.183.169", port: 47238}, {ip: "195.206.45.112", port: 53281}, {ip: "195.208.172.70", port: 8080}, {ip: "195.209.141.67", port: 31927}, {ip: "195.209.176.2", port: 8080}, {ip: "195.210.144.166", port: 30088}, {ip: "195.211.160.88", port: 44464}, {ip: "195.218.144.182", port: 31705}, {ip: "195.46.168.147", port: 8080}, {ip: "195.9.188.78", port: 53281}, {ip: "195.9.209.10", port: 35242}, {ip: "195.9.223.246", port: 52098}, {ip: "195.9.237.66", port: 8080}, {ip: "195.9.91.66", port: 33199}, {ip: "195.91.132.20", port: 19600}, {ip: "195.98.183.82", port: 30953}, {ip: "212.104.82.246", port: 36495}, {ip: "212.119.229.18", port: 33852}, {ip: "212.13.97.122", port: 30466}, {ip: "212.19.21.19", port: 53264}, {ip: "212.19.5.157", port: 58442}, {ip: "212.19.8.223", port: 30281}, {ip: "212.19.8.239", port: 55602}, {ip: "212.192.202.207", port: 4550}, {ip: "212.22.80.224", port: 34822}, {ip: "212.26.247.178", port: 38418}, {ip: "212.33.228.161", port: 37971}, {ip: "212.33.243.83", port: 38605}, {ip: "212.34.53.126", port: 44369}, {ip: "212.5.107.81", port: 56481}, {ip: "212.7.230.7", port: 51405}, {ip: "212.77.138.161", port: 41258}, {ip: "213.108.221.201", port: 32800}, {ip: "213.109.7.135", port: 59918}, {ip: "213.128.9.204", port: 35549}, {ip: "213.134.196.12", port: 38723}, {ip: "213.168.37.86", port: 8080}, {ip: "213.187.118.184", port: 53281}, {ip: "213.21.23.98", port: 53281}, {ip: "213.210.67.166", port: 53281}, {ip: "213.234.0.242", port: 56503}, {ip: "213.247.192.131", port: 41258}, {ip: "213.251.226.208", port: 56900}, {ip: "213.33.155.80", port: 44387}, {ip: "213.33.199.194", port: 36411}, {ip: "213.33.224.82", port: 8080}, {ip: "213.59.153.19", port: 53281}, {ip: "217.10.45.103", port: 8080}, {ip: "217.107.197.39", port: 33628}, {ip: "217.116.60.66", port: 21231}, {ip: "217.195.87.58", port: 41258}, {ip: "217.197.239.54", port: 34463}, {ip: "217.74.161.42", port: 34175}, {ip: "217.8.84.76", port: 46378}, {ip: "31.131.67.14", port: 8080}, {ip: "31.132.127.142", port: 35432}, {ip: "31.132.218.252", port: 32423}, {ip: "31.173.17.118", port: 51317}, {ip: "31.193.124.70", port: 53281}, {ip: "31.210.211.147", port: 8080}, {ip: "31.220.183.217", port: 53281}, {ip: "31.29.212.82", port: 35066}, {ip: "31.42.254.24", port: 30912}, {ip: "31.47.189.14", port: 38473}, {ip: "37.113.129.98", port: 41665}, {ip: "37.192.103.164", port: 34835}, {ip: "37.192.194.50", port: 50165}, {ip: "37.192.99.151", port: 51417}, {ip: "37.205.83.91", port: 35888}, {ip: "37.233.85.155", port: 53281}, {ip: "37.235.167.66", port: 53281}, {ip: "37.235.65.2", port: 47816}, {ip: "37.235.67.178", port: 34450}, {ip: "37.9.134.133", port: 41262}, {ip: "46.150.174.90", port: 53281}, {ip: "46.151.156.198", port: 56013}, {ip: "46.16.226.10", port: 8080}, {ip: "46.163.131.55", port: 48306}, {ip: "46.173.191.51", port: 53281}, {ip: "46.174.222.61", port: 34977}, {ip: "46.180.96.79", port: 42319}, {ip: "46.181.151.79", port: 39386}, {ip: "46.21.74.130", port: 8080}, {ip: "46.227.162.98", port: 51558}, {ip: "46.229.187.169", port: 53281}, {ip: "46.229.67.198", port: 47437}, {ip: "46.243.179.221", port: 41598}, {ip: "46.254.217.54", port: 53281}, {ip: "46.32.68.188", port: 39707}, {ip: "46.39.224.112", port: 36765}, {ip: "46.63.162.171", port: 8080}, {ip: "46.73.33.253", port: 8080}, {ip: "5.128.32.12", port: 51959}, {ip: "5.129.155.3", port: 51390}, {ip: "5.129.16.27", port: 48935}, {ip: "5.141.81.65", port: 61853}, {ip: "5.16.15.234", port: 8080}, {ip: "5.167.51.235", port: 8080}, {ip: "5.167.96.238", port: 3128}, {ip: "5.19.165.235", port: 30793}, {ip: "5.35.93.157", port: 31773}, {ip: "5.59.137.90", port: 8888}, {ip: "5.8.207.160", port: 57192}, {ip: "62.122.97.66", port: 59143}, {ip: "62.148.151.253", port: 53570}, {ip: "62.152.85.158", port: 31156}, {ip: "62.165.54.153", port: 55522}, {ip: "62.173.140.14", port: 8080}, {ip: "62.173.155.206", port: 41258}, {ip: "62.182.206.19", port: 37715}, {ip: "62.213.14.166", port: 8080}, {ip: "62.76.123.224", port: 8080}, {ip: "77.221.220.133", port: 44331}, {ip: "77.232.153.248", port: 60950}, {ip: "77.233.10.37", port: 54210}, {ip: "77.244.27.109", port: 47554}, {ip: "77.37.142.203", port: 53281}, {ip: "77.39.29.29", port: 49243}, {ip: "77.75.6.34", port: 8080}, {ip: "77.87.102.7", port: 42601}, {ip: "77.94.121.212", port: 36896}, {ip: "77.94.121.51", port: 45293}, {ip: "78.110.154.177", port: 59888}, {ip: "78.140.201.226", port: 8090}, {ip: "78.153.4.122", port: 9001}, {ip: "78.156.225.170", port: 41258}, {ip: "78.156.243.146", port: 59730}, {ip: "78.29.14.201", port: 39001}, {ip: "78.81.24.112", port: 8080}, {ip: "78.85.36.203", port: 8080}, {ip: "79.104.219.125", port: 3128}, {ip: "79.104.55.134", port: 8080}, {ip: "79.137.181.170", port: 8080}, {ip: "79.173.124.194", port: 47832}, {ip: "79.173.124.207", port: 53281}, {ip: "79.174.186.168", port: 45710}, {ip: "79.175.51.13", port: 54853}, {ip: "79.175.57.77", port: 55477}, {ip: "80.234.107.118", port: 56952}, {ip: "80.237.6.1", port: 34880}, {ip: "80.243.14.182", port: 49320}, {ip: "80.251.48.215", port: 45157}, {ip: "80.254.121.66", port: 41055}, {ip: "80.254.125.236", port: 80}, {ip: "80.72.121.185", port: 52379}, {ip: "80.89.133.210", port: 3128}, {ip: "80.91.17.113", port: 41258}, {ip: "81.162.61.166", port: 40392}, {ip: "81.163.57.121", port: 41258}, {ip: "81.163.57.46", port: 41258}, {ip: "81.163.62.136", port: 41258}, {ip: "81.23.112.98", port: 55269}, {ip: "81.23.118.106", port: 60427}, {ip: "81.23.177.245", port: 8080}, {ip: "81.24.126.166", port: 8080}, {ip: "81.30.216.147", port: 41258}, {ip: "81.95.131.10", port: 44292}, {ip: "82.114.125.22", port: 8080}, {ip: "82.151.208.20", port: 8080}, {ip: "83.221.216.110", port: 47326}, {ip: "83.246.139.24", port: 8080}, {ip: "83.97.108.8", port: 41258}, {ip: "84.22.154.76", port: 8080}, {ip: "84.52.110.36", port: 38674}, {ip: "84.52.74.194", port: 8080}, {ip: "84.52.77.227", port: 41806}, {ip: "84.52.79.166", port: 43548}, {ip: "84.52.84.157", port: 44331}, {ip: "84.52.88.125", port: 32666}, {ip: "85.113.48.148", port: 8080}, {ip: "85.113.49.220", port: 8080}, {ip: "85.12.193.210", port: 58470}, {ip: "85.15.179.5", port: 8080}, {ip: "85.173.244.102", port: 53281}, {ip: "85.174.227.52", port: 59280}, {ip: "85.192.184.133", port: 8080}, {ip: "85.192.184.133", port: 80}, {ip: "85.21.240.193", port: 55820}, {ip: "85.21.63.219", port: 53281}, {ip: "85.235.190.18", port: 42494}, {ip: "85.237.56.193", port: 8080}, {ip: "85.91.119.6", port: 8080}, {ip: "86.102.116.30", port: 8080}, {ip: "86.110.30.146", port: 38109}, {ip: "87.117.3.129", port: 3128}, {ip: "87.225.108.195", port: 8080}, {ip: "87.228.103.111", port: 8080}, {ip: "87.228.103.43", port: 8080}, {ip: "87.229.143.10", port: 48872}, {ip: "87.249.205.103", port: 8080}, {ip: "87.249.21.193", port: 43079}, {ip: "87.255.13.217", port: 8080}, {ip: "88.147.159.167", port: 53281}, {ip: "88.200.225.32", port: 38583}, {ip: "88.204.59.177", port: 32666}, {ip: "88.84.209.69", port: 30819}, {ip: "88.87.72.72", port: 8080}, {ip: "88.87.79.20", port: 8080}, {ip: "88.87.91.163", port: 48513}, {ip: "88.87.93.20", port: 33277}, {ip: "89.109.12.82", port: 47972}, {ip: "89.109.21.43", port: 9090}, {ip: "89.109.239.183", port: 41041}, {ip: "89.109.54.137", port: 36469}, {ip: "89.17.37.218", port: 52957}, {ip: "89.189.130.103", port: 32626}, {ip: "89.189.159.214", port: 42530}, {ip: "89.189.174.121", port: 52636}, {ip: "89.23.18.29", port: 53281}, {ip: "89.249.251.21", port: 3128}, {ip: "89.250.149.114", port: 60981}, {ip: "89.250.17.209", port: 8080}, {ip: "89.250.19.173", port: 8080}, {ip: "90.150.87.172", port: 81}, {ip: "90.154.125.173", port: 33078}, {ip: "90.188.38.81", port: 60585}, {ip: "90.189.151.183", port: 32601}, {ip: "91.103.208.114", port: 57063}, {ip: "91.122.100.222", port: 44331}, {ip: "91.122.207.229", port: 8080}, {ip: "91.144.139.93", port: 3128}, {ip: "91.144.142.19", port: 44617}, {ip: "91.146.16.54", port: 57902}, {ip: "91.190.116.194", port: 38783}, {ip: "91.190.80.100", port: 31659}, {ip: "91.190.85.97", port: 34286}, {ip: "91.203.36.188", port: 8080}, {ip: "91.205.131.102", port: 8080}, {ip: "91.205.146.25", port: 37501}, {ip: "91.210.94.212", port: 52635}, {ip: "91.213.23.110", port: 8080}, {ip: "91.215.22.51", port: 53305}, {ip: "91.217.42.3", port: 8080}, {ip: "91.217.42.4", port: 8080}, {ip: "91.220.135.146", port: 41258}, {ip: "91.222.167.213", port: 38057}, {ip: "91.226.140.71", port: 33199}, {ip: "91.235.7.216", port: 59067}, {ip: "92.124.195.22", port: 3128}, {ip: "92.126.193.180", port: 8080}, {ip: "92.241.110.223", port: 53281}, {ip: "92.252.240.1", port: 53281}, {ip: "92.255.164.187", port: 3128}, {ip: "92.255.195.57", port: 53281}, {ip: "92.255.229.146", port: 55785}, {ip: "92.255.5.2", port: 41012}, {ip: "92.38.32.36", port: 56113}, {ip: "92.39.138.98", port: 31150}, {ip: "92.51.16.155", port: 46202}, {ip: "92.55.59.63", port: 33030}, {ip: "93.170.112.200", port: 47995}, {ip: "93.183.86.185", port: 53281}, {ip: "93.188.45.157", port: 8080}, {ip: "93.81.246.5", port: 53281}, {ip: "93.91.112.247", port: 41258}, {ip: "94.127.217.66", port: 40115}, {ip: "94.154.85.214", port: 8080}, {ip: "94.180.106.94", port: 32767}, {ip: "94.180.249.187", port: 38051}, {ip: "94.230.243.6", port: 8080}, {ip: "94.232.57.231", port: 51064}, {ip: "94.24.244.170", port: 48936}, {ip: "94.242.55.108", port: 10010}, {ip: "94.242.55.108", port: 1448}, {ip: "94.242.57.136", port: 10010}, {ip: "94.242.57.136", port: 1448}, {ip: "94.242.58.108", port: 10010}, {ip: "94.242.58.108", port: 1448}, {ip: "94.242.58.14", port: 10010}, {ip: "94.242.58.14", port: 1448}, {ip: "94.242.58.142", port: 10010}, {ip: "94.242.58.142", port: 1448}, {ip: "94.242.59.245", port: 10010}, {ip: "94.242.59.245", port: 1448}, {ip: "94.247.241.70", port: 53640}, {ip: "94.247.62.165", port: 33176}, {ip: "94.253.13.228", port: 54935}, {ip: "94.253.14.187", port: 55045}, {ip: "94.28.94.154", port: 46966}, {ip: "94.73.217.125", port: 40858}, {ip: "95.140.19.9", port: 8080}, {ip: "95.140.20.94", port: 33994}, {ip: "95.154.137.66", port: 41258}, {ip: "95.154.159.119", port: 44242}, {ip: "95.154.82.254", port: 52484}, {ip: "95.161.157.227", port: 43170}, {ip: "95.161.182.146", port: 33877}, {ip: "95.161.189.26", port: 61522}, {ip: "95.165.163.146", port: 8888}, {ip: "95.165.172.90", port: 60496}, {ip: "95.165.182.18", port: 38950}, {ip: "95.165.203.222", port: 33805}, {ip: "95.165.244.122", port: 58162}, {ip: "95.167.123.54", port: 58664}, {ip: "95.167.241.242", port: 49636}, {ip: "95.171.1.92", port: 35956}, {ip: "95.172.52.230", port: 35989}, {ip: "95.181.35.30", port: 40804}, {ip: "95.181.56.178", port: 39144}, {ip: "95.181.75.228", port: 53281}, {ip: "95.188.74.194", port: 57122}, {ip: "95.189.112.214", port: 35508}, {ip: "95.31.10.247", port: 30711}, {ip: "95.31.197.77", port: 41651}, {ip: "95.31.2.199", port: 33632}, {ip: "95.71.125.50", port: 49882}, {ip: "95.73.62.13", port: 32185}, {ip: "95.79.36.55", port: 44861}, {ip: "95.79.55.196", port: 53281}, {ip: "95.79.99.148", port: 3128}, {ip: "95.80.65.39", port: 43555}, {ip: "95.80.93.44", port: 41258}, {ip: "95.80.98.41", port: 8080}, {ip: "95.83.156.250", port: 58438}, {ip: "95.84.128.25", port: 33765}, {ip: "95.84.154.73", port: 57423}], - "CA" => [{ip: "144.217.161.149", port: 8080}, {ip: "24.37.9.6", port: 54154}, {ip: "54.39.138.144", port: 3128}, {ip: "54.39.138.145", port: 3128}, {ip: "54.39.138.151", port: 3128}, {ip: "54.39.138.152", port: 3128}, {ip: "54.39.138.153", port: 3128}, {ip: "54.39.138.154", port: 3128}, {ip: "54.39.138.155", port: 3128}, {ip: "54.39.138.156", port: 3128}, {ip: "54.39.138.157", port: 3128}, {ip: "54.39.53.104", port: 3128}, {ip: "66.70.167.113", port: 3128}, {ip: "66.70.167.116", port: 3128}, {ip: "66.70.167.117", port: 3128}, {ip: "66.70.167.119", port: 3128}, {ip: "66.70.167.120", port: 3128}, {ip: "66.70.167.125", port: 3128}, {ip: "66.70.188.148", port: 3128}, {ip: "70.35.213.229", port: 36127}, {ip: "70.65.233.174", port: 8080}, {ip: "72.139.24.66", port: 38861}, {ip: "74.15.191.160", port: 41564}], - "JP" => [{ip: "47.91.20.67", port: 8080}, {ip: "61.118.35.94", port: 55725}], - "IT" => [{ip: "109.70.201.97", port: 53517}, {ip: "176.31.82.212", port: 8080}, {ip: "185.132.228.118", port: 55583}, {ip: "185.49.58.88", port: 56006}, {ip: "185.94.89.179", port: 41258}, {ip: "213.203.134.10", port: 41258}, {ip: "217.61.172.12", port: 41369}, {ip: "46.232.143.126", port: 41258}, {ip: "46.232.143.253", port: 41258}, {ip: "93.67.154.125", port: 8080}, {ip: "93.67.154.125", port: 80}, {ip: "95.169.95.242", port: 53803}], - "TH" => [{ip: "1.10.184.166", port: 57330}, {ip: "1.10.186.100", port: 55011}, {ip: "1.10.186.209", port: 32431}, {ip: "1.10.186.245", port: 34360}, {ip: "1.10.186.93", port: 53711}, {ip: "1.10.187.118", port: 62000}, {ip: "1.10.187.34", port: 51635}, {ip: "1.10.187.43", port: 38715}, {ip: "1.10.188.181", port: 51093}, {ip: "1.10.188.83", port: 31940}, {ip: "1.10.188.95", port: 30593}, {ip: "1.10.189.58", port: 48564}, {ip: "1.179.157.237", port: 46178}, {ip: "1.179.164.213", port: 8080}, {ip: "1.179.198.37", port: 8080}, {ip: "1.20.100.99", port: 53794}, {ip: "1.20.101.221", port: 55707}, {ip: "1.20.101.254", port: 35394}, {ip: "1.20.101.80", port: 36234}, {ip: "1.20.102.133", port: 40296}, {ip: "1.20.103.13", port: 40544}, {ip: "1.20.103.56", port: 55422}, {ip: "1.20.96.234", port: 53142}, {ip: "1.20.97.54", port: 60122}, {ip: "1.20.99.63", port: 32123}, {ip: "101.108.92.20", port: 8080}, {ip: "101.109.143.71", port: 36127}, {ip: "101.51.141.110", port: 42860}, {ip: "101.51.141.60", port: 60417}, {ip: "103.246.17.237", port: 3128}, {ip: "110.164.73.131", port: 8080}, {ip: "110.164.87.80", port: 35844}, {ip: "110.77.134.106", port: 8080}, {ip: "113.53.29.92", port: 47297}, {ip: "113.53.83.192", port: 32780}, {ip: "113.53.83.195", port: 35686}, {ip: "113.53.91.214", port: 8080}, {ip: "115.87.27.0", port: 53276}, {ip: "118.172.211.3", port: 58535}, {ip: "118.172.211.40", port: 30430}, {ip: "118.174.196.174", port: 23500}, {ip: "118.174.196.203", port: 23500}, {ip: "118.174.220.107", port: 41222}, {ip: "118.174.220.110", port: 39025}, {ip: "118.174.220.115", port: 41011}, {ip: "118.174.220.118", port: 59556}, {ip: "118.174.220.136", port: 55041}, {ip: "118.174.220.163", port: 31561}, {ip: "118.174.220.168", port: 47455}, {ip: "118.174.220.231", port: 40924}, {ip: "118.174.220.238", port: 46326}, {ip: "118.174.234.13", port: 53084}, {ip: "118.174.234.26", port: 41926}, {ip: "118.174.234.32", port: 57403}, {ip: "118.174.234.59", port: 59149}, {ip: "118.174.234.68", port: 42626}, {ip: "118.174.234.83", port: 38006}, {ip: "118.175.207.104", port: 38959}, {ip: "118.175.244.111", port: 8080}, {ip: "118.175.93.207", port: 50738}, {ip: "122.154.38.53", port: 8080}, {ip: "122.154.59.6", port: 8080}, {ip: "122.154.72.102", port: 8080}, {ip: "122.155.222.98", port: 3128}, {ip: "124.121.22.121", port: 61699}, {ip: "125.24.156.16", port: 44321}, {ip: "125.25.165.105", port: 33850}, {ip: "125.25.165.111", port: 40808}, {ip: "125.25.165.42", port: 47221}, {ip: "125.25.201.14", port: 30100}, {ip: "125.26.99.135", port: 55637}, {ip: "125.26.99.141", port: 38537}, {ip: "125.26.99.148", port: 31818}, {ip: "134.236.247.137", port: 8080}, {ip: "159.192.98.224", port: 3128}, {ip: "171.100.2.154", port: 8080}, {ip: "171.100.9.126", port: 49163}, {ip: "180.180.156.116", port: 48431}, {ip: "180.180.156.46", port: 48507}, {ip: "180.180.156.87", port: 36628}, {ip: "180.180.218.204", port: 51565}, {ip: "180.180.8.34", port: 8080}, {ip: "182.52.238.125", port: 58861}, {ip: "182.52.74.73", port: 36286}, {ip: "182.52.74.76", port: 34084}, {ip: "182.52.74.77", port: 34825}, {ip: "182.52.74.78", port: 48708}, {ip: "182.52.90.45", port: 53799}, {ip: "182.53.206.155", port: 34307}, {ip: "182.53.206.43", port: 45330}, {ip: "182.53.206.49", port: 54228}, {ip: "183.88.212.141", port: 8080}, {ip: "183.88.212.184", port: 8080}, {ip: "183.88.213.85", port: 8080}, {ip: "183.88.214.47", port: 8080}, {ip: "184.82.128.211", port: 8080}, {ip: "202.183.201.13", port: 8081}, {ip: "202.29.20.151", port: 43083}, {ip: "203.150.172.151", port: 8080}, {ip: "27.131.157.94", port: 8080}, {ip: "27.145.100.22", port: 8080}, {ip: "27.145.100.243", port: 8080}, {ip: "49.231.196.114", port: 53281}, {ip: "58.97.72.83", port: 8080}, {ip: "61.19.145.66", port: 8080}], - "ES" => [{ip: "185.198.184.14", port: 48122}, {ip: "185.26.226.241", port: 36012}, {ip: "194.224.188.82", port: 3128}, {ip: "195.235.68.61", port: 3128}, {ip: "195.53.237.122", port: 3128}, {ip: "195.53.86.82", port: 3128}, {ip: "213.96.245.47", port: 8080}, {ip: "217.125.71.214", port: 33950}, {ip: "62.14.178.72", port: 53281}, {ip: "80.35.254.42", port: 53281}, {ip: "81.33.4.214", port: 61711}, {ip: "83.175.238.170", port: 53281}, {ip: "85.217.137.77", port: 3128}, {ip: "90.170.205.178", port: 33680}, {ip: "93.156.177.91", port: 53281}, {ip: "95.60.152.139", port: 37995}], - "AE" => [{ip: "178.32.5.90", port: 36159}], - "KR" => [{ip: "112.217.219.179", port: 3128}, {ip: "114.141.229.2", port: 58115}, {ip: "121.139.218.165", port: 31409}, {ip: "122.49.112.2", port: 38592}, {ip: "61.42.18.132", port: 53281}], - "BR" => [{ip: "128.201.97.157", port: 53281}, {ip: "128.201.97.158", port: 53281}, {ip: "131.0.246.157", port: 35252}, {ip: "131.161.26.90", port: 8080}, {ip: "131.72.143.100", port: 41396}, {ip: "138.0.24.66", port: 53281}, {ip: "138.121.130.50", port: 50600}, {ip: "138.121.155.127", port: 61932}, {ip: "138.121.32.133", port: 23492}, {ip: "138.185.176.63", port: 53281}, {ip: "138.204.233.190", port: 53281}, {ip: "138.204.233.242", port: 53281}, {ip: "138.219.71.74", port: 52688}, {ip: "138.36.107.24", port: 41184}, {ip: "138.94.115.166", port: 8080}, {ip: "143.0.188.161", port: 53281}, {ip: "143.202.218.135", port: 8080}, {ip: "143.208.2.42", port: 53281}, {ip: "143.208.79.223", port: 8080}, {ip: "143.255.52.102", port: 40687}, {ip: "143.255.52.116", port: 57856}, {ip: "143.255.52.117", port: 37279}, {ip: "144.217.22.128", port: 8080}, {ip: "168.0.8.225", port: 8080}, {ip: "168.0.8.55", port: 8080}, {ip: "168.121.139.54", port: 40056}, {ip: "168.181.168.23", port: 53281}, {ip: "168.181.170.198", port: 31935}, {ip: "168.232.198.25", port: 32009}, {ip: "168.232.198.35", port: 42267}, {ip: "168.232.207.145", port: 46342}, {ip: "170.0.104.107", port: 60337}, {ip: "170.0.112.2", port: 50359}, {ip: "170.0.112.229", port: 50359}, {ip: "170.238.118.107", port: 34314}, {ip: "170.239.144.9", port: 3128}, {ip: "170.247.29.138", port: 8080}, {ip: "170.81.237.36", port: 37124}, {ip: "170.84.51.74", port: 53281}, {ip: "170.84.60.222", port: 42981}, {ip: "177.10.202.67", port: 8080}, {ip: "177.101.60.86", port: 80}, {ip: "177.103.231.211", port: 55091}, {ip: "177.12.80.50", port: 50556}, {ip: "177.131.13.9", port: 20183}, {ip: "177.135.178.115", port: 42510}, {ip: "177.135.248.75", port: 20183}, {ip: "177.184.206.238", port: 39508}, {ip: "177.185.148.46", port: 58623}, {ip: "177.200.83.238", port: 8080}, {ip: "177.21.24.146", port: 666}, {ip: "177.220.188.120", port: 47556}, {ip: "177.220.188.213", port: 8080}, {ip: "177.222.229.243", port: 23500}, {ip: "177.234.161.42", port: 8080}, {ip: "177.36.11.241", port: 3128}, {ip: "177.36.12.193", port: 23500}, {ip: "177.37.199.175", port: 49608}, {ip: "177.39.187.70", port: 37315}, {ip: "177.44.175.199", port: 8080}, {ip: "177.46.148.126", port: 3128}, {ip: "177.46.148.142", port: 3128}, {ip: "177.47.194.98", port: 21231}, {ip: "177.5.98.58", port: 20183}, {ip: "177.52.55.19", port: 60901}, {ip: "177.54.200.66", port: 57526}, {ip: "177.55.255.74", port: 37147}, {ip: "177.67.217.94", port: 53281}, {ip: "177.73.248.6", port: 54381}, {ip: "177.73.4.234", port: 23500}, {ip: "177.75.143.211", port: 35955}, {ip: "177.75.161.206", port: 3128}, {ip: "177.75.86.49", port: 20183}, {ip: "177.8.216.106", port: 8080}, {ip: "177.8.216.114", port: 8080}, {ip: "177.8.37.247", port: 56052}, {ip: "177.84.216.17", port: 50569}, {ip: "177.85.200.254", port: 53095}, {ip: "177.87.169.1", port: 53281}, {ip: "179.107.97.178", port: 3128}, {ip: "179.109.144.25", port: 8080}, {ip: "179.109.193.137", port: 53281}, {ip: "179.189.125.206", port: 8080}, {ip: "179.97.30.46", port: 53100}, {ip: "186.192.195.220", port: 38983}, {ip: "186.193.11.226", port: 48999}, {ip: "186.193.26.106", port: 3128}, {ip: "186.208.220.248", port: 3128}, {ip: "186.209.243.142", port: 3128}, {ip: "186.209.243.233", port: 3128}, {ip: "186.211.106.227", port: 34334}, {ip: "186.211.160.178", port: 36756}, {ip: "186.215.133.170", port: 20183}, {ip: "186.216.81.21", port: 31773}, {ip: "186.219.214.13", port: 32708}, {ip: "186.224.94.6", port: 48957}, {ip: "186.225.97.246", port: 43082}, {ip: "186.226.171.163", port: 48698}, {ip: "186.226.179.2", port: 56089}, {ip: "186.226.234.67", port: 33834}, {ip: "186.228.147.58", port: 20183}, {ip: "186.233.97.163", port: 8888}, {ip: "186.248.170.82", port: 53281}, {ip: "186.249.213.101", port: 53482}, {ip: "186.249.213.65", port: 52018}, {ip: "186.250.213.225", port: 60774}, {ip: "186.250.96.70", port: 8080}, {ip: "186.250.96.77", port: 8080}, {ip: "187.1.43.246", port: 53396}, {ip: "187.108.36.250", port: 20183}, {ip: "187.108.38.10", port: 20183}, {ip: "187.109.36.251", port: 20183}, {ip: "187.109.40.9", port: 20183}, {ip: "187.109.56.101", port: 20183}, {ip: "187.111.90.89", port: 53281}, {ip: "187.115.10.50", port: 20183}, {ip: "187.19.62.7", port: 59010}, {ip: "187.33.79.61", port: 33469}, {ip: "187.35.158.150", port: 38872}, {ip: "187.44.1.167", port: 8080}, {ip: "187.45.127.87", port: 20183}, {ip: "187.45.156.109", port: 8080}, {ip: "187.5.218.215", port: 20183}, {ip: "187.58.65.225", port: 3128}, {ip: "187.63.111.37", port: 3128}, {ip: "187.72.166.10", port: 8080}, {ip: "187.73.68.14", port: 53281}, {ip: "187.84.177.6", port: 45903}, {ip: "187.84.191.170", port: 43936}, {ip: "187.87.204.210", port: 45597}, {ip: "187.87.39.247", port: 31793}, {ip: "189.1.16.162", port: 23500}, {ip: "189.113.124.162", port: 8080}, {ip: "189.124.195.185", port: 37318}, {ip: "189.3.196.18", port: 61595}, {ip: "189.37.33.59", port: 35532}, {ip: "189.7.49.66", port: 42700}, {ip: "189.90.194.35", port: 30843}, {ip: "189.90.248.75", port: 8080}, {ip: "189.91.231.43", port: 3128}, {ip: "191.239.243.156", port: 3128}, {ip: "191.240.154.246", port: 23500}, {ip: "191.240.156.154", port: 36127}, {ip: "191.240.99.142", port: 9090}, {ip: "191.241.226.230", port: 53281}, {ip: "191.241.228.74", port: 20183}, {ip: "191.241.228.78", port: 20183}, {ip: "191.241.33.238", port: 39188}, {ip: "191.241.36.170", port: 8080}, {ip: "191.241.36.218", port: 3128}, {ip: "191.242.182.132", port: 8081}, {ip: "191.243.221.130", port: 3128}, {ip: "191.255.207.231", port: 20183}, {ip: "191.36.192.196", port: 3128}, {ip: "191.36.244.230", port: 51377}, {ip: "191.5.0.79", port: 53281}, {ip: "191.6.228.6", port: 53281}, {ip: "191.7.193.18", port: 38133}, {ip: "191.7.20.134", port: 3128}, {ip: "192.140.91.173", port: 20183}, {ip: "200.150.86.138", port: 44677}, {ip: "200.155.36.185", port: 3128}, {ip: "200.155.36.188", port: 3128}, {ip: "200.155.39.41", port: 3128}, {ip: "200.174.158.26", port: 34112}, {ip: "200.187.177.105", port: 20183}, {ip: "200.187.87.138", port: 20183}, {ip: "200.192.252.201", port: 8080}, {ip: "200.192.255.102", port: 8080}, {ip: "200.203.144.2", port: 50262}, {ip: "200.229.238.42", port: 20183}, {ip: "200.233.134.85", port: 43172}, {ip: "200.233.136.177", port: 20183}, {ip: "200.241.44.3", port: 20183}, {ip: "200.255.122.170", port: 8080}, {ip: "200.255.122.174", port: 8080}, {ip: "201.12.21.57", port: 8080}, {ip: "201.131.224.21", port: 56200}, {ip: "201.182.223.16", port: 37492}, {ip: "201.20.89.126", port: 8080}, {ip: "201.22.95.10", port: 8080}, {ip: "201.57.167.34", port: 8080}, {ip: "201.59.200.246", port: 80}, {ip: "201.6.167.178", port: 3128}, {ip: "201.90.36.194", port: 3128}, {ip: "45.226.20.6", port: 8080}, {ip: "45.234.139.129", port: 20183}, {ip: "45.234.200.18", port: 53281}, {ip: "45.235.87.4", port: 51996}, {ip: "45.6.136.38", port: 53281}, {ip: "45.6.80.131", port: 52080}, {ip: "45.6.93.10", port: 8080}, {ip: "45.71.108.162", port: 53281}], - "PK" => [{ip: "103.18.243.154", port: 8080}, {ip: "110.36.218.126", port: 36651}, {ip: "110.36.234.210", port: 8080}, {ip: "110.39.162.74", port: 53281}, {ip: "110.39.174.58", port: 8080}, {ip: "111.68.108.34", port: 8080}, {ip: "125.209.116.182", port: 31653}, {ip: "125.209.78.21", port: 8080}, {ip: "125.209.82.78", port: 35087}, {ip: "180.92.156.150", port: 8080}, {ip: "202.142.158.114", port: 8080}, {ip: "202.147.173.10", port: 8080}, {ip: "202.147.173.10", port: 80}, {ip: "202.69.38.82", port: 8080}, {ip: "203.128.16.126", port: 59538}, {ip: "203.128.16.154", port: 33002}, {ip: "27.255.4.170", port: 8080}], - "ID" => [{ip: "101.128.68.113", port: 8080}, {ip: "101.255.116.113", port: 53281}, {ip: "101.255.120.170", port: 6969}, {ip: "101.255.121.74", port: 8080}, {ip: "101.255.124.242", port: 8080}, {ip: "101.255.124.242", port: 80}, {ip: "101.255.56.138", port: 53560}, {ip: "103.10.171.132", port: 41043}, {ip: "103.10.81.172", port: 80}, {ip: "103.108.158.3", port: 48196}, {ip: "103.111.219.159", port: 53281}, {ip: "103.111.54.26", port: 49781}, {ip: "103.111.54.74", port: 8080}, {ip: "103.19.110.177", port: 8080}, {ip: "103.2.146.66", port: 49089}, {ip: "103.206.168.177", port: 53281}, {ip: "103.206.253.58", port: 49573}, {ip: "103.21.92.254", port: 33929}, {ip: "103.226.49.83", port: 23500}, {ip: "103.227.147.142", port: 37581}, {ip: "103.23.101.58", port: 8080}, {ip: "103.24.107.2", port: 8181}, {ip: "103.245.19.222", port: 53281}, {ip: "103.247.122.38", port: 8080}, {ip: "103.247.218.166", port: 3128}, {ip: "103.248.219.26", port: 53634}, {ip: "103.253.2.165", port: 33543}, {ip: "103.253.2.168", port: 51229}, {ip: "103.253.2.174", port: 30827}, {ip: "103.28.114.134", port: 8080}, {ip: "103.28.220.73", port: 53281}, {ip: "103.30.246.47", port: 3128}, {ip: "103.31.45.169", port: 57655}, {ip: "103.41.122.14", port: 53281}, {ip: "103.75.101.97", port: 8080}, {ip: "103.76.17.151", port: 23500}, {ip: "103.76.50.181", port: 8080}, {ip: "103.76.50.181", port: 80}, {ip: "103.76.50.182", port: 8080}, {ip: "103.78.74.170", port: 3128}, {ip: "103.78.80.194", port: 33442}, {ip: "103.8.122.5", port: 53297}, {ip: "103.80.236.107", port: 53281}, {ip: "103.80.238.203", port: 53281}, {ip: "103.86.140.74", port: 59538}, {ip: "103.94.122.254", port: 8080}, {ip: "103.94.125.244", port: 41508}, {ip: "103.94.169.19", port: 8080}, {ip: "103.94.7.254", port: 53281}, {ip: "106.0.51.50", port: 17385}, {ip: "110.93.13.202", port: 34881}, {ip: "112.78.37.6", port: 54791}, {ip: "114.199.110.58", port: 55898}, {ip: "114.199.112.170", port: 23500}, {ip: "114.199.123.194", port: 8080}, {ip: "114.57.33.162", port: 46935}, {ip: "114.57.33.214", port: 8080}, {ip: "114.6.197.254", port: 8080}, {ip: "114.7.15.146", port: 8080}, {ip: "114.7.162.254", port: 53281}, {ip: "115.124.75.226", port: 53990}, {ip: "115.124.75.228", port: 3128}, {ip: "117.102.78.42", port: 8080}, {ip: "117.102.93.251", port: 8080}, {ip: "117.102.94.186", port: 8080}, {ip: "117.102.94.186", port: 80}, {ip: "117.103.2.249", port: 58276}, {ip: "117.54.13.174", port: 34190}, {ip: "117.74.124.129", port: 8088}, {ip: "118.97.100.83", port: 35220}, {ip: "118.97.191.162", port: 80}, {ip: "118.97.191.203", port: 8080}, {ip: "118.97.36.18", port: 8080}, {ip: "118.97.73.85", port: 53281}, {ip: "118.99.105.226", port: 8080}, {ip: "119.252.168.53", port: 53281}, {ip: "122.248.45.35", port: 53281}, {ip: "122.50.6.186", port: 8080}, {ip: "122.50.6.186", port: 80}, {ip: "123.231.226.114", port: 47562}, {ip: "123.255.202.83", port: 32523}, {ip: "124.158.164.195", port: 8080}, {ip: "124.81.99.30", port: 3128}, {ip: "137.59.162.10", port: 3128}, {ip: "139.0.29.20", port: 59532}, {ip: "139.255.123.194", port: 4550}, {ip: "139.255.16.171", port: 31773}, {ip: "139.255.17.2", port: 47421}, {ip: "139.255.19.162", port: 42371}, {ip: "139.255.7.81", port: 53281}, {ip: "139.255.91.115", port: 8080}, {ip: "139.255.92.26", port: 53281}, {ip: "158.140.181.140", port: 54041}, {ip: "160.202.40.20", port: 55655}, {ip: "175.103.42.147", port: 8080}, {ip: "180.178.98.198", port: 8080}, {ip: "180.250.101.146", port: 8080}, {ip: "182.23.107.212", port: 3128}, {ip: "182.23.2.101", port: 49833}, {ip: "182.23.7.226", port: 8080}, {ip: "182.253.209.203", port: 3128}, {ip: "183.91.66.210", port: 80}, {ip: "202.137.10.179", port: 57338}, {ip: "202.137.25.53", port: 3128}, {ip: "202.137.25.8", port: 8080}, {ip: "202.138.242.76", port: 4550}, {ip: "202.138.249.202", port: 43108}, {ip: "202.148.2.254", port: 8000}, {ip: "202.162.201.94", port: 53281}, {ip: "202.165.47.26", port: 8080}, {ip: "202.43.167.130", port: 8080}, {ip: "202.51.126.10", port: 53281}, {ip: "202.59.171.164", port: 58567}, {ip: "202.93.128.98", port: 3128}, {ip: "203.142.72.114", port: 808}, {ip: "203.153.117.65", port: 54144}, {ip: "203.189.89.1", port: 53281}, {ip: "203.77.239.18", port: 37002}, {ip: "203.99.123.25", port: 61502}, {ip: "220.247.168.163", port: 53281}, {ip: "220.247.173.154", port: 53281}, {ip: "220.247.174.206", port: 53445}, {ip: "222.124.131.211", port: 47343}, {ip: "222.124.173.146", port: 53281}, {ip: "222.124.2.131", port: 8080}, {ip: "222.124.2.186", port: 8080}, {ip: "222.124.215.187", port: 38913}, {ip: "222.124.221.179", port: 53281}, {ip: "223.25.101.242", port: 59504}, {ip: "223.25.97.62", port: 8080}, {ip: "223.25.99.38", port: 80}, {ip: "27.111.44.202", port: 80}, {ip: "27.111.47.3", port: 51144}, {ip: "36.37.124.234", port: 36179}, {ip: "36.37.124.235", port: 36179}, {ip: "36.37.81.135", port: 8080}, {ip: "36.37.89.98", port: 32323}, {ip: "36.66.217.179", port: 8080}, {ip: "36.66.98.6", port: 53281}, {ip: "36.67.143.183", port: 48746}, {ip: "36.67.206.187", port: 8080}, {ip: "36.67.32.87", port: 8080}, {ip: "36.67.93.220", port: 3128}, {ip: "36.67.93.220", port: 80}, {ip: "36.89.10.51", port: 34115}, {ip: "36.89.119.149", port: 8080}, {ip: "36.89.157.23", port: 37728}, {ip: "36.89.181.155", port: 60165}, {ip: "36.89.188.11", port: 39507}, {ip: "36.89.194.113", port: 37811}, {ip: "36.89.226.254", port: 8081}, {ip: "36.89.232.138", port: 23500}, {ip: "36.89.39.10", port: 3128}, {ip: "36.89.65.253", port: 60997}, {ip: "43.243.141.114", port: 8080}, {ip: "43.245.184.202", port: 41102}, {ip: "43.245.184.238", port: 80}, {ip: "66.96.233.225", port: 35053}, {ip: "66.96.237.253", port: 8080}], - "BD" => [{ip: "103.103.88.91", port: 8080}, {ip: "103.106.119.154", port: 8080}, {ip: "103.106.236.1", port: 8080}, {ip: "103.106.236.41", port: 8080}, {ip: "103.108.144.139", port: 53281}, {ip: "103.109.57.218", port: 8080}, {ip: "103.109.58.242", port: 8080}, {ip: "103.112.129.106", port: 31094}, {ip: "103.112.129.82", port: 53281}, {ip: "103.114.10.177", port: 8080}, {ip: "103.114.10.250", port: 8080}, {ip: "103.15.245.26", port: 8080}, {ip: "103.195.204.73", port: 21776}, {ip: "103.197.49.106", port: 49688}, {ip: "103.198.168.29", port: 21776}, {ip: "103.214.200.6", port: 59008}, {ip: "103.218.25.161", port: 8080}, {ip: "103.218.25.41", port: 8080}, {ip: "103.218.26.204", port: 8080}, {ip: "103.218.27.221", port: 8080}, {ip: "103.231.229.90", port: 53281}, {ip: "103.239.252.233", port: 8080}, {ip: "103.239.252.50", port: 8080}, {ip: "103.239.253.193", port: 8080}, {ip: "103.250.68.193", port: 51370}, {ip: "103.5.232.146", port: 8080}, {ip: "103.73.224.53", port: 23500}, {ip: "103.9.134.73", port: 65301}, {ip: "113.11.47.242", port: 40071}, {ip: "113.11.5.67", port: 40071}, {ip: "114.31.5.34", port: 52606}, {ip: "115.127.51.226", port: 42764}, {ip: "115.127.64.62", port: 39611}, {ip: "115.127.91.106", port: 8080}, {ip: "119.40.85.198", port: 36899}, {ip: "123.200.29.110", port: 23500}, {ip: "123.49.51.42", port: 55124}, {ip: "163.47.36.90", port: 3128}, {ip: "180.211.134.158", port: 23500}, {ip: "180.211.193.74", port: 40536}, {ip: "180.92.238.226", port: 53451}, {ip: "182.160.104.213", port: 8080}, {ip: "202.191.126.58", port: 23500}, {ip: "202.4.126.170", port: 8080}, {ip: "202.5.37.241", port: 33623}, {ip: "202.5.57.5", port: 61729}, {ip: "202.79.17.65", port: 60122}, {ip: "203.188.248.52", port: 23500}, {ip: "27.147.146.78", port: 52220}, {ip: "27.147.164.10", port: 52344}, {ip: "27.147.212.38", port: 53281}, {ip: "27.147.217.154", port: 43252}, {ip: "27.147.219.102", port: 49464}, {ip: "43.239.74.137", port: 8080}, {ip: "43.240.103.252", port: 8080}, {ip: "45.125.223.57", port: 8080}, {ip: "45.125.223.81", port: 8080}, {ip: "45.251.228.122", port: 41418}, {ip: "45.64.132.137", port: 8080}, {ip: "45.64.132.137", port: 80}, {ip: "61.247.186.137", port: 8080}], - "MX" => [{ip: "148.217.94.54", port: 3128}, {ip: "177.244.28.77", port: 53281}, {ip: "187.141.73.147", port: 53281}, {ip: "187.185.15.35", port: 53281}, {ip: "187.188.46.172", port: 53455}, {ip: "187.216.83.185", port: 8080}, {ip: "187.216.90.46", port: 53281}, {ip: "187.243.253.182", port: 33796}, {ip: "189.195.132.86", port: 43286}, {ip: "189.204.158.161", port: 8080}, {ip: "200.79.180.115", port: 8080}, {ip: "201.140.113.90", port: 37193}, {ip: "201.144.14.229", port: 53281}, {ip: "201.163.73.93", port: 53281}], - "PH" => [{ip: "103.86.187.242", port: 23500}, {ip: "122.54.101.69", port: 8080}, {ip: "122.54.65.150", port: 8080}, {ip: "125.5.20.134", port: 53281}, {ip: "146.88.77.51", port: 8080}, {ip: "182.18.200.92", port: 8080}, {ip: "219.90.87.91", port: 53281}, {ip: "58.69.12.210", port: 8080}], - "EG" => [{ip: "41.65.0.167", port: 8080}], - "VN" => [{ip: "1.55.240.156", port: 53281}, {ip: "101.99.23.136", port: 3128}, {ip: "103.15.51.160", port: 8080}, {ip: "113.161.128.169", port: 60427}, {ip: "113.161.161.143", port: 57967}, {ip: "113.161.173.10", port: 3128}, {ip: "113.161.35.108", port: 30028}, {ip: "113.164.79.177", port: 46281}, {ip: "113.190.235.50", port: 34619}, {ip: "115.78.160.247", port: 8080}, {ip: "117.2.155.29", port: 47228}, {ip: "117.2.17.26", port: 53281}, {ip: "117.2.22.41", port: 41973}, {ip: "117.4.145.16", port: 51487}, {ip: "118.69.219.185", port: 55184}, {ip: "118.69.61.212", port: 53281}, {ip: "118.70.116.227", port: 61651}, {ip: "118.70.219.124", port: 53281}, {ip: "221.121.12.238", port: 36077}, {ip: "27.2.7.59", port: 52148}], - "CD" => [{ip: "41.79.233.45", port: 8080}], - "TR" => [{ip: "151.80.65.175", port: 3128}, {ip: "176.235.186.242", port: 37043}, {ip: "178.250.92.18", port: 8080}, {ip: "185.203.170.92", port: 8080}, {ip: "185.203.170.94", port: 8080}, {ip: "185.203.170.95", port: 8080}, {ip: "185.51.36.152", port: 41258}, {ip: "195.137.223.50", port: 41336}, {ip: "195.155.98.70", port: 52598}, {ip: "212.156.146.22", port: 40080}, {ip: "213.14.31.122", port: 44621}, {ip: "31.145.137.139", port: 31871}, {ip: "31.145.138.129", port: 31871}, {ip: "31.145.138.146", port: 34159}, {ip: "31.145.187.172", port: 30636}, {ip: "78.188.4.124", port: 34514}, {ip: "88.248.23.216", port: 36426}, {ip: "93.182.72.36", port: 8080}, {ip: "95.0.194.241", port: 9090}], -} diff --git a/src/invidious/yt_backend/url_sanitizer.cr b/src/invidious/yt_backend/url_sanitizer.cr new file mode 100644 index 00000000..d539dadb --- /dev/null +++ b/src/invidious/yt_backend/url_sanitizer.cr @@ -0,0 +1,121 @@ +require "uri" + +module UrlSanitizer + extend self + + ALLOWED_QUERY_PARAMS = { + channel: ["u", "user", "lb"], + playlist: ["list"], + search: ["q", "search_query", "sp"], + watch: [ + "v", # Video ID + "list", "index", # Playlist-related + "playlist", # Unnamed playlist (id,id,id,...) (embed-only?) + "t", "time_continue", "start", "end", # Timestamp + "lc", # Highlighted comment (watch page only) + ], + } + + # Returns whether the given string is an ASCII word. This is the same as + # running the following regex in US-ASCII locale: /^[\w-]+$/ + private def ascii_word?(str : String) : Bool + return false if str.bytesize != str.size + + str.each_byte do |byte| + next if 'a'.ord <= byte <= 'z'.ord + next if 'A'.ord <= byte <= 'Z'.ord + next if '0'.ord <= byte <= '9'.ord + next if byte == '-'.ord || byte == '_'.ord + + return false + end + + return true + end + + # Return which kind of parameters are allowed based on the + # first path component (breadcrumb 0). + private def determine_allowed(path_root : String) + case path_root + when "watch", "w", "v", "embed", "e", "shorts", "clip" + return :watch + when .starts_with?("@"), "c", "channel", "user", "profile", "attribution_link" + return :channel + when "playlist", "mix" + return :playlist + when "results", "search" + return :search + else # hashtag, post, trending, brand URLs, etc.. + return nil + end + end + + # Create a new URI::Param containing only the allowed parameters + private def copy_params(unsafe_params : URI::Params, allowed_type) : URI::Params + new_params = URI::Params.new + + ALLOWED_QUERY_PARAMS[allowed_type].each do |name| + if unsafe_params[name]? + # Only copy the last parameter, in case there is more than one + new_params[name] = unsafe_params.fetch_all(name)[-1] + end + end + + return new_params + end + + # Transform any user-supplied youtube URL into something we can trust + # and use across the code. + def process(str : String) : URI + # Because URI follows RFC3986 specifications, URL without a scheme + # will be parsed as a relative path. So we have to add a scheme ourselves. + str = "https://#{str}" if !str.starts_with?(/https?:\/\//) + + unsafe_uri = URI.parse(str) + unsafe_host = unsafe_uri.host + unsafe_path = unsafe_uri.path + + new_uri = URI.new(path: "/") + + # Redirect to homepage for bogus URLs + return new_uri if (unsafe_host.nil? || unsafe_path.nil?) + + breadcrumbs = unsafe_path + .split('/', remove_empty: true) + .compact_map do |bc| + # Exclude attempts at path trasversal + next if bc == "." || bc == ".." + + # Non-alnum characters are unlikely in a genuine URL + next if !ascii_word?(bc) + + bc + end + + # If nothing remains, it's either a legit URL to the homepage + # (who does that!?) or because we filtered some junk earlier. + return new_uri if breadcrumbs.empty? + + # Replace the original query parameters with the sanitized ones + case unsafe_host + when .ends_with?("youtube.com") + # Use our sanitized path (not forgetting the leading '/') + new_uri.path = "/#{breadcrumbs.join('/')}" + + # Then determine which params are allowed, and copy them over + if allowed = determine_allowed(breadcrumbs[0]) + new_uri.query_params = copy_params(unsafe_uri.query_params, allowed) + end + when "youtu.be" + # Always redirect to the watch page + new_uri.path = "/watch" + + new_params = copy_params(unsafe_uri.query_params, :watch) + new_params["v"] = breadcrumbs[0] + + new_uri.query_params = new_params + end + + return new_uri + end +end diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index a5e621f2..ec080d8c 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -5,19 +5,21 @@ module YoutubeAPI extend self - private DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" - - private ANDROID_APP_VERSION = "18.20.38" - # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1308 - private ANDROID_USER_AGENT = "com.google.android.youtube/18.20.38 (Linux; U; Android 12; US) gzip" - private ANDROID_SDK_VERSION = 31_i64 + # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history + private ANDROID_APP_VERSION = "19.32.34" private ANDROID_VERSION = "12" + private ANDROID_USER_AGENT = "com.google.android.youtube/#{ANDROID_APP_VERSION} (Linux; U; Android #{ANDROID_VERSION}; US) gzip" + private ANDROID_SDK_VERSION = 31_i64 + + private ANDROID_TS_APP_VERSION = "1.9" + private ANDROID_TS_USER_AGENT = "com.google.android.youtube/1.9 (Linux; U; Android 12; US) gzip" - private IOS_APP_VERSION = "18.21.3" - # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1330 - private IOS_USER_AGENT = "com.google.ios.youtube/18.21.3 (iPhone14,5; U; CPU iOS 15_6 like Mac OS X;)" - # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1224 - private IOS_VERSION = "15.6.0.19G71" + # For Apple device names, see https://gist.github.com/adamawolf/3048717 + # For iOS versions, see https://en.wikipedia.org/wiki/IOS_version_history#Releases, + # then go to the dedicated article of the major version you want. + private IOS_APP_VERSION = "19.32.8" + private IOS_USER_AGENT = "com.google.ios.youtube/#{IOS_APP_VERSION} (iPhone14,5; U; CPU iOS 17_6 like Mac OS X;)" + private IOS_VERSION = "17.6.1.21G93" # Major.Minor.Patch.Build private WINDOWS_VERSION = "10.0" @@ -27,10 +29,12 @@ module YoutubeAPI WebEmbeddedPlayer WebMobile WebScreenEmbed + WebCreator Android AndroidEmbeddedPlayer AndroidScreenEmbed + AndroidTestSuite IOS IOSEmbedded @@ -45,8 +49,7 @@ module YoutubeAPI ClientType::Web => { name: "WEB", name_proto: "1", - version: "2.20230602.01.00", - api_key: DEFAULT_API_KEY, + version: "2.20240814.00.00", screen: "WATCH_FULL_SCREEN", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -55,8 +58,7 @@ module YoutubeAPI ClientType::WebEmbeddedPlayer => { name: "WEB_EMBEDDED_PLAYER", name_proto: "56", - version: "1.20220803.01.00", - api_key: DEFAULT_API_KEY, + version: "1.20240812.01.00", screen: "EMBED", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -65,8 +67,7 @@ module YoutubeAPI ClientType::WebMobile => { name: "MWEB", name_proto: "2", - version: "2.20230531.05.00", - api_key: DEFAULT_API_KEY, + version: "2.20240813.02.00", os_name: "Android", os_version: ANDROID_VERSION, platform: "MOBILE", @@ -74,13 +75,20 @@ module YoutubeAPI ClientType::WebScreenEmbed => { name: "WEB", name_proto: "1", - version: "2.20220804.00.00", - api_key: DEFAULT_API_KEY, + version: "2.20240814.00.00", screen: "EMBED", os_name: "Windows", os_version: WINDOWS_VERSION, platform: "DESKTOP", }, + ClientType::WebCreator => { + name: "WEB_CREATOR", + name_proto: "62", + version: "1.20240918.03.00", + os_name: "Windows", + os_version: WINDOWS_VERSION, + platform: "DESKTOP", + }, # Android @@ -88,7 +96,6 @@ module YoutubeAPI name: "ANDROID", name_proto: "3", version: ANDROID_APP_VERSION, - api_key: "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w", android_sdk_version: ANDROID_SDK_VERSION, user_agent: ANDROID_USER_AGENT, os_name: "Android", @@ -99,13 +106,11 @@ module YoutubeAPI name: "ANDROID_EMBEDDED_PLAYER", name_proto: "55", version: ANDROID_APP_VERSION, - api_key: DEFAULT_API_KEY, }, ClientType::AndroidScreenEmbed => { name: "ANDROID", name_proto: "3", version: ANDROID_APP_VERSION, - api_key: DEFAULT_API_KEY, screen: "EMBED", android_sdk_version: ANDROID_SDK_VERSION, user_agent: ANDROID_USER_AGENT, @@ -113,6 +118,16 @@ module YoutubeAPI os_version: ANDROID_VERSION, platform: "MOBILE", }, + ClientType::AndroidTestSuite => { + name: "ANDROID_TESTSUITE", + name_proto: "30", + version: ANDROID_TS_APP_VERSION, + android_sdk_version: ANDROID_SDK_VERSION, + user_agent: ANDROID_TS_USER_AGENT, + os_name: "Android", + os_version: ANDROID_VERSION, + platform: "MOBILE", + }, # IOS @@ -120,7 +135,6 @@ module YoutubeAPI name: "IOS", name_proto: "5", version: IOS_APP_VERSION, - api_key: "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc", user_agent: IOS_USER_AGENT, device_make: "Apple", device_model: "iPhone14,5", @@ -132,7 +146,6 @@ module YoutubeAPI name: "IOS_MESSAGES_EXTENSION", name_proto: "66", version: IOS_APP_VERSION, - api_key: DEFAULT_API_KEY, user_agent: IOS_USER_AGENT, device_make: "Apple", device_model: "iPhone14,5", @@ -143,9 +156,8 @@ module YoutubeAPI ClientType::IOSMusic => { name: "IOS_MUSIC", name_proto: "26", - version: "5.21", - api_key: "AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s", - user_agent: "com.google.ios.youtubemusic/5.21 (iPhone14,5; U; CPU iOS 15_6 like Mac OS X;)", + version: "7.14", + user_agent: "com.google.ios.youtubemusic/7.14 (iPhone14,5; U; CPU iOS 17_6 like Mac OS X;)", device_make: "Apple", device_model: "iPhone14,5", os_name: "iPhone", @@ -158,14 +170,12 @@ module YoutubeAPI ClientType::TvHtml5 => { name: "TVHTML5", name_proto: "7", - version: "7.20220325", - api_key: DEFAULT_API_KEY, + version: "7.20240813.07.00", }, ClientType::TvHtml5ScreenEmbed => { name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER", name_proto: "85", version: "2.0", - api_key: DEFAULT_API_KEY, screen: "EMBED", }, } @@ -187,10 +197,6 @@ module YoutubeAPI # conf_2 = ClientConfig.new(client_type: ClientType::Android) # YoutubeAPI::player(video_id: "dQw4w9WgXcQ", client_config: conf_2) # - # # Proxy request through russian proxies - # conf_3 = ClientConfig.new(proxy_region: "RU") - # YoutubeAPI::next({video_id: "dQw4w9WgXcQ"}, client_config: conf_3) - # ``` # struct ClientConfig # Type of client to emulate. @@ -201,16 +207,11 @@ module YoutubeAPI # (this is passed as the `gl` parameter). property region : String | Nil - # ISO code of country where the proxy is located. - # Used in case of geo-restricted videos. - property proxy_region : String | Nil - # Initialization function def initialize( *, @client_type = ClientType::Web, @region = "US", - @proxy_region = nil ) end @@ -230,11 +231,6 @@ module YoutubeAPI end # :ditto: - def api_key : String - HARDCODED_CLIENTS[@client_type][:api_key] - end - - # :ditto: def screen : String HARDCODED_CLIENTS[@client_type][:screen]? || "" end @@ -270,9 +266,8 @@ module YoutubeAPI # Convert to string, for logging purposes def to_s return { - client_type: self.name, - region: @region, - proxy_region: @proxy_region, + client_type: self.name, + region: @region, }.to_s end end @@ -286,7 +281,7 @@ module YoutubeAPI # Return, as a Hash, the "context" data required to request the # youtube API endpoints. # - private def make_context(client_config : ClientConfig | Nil) : Hash + private def make_context(client_config : ClientConfig | Nil, video_id = "dQw4w9WgXcQ") : Hash # Use the default client config if nil is passed client_config ||= DEFAULT_CLIENT_CONFIG @@ -306,7 +301,7 @@ module YoutubeAPI if client_config.screen == "EMBED" client_context["thirdParty"] = { - "embedUrl" => "https://www.youtube.com/embed/dQw4w9WgXcQ", + "embedUrl" => "https://www.youtube.com/embed/#{video_id}", } of String => String | Int64 end @@ -334,6 +329,10 @@ module YoutubeAPI client_context["client"]["platform"] = platform end + if CONFIG.visitor_data.is_a?(String) + client_context["client"]["visitorData"] = CONFIG.visitor_data.as(String) + end + return client_context end @@ -371,7 +370,7 @@ module YoutubeAPI browse_id : String, *, # Force the following parameters to be passed by name params : String, - client_config : ClientConfig | Nil = nil + client_config : ClientConfig | Nil = nil, ) # JSON Request data, required by the API data = { @@ -465,21 +464,34 @@ module YoutubeAPI video_id : String, *, # Force the following parameters to be passed by name params : String, - client_config : ClientConfig | Nil = nil + client_config : ClientConfig | Nil = nil, ) + # Playback context, separate because it can be different between clients + playback_ctx = { + "html5Preference" => "HTML5_PREF_WANTS", + "referer" => "https://www.youtube.com/watch?v=#{video_id}", + } of String => String | Int64 + + if {"WEB", "TVHTML5"}.any? { |s| client_config.name.starts_with? s } + if sts = DECRYPT_FUNCTION.try &.get_sts + playback_ctx["signatureTimestamp"] = sts.to_i64 + end + end + # JSON Request data, required by the API data = { "contentCheckOk" => true, "videoId" => video_id, - "context" => self.make_context(client_config), + "context" => self.make_context(client_config, video_id), "racyCheckOk" => true, "user" => { "lockedSafetyMode" => false, }, "playbackContext" => { - "contentPlaybackContext" => { - "html5Preference": "HTML5_PREF_WANTS", - }, + "contentPlaybackContext" => playback_ctx, + }, + "serviceIntegrityDimensions" => { + "poToken" => CONFIG.po_token, }, } @@ -545,7 +557,7 @@ module YoutubeAPI def search( search_query : String, params : String, - client_config : ClientConfig | Nil = nil + client_config : ClientConfig | Nil = nil, ) # JSON Request data, required by the API data = { @@ -571,7 +583,7 @@ module YoutubeAPI def get_transcript( params : String, - client_config : ClientConfig | Nil = nil + client_config : ClientConfig | Nil = nil, ) : Hash(String, JSON::Any) data = { "context" => self.make_context(client_config), @@ -593,13 +605,13 @@ module YoutubeAPI def _post_json( endpoint : String, data : Hash, - client_config : ClientConfig | Nil + client_config : ClientConfig | Nil, ) : Hash(String, JSON::Any) # Use the default client config if nil is passed client_config ||= DEFAULT_CLIENT_CONFIG # Query parameters - url = "#{endpoint}?key=#{client_config.api_key}&prettyPrint=false" + url = "#{endpoint}?prettyPrint=false" headers = HTTP::Headers{ "Content-Type" => "application/json; charset=UTF-8", @@ -613,14 +625,23 @@ module YoutubeAPI headers["User-Agent"] = user_agent end + if CONFIG.visitor_data.is_a?(String) + headers["X-Goog-Visitor-Id"] = CONFIG.visitor_data.as(String) + end + # Logging LOGGER.debug("YoutubeAPI: Using endpoint: \"#{endpoint}\"") LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}") LOGGER.trace("YoutubeAPI: POST data: #{data}") # Send the POST request - body = YT_POOL.client(client_config.proxy_region) do |client| + body = YT_POOL.client() do |client| client.post(url, headers: headers, body: data.to_json) do |response| + if response.status_code != 200 + raise InfoException.new("Error: non 200 status code. Youtube API returned \ + status code #{response.status_code}. See <a href=\"https://docs.invidious.io/youtube-errors-explained/\"> \ + https://docs.invidious.io/youtube-errors-explained/</a> for troubleshooting.") + end self._decompress(response.body_io, response.headers["Content-Encoding"]?) end end |