diff options
Diffstat (limited to 'src')
47 files changed, 621 insertions, 1105 deletions
diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index 8b60a728..b5a27667 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -14,6 +14,7 @@ record AboutChannel, is_family_friendly : Bool, allowed_regions : Array(String), tabs : Array(String), + tags : Array(String), verified : Bool def get_about_info(ucid, locale) : AboutChannel @@ -43,6 +44,8 @@ def get_about_info(ucid, locale) : AboutChannel auto_generated = true end + tags = [] of String + if auto_generated author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s @@ -52,7 +55,13 @@ def get_about_info(ucid, locale) : AboutChannel banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? banner = banners.try &.[-1]?.try &.["url"].as_s? - description_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] + description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] + # some channels have the description in a simpleText + # ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/ + description_node = description_base_node.dig?("simpleText") || description_base_node + + tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges") + .try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String else author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s @@ -70,6 +79,7 @@ def get_about_info(ucid, locale) : AboutChannel # end description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]? + tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String end is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool @@ -131,7 +141,8 @@ def get_about_info(ucid, locale) : AboutChannel # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] auto_generated = ( (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \ - extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" + extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" || + channel_about_meta.dig?("links", 0, "channelExternalLinkViewModel", "title", "content").try &.as_s == "Auto-generated by YouTube" ) end end @@ -155,6 +166,7 @@ def get_about_info(ucid, locale) : AboutChannel is_family_friendly: is_family_friendly, allowed_regions: allowed_regions, tabs: tab_names, + tags: tags, verified: author_verified || false, ) end diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index c3d6124f..be739673 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -93,7 +93,7 @@ struct ChannelVideo def to_tuple {% begin %} { - {{*@type.instance_vars.map(&.name)}} + {{@type.instance_vars.map(&.name).splat}} } {% end %} end diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index beb86e08..6cc30142 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -1,4 +1,4 @@ -def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) +def produce_channel_content_continuation(ucid, content_type, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) object_inner_2 = { "2:0:embedded" => { "1:0:varint" => 0_i64, @@ -16,6 +16,13 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } + content_type_numerical = + case content_type + when "videos" then 15 + when "livestreams" then 14 + else 15 # Fallback to "videos" + end + sort_by_numerical = case sort_by when "newest" then 1_i64 @@ -27,7 +34,7 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so object_inner_1 = { "110:embedded" => { "3:embedded" => { - "15:embedded" => { + "#{content_type_numerical}:embedded" => { "1:embedded" => { "1:string" => object_inner_2_encoded, }, @@ -62,10 +69,8 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so return continuation end -# Used in bypass_captcha_job.cr -def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2) - return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" +def make_initial_content_ctoken(ucid, content_type, sort_by) : String + return produce_channel_content_continuation(ucid, content_type, sort_by: sort_by) end module Invidious::Channel::Tabs @@ -75,10 +80,6 @@ module Invidious::Channel::Tabs # Regular videos # ------------------- - def make_initial_video_ctoken(ucid, sort_by) : String - return produce_channel_videos_continuation(ucid, sort_by: sort_by) - end - # Wrapper for AboutChannel, as we still need to call get_videos with # an author name and ucid directly (e.g in RSS feeds). # TODO: figure out how to get rid of that @@ -100,7 +101,7 @@ module Invidious::Channel::Tabs end def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest") - continuation ||= make_initial_video_ctoken(ucid, sort_by) + continuation ||= make_initial_content_ctoken(ucid, "videos", sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, author, ucid) @@ -144,21 +145,18 @@ module Invidious::Channel::Tabs # Livestreams # ------------------- - def get_livestreams(channel : AboutChannel, continuation : String? = nil) - if continuation.nil? - # EgdzdHJlYW1z8gYECgJ6AA%3D%3D is the protobuf object to load "streams" - initial_data = YoutubeAPI.browse(channel.ucid, params: "EgdzdHJlYW1z8gYECgJ6AA%3D%3D") - else - initial_data = YoutubeAPI.browse(continuation: continuation) - end + def get_livestreams(channel : AboutChannel, continuation : String? = nil, sort_by = "newest") + continuation ||= make_initial_content_ctoken(channel.ucid, "livestreams", sort_by) + + initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, channel.author, channel.ucid) end - def get_60_livestreams(channel : AboutChannel, continuation : String? = nil) + def get_60_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") if continuation.nil? - # Fetch the first "page" of streams - items, next_continuation = get_livestreams(channel) + # Fetch the first "page" of stream + items, next_continuation = get_livestreams(channel, sort_by: sort_by) else # Fetch a "page" of streams using the given continuation token items, next_continuation = get_livestreams(channel, continuation: continuation) diff --git a/src/invidious/comments/content.cr b/src/invidious/comments/content.cr index c8cdc2df..beefd9ad 100644 --- a/src/invidious/comments/content.cr +++ b/src/invidious/comments/content.cr @@ -64,15 +64,15 @@ def content_to_comment_html(content, video_id : String? = "") # check for custom emojis if run["emoji"]? if run["emoji"]["isCustomEmoji"]?.try &.as_bool - if emojiImage = run.dig?("emoji", "image") - emojiAlt = emojiImage.dig?("accessibility", "accessibilityData", "label").try &.as_s || text - emojiThumb = emojiImage["thumbnails"][0] + if emoji_image = run.dig?("emoji", "image") + emoji_alt = emoji_image.dig?("accessibility", "accessibilityData", "label").try &.as_s || text + emoji_thumb = emoji_image["thumbnails"][0] text = String.build do |str| - str << %(<img alt=") << emojiAlt << "\" " - str << %(src="/ggpht) << URI.parse(emojiThumb["url"].as_s).request_target << "\" " - str << %(title=") << emojiAlt << "\" " - str << %(width=") << emojiThumb["width"] << "\" " - str << %(height=") << emojiThumb["height"] << "\" " + str << %(<img alt=") << emoji_alt << "\" " + str << %(src="/ggpht) << URI.parse(emoji_thumb["url"].as_s).request_target << "\" " + str << %(title=") << emoji_alt << "\" " + str << %(width=") << emoji_thumb["width"] << "\" " + str << %(height=") << emoji_thumb["height"] << "\" " str << %(class="channel-emoji" />) end else diff --git a/src/invidious/comments/youtube.cr b/src/invidious/comments/youtube.cr index 185d8e43..0716fcde 100644 --- a/src/invidious/comments/youtube.cr +++ b/src/invidious/comments/youtube.cr @@ -57,7 +57,7 @@ module Invidious::Comments return initial_data end - def parse_youtube(id, response, format, locale, thin_mode, sort_by = "top", isPost = false) + def parse_youtube(id, response, format, locale, thin_mode, sort_by = "top", is_post = false) contents = nil if on_response_received_endpoints = response["onResponseReceivedEndpoints"]? @@ -104,6 +104,8 @@ module Invidious::Comments end end + mutations = response.dig?("frameworkUpdates", "entityBatchUpdate", "mutations").try &.as_a || [] of JSON::Any + response = JSON.build do |json| json.object do if header @@ -113,7 +115,7 @@ module Invidious::Comments json.field "commentCount", comment_count end - if isPost + if is_post json.field "postId", id else json.field "videoId", id @@ -131,73 +133,138 @@ module Invidious::Comments node_replies = node["replies"]["commentRepliesRenderer"] end - if node["comment"]? - node_comment = node["comment"]["commentRenderer"] - else - node_comment = node["commentRenderer"] - end + if cvm = node["commentViewModel"]? + # two commentViewModels for inital request + # one commentViewModel when getting a replies to a comment + cvm = cvm["commentViewModel"] if cvm["commentViewModel"]? + + comment_key = cvm["commentKey"] + toolbar_key = cvm["toolbarStateKey"] + comment_mutation = mutations.find { |i| i.dig?("payload", "commentEntityPayload", "key") == comment_key } + toolbar_mutation = mutations.find { |i| i.dig?("entityKey") == toolbar_key } + + if !comment_mutation.nil? && !toolbar_mutation.nil? + # todo parse styleRuns, commandRuns and attachmentRuns for comments + html_content = parse_description(comment_mutation.dig("payload", "commentEntityPayload", "properties", "content"), id) + comment_author = comment_mutation.dig("payload", "commentEntityPayload", "author") + json.field "authorId", comment_author["channelId"].as_s + json.field "authorUrl", "/channel/#{comment_author["channelId"].as_s}" + json.field "author", comment_author["displayName"].as_s + json.field "verified", comment_author["isVerified"].as_bool + json.field "authorThumbnails" do + json.array do + comment_mutation.dig?("payload", "commentEntityPayload", "avatar", "image", "sources").try &.as_a.each do |thumbnail| + json.object do + json.field "url", thumbnail["url"] + json.field "width", thumbnail["width"] + json.field "height", thumbnail["height"] + end + end + end + end - content_html = node_comment["contentText"]?.try { |t| parse_content(t, id) } || "" - author = node_comment["authorText"]?.try &.["simpleText"]? || "" + json.field "authorIsChannelOwner", comment_author["isCreator"].as_bool + json.field "isSponsor", (comment_author["sponsorBadgeUrl"]? != nil) - json.field "verified", (node_comment["authorCommentBadge"]? != nil) + if sponsor_badge_url = comment_author["sponsorBadgeUrl"]? + # Sponsor icon thumbnails always have one object and there's only ever the url property in it + json.field "sponsorIconUrl", sponsor_badge_url + end - json.field "author", author - json.field "authorThumbnails" do - json.array do - node_comment["authorThumbnail"]["thumbnails"].as_a.each do |thumbnail| - json.object do - json.field "url", thumbnail["url"] - json.field "width", thumbnail["width"] - json.field "height", thumbnail["height"] + comment_toolbar = comment_mutation.dig("payload", "commentEntityPayload", "toolbar") + json.field "likeCount", short_text_to_number(comment_toolbar["likeCountNotliked"].as_s) + reply_count = short_text_to_number(comment_toolbar["replyCount"]?.try &.as_s || "0") + + if heart_state = toolbar_mutation.dig?("payload", "engagementToolbarStateEntityPayload", "heartState") + if heart_state.as_s == "TOOLBAR_HEART_STATE_HEARTED" + json.field "creatorHeart" do + json.object do + json.field "creatorThumbnail", comment_toolbar["creatorThumbnailUrl"].as_s + json.field "creatorName", comment_toolbar["heartActiveTooltip"].as_s.sub("❤ by ", "") + end + end end end + + published_text = comment_mutation.dig?("payload", "commentEntityPayload", "properties", "publishedTime").try &.as_s end - end - if node_comment["authorEndpoint"]? - json.field "authorId", node_comment["authorEndpoint"]["browseEndpoint"]["browseId"] - json.field "authorUrl", node_comment["authorEndpoint"]["browseEndpoint"]["canonicalBaseUrl"] + json.field "isPinned", (cvm.dig?("pinnedText") != nil) + json.field "commentId", cvm["commentId"] else - json.field "authorId", "" - json.field "authorUrl", "" - end + if node["comment"]? + node_comment = node["comment"]["commentRenderer"] + else + node_comment = node["commentRenderer"] + end + json.field "commentId", node_comment["commentId"] + html_content = node_comment["contentText"]?.try { |t| parse_content(t, id) } + + json.field "verified", (node_comment["authorCommentBadge"]? != nil) + + json.field "author", node_comment["authorText"]?.try &.["simpleText"]? || "" + json.field "authorThumbnails" do + json.array do + node_comment["authorThumbnail"]["thumbnails"].as_a.each do |thumbnail| + json.object do + json.field "url", thumbnail["url"] + json.field "width", thumbnail["width"] + json.field "height", thumbnail["height"] + end + end + end + end + + if comment_action_buttons_renderer = node_comment.dig?("actionButtons", "commentActionButtonsRenderer") + json.field "likeCount", comment_action_buttons_renderer["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"].as_s.scan(/\d/).map(&.[0]).join.to_i + if comment_action_buttons_renderer["creatorHeart"]? + heart_data = comment_action_buttons_renderer["creatorHeart"]["creatorHeartRenderer"]["creatorThumbnail"] + json.field "creatorHeart" do + json.object do + json.field "creatorThumbnail", heart_data["thumbnails"][-1]["url"] + json.field "creatorName", heart_data["accessibility"]["accessibilityData"]["label"] + end + end + end + end - published_text = node_comment["publishedTimeText"]["runs"][0]["text"].as_s - published = decode_date(published_text.rchop(" (edited)")) + if node_comment["authorEndpoint"]? + json.field "authorId", node_comment["authorEndpoint"]["browseEndpoint"]["browseId"] + json.field "authorUrl", node_comment["authorEndpoint"]["browseEndpoint"]["canonicalBaseUrl"] + else + json.field "authorId", "" + json.field "authorUrl", "" + end - if published_text.includes?(" (edited)") - json.field "isEdited", true - else - json.field "isEdited", false - end + json.field "authorIsChannelOwner", node_comment["authorIsChannelOwner"] + json.field "isPinned", (node_comment["pinnedCommentBadge"]? != nil) + published_text = node_comment["publishedTimeText"]["runs"][0]["text"].as_s - json.field "content", html_to_content(content_html) - json.field "contentHtml", content_html + json.field "isSponsor", (node_comment["sponsorCommentBadge"]? != nil) + if node_comment["sponsorCommentBadge"]? + # Sponsor icon thumbnails always have one object and there's only ever the url property in it + json.field "sponsorIconUrl", node_comment.dig("sponsorCommentBadge", "sponsorCommentBadgeRenderer", "customBadge", "thumbnails", 0, "url").to_s + end - json.field "isPinned", (node_comment["pinnedCommentBadge"]? != nil) - json.field "isSponsor", (node_comment["sponsorCommentBadge"]? != nil) - if node_comment["sponsorCommentBadge"]? - # Sponsor icon thumbnails always have one object and there's only ever the url property in it - json.field "sponsorIconUrl", node_comment.dig("sponsorCommentBadge", "sponsorCommentBadgeRenderer", "customBadge", "thumbnails", 0, "url").to_s + reply_count = node_comment["replyCount"]? end - json.field "published", published.to_unix - json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale)) - comment_action_buttons_renderer = node_comment["actionButtons"]["commentActionButtonsRenderer"] - - json.field "likeCount", comment_action_buttons_renderer["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"].as_s.scan(/\d/).map(&.[0]).join.to_i - json.field "commentId", node_comment["commentId"] - json.field "authorIsChannelOwner", node_comment["authorIsChannelOwner"] + content_html = html_content || "" + json.field "content", html_to_content(content_html) + json.field "contentHtml", content_html - if comment_action_buttons_renderer["creatorHeart"]? - hearth_data = comment_action_buttons_renderer["creatorHeart"]["creatorHeartRenderer"]["creatorThumbnail"] - json.field "creatorHeart" do - json.object do - json.field "creatorThumbnail", hearth_data["thumbnails"][-1]["url"] - json.field "creatorName", hearth_data["accessibility"]["accessibilityData"]["label"] - end + if published_text != nil + published_text = published_text.to_s + if published_text.includes?(" (edited)") + json.field "isEdited", true + published = decode_date(published_text.rchop(" (edited)")) + else + json.field "isEdited", false + published = decode_date(published_text) end + + json.field "published", published.to_unix + json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale)) end if node_replies && !response["commentRepliesContinuation"]? @@ -210,7 +277,7 @@ module Invidious::Comments json.field "replies" do json.object do - json.field "replyCount", node_comment["replyCount"]? || 1 + json.field "replyCount", reply_count || 1 json.field "continuation", continuation end end @@ -236,7 +303,6 @@ module Invidious::Comments if format == "html" response = JSON.parse(response) content_html = Frontend::Comments.template_youtube(response, locale, thin_mode) - response = JSON.build do |json| json.object do json.field "contentHtml", content_html diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 429d9246..09c2168b 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -48,7 +48,7 @@ struct ConfigPreferences def to_tuple {% begin %} { - {{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}} + {{(@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }).splat}} } {% end %} end @@ -133,10 +133,6 @@ class Config # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] property cookies : HTTP::Cookies = HTTP::Cookies.new - # Key for Anti-Captcha - property captcha_key : String? = nil - # API URL for Anti-Captcha - property captcha_api_url : String = "https://api.anti-captcha.com" # Playlist length limit property playlist_length_limit : Int32 = 500 diff --git a/src/invidious/database/statistics.cr b/src/invidious/database/statistics.cr index 1df549e2..9e4963fd 100644 --- a/src/invidious/database/statistics.cr +++ b/src/invidious/database/statistics.cr @@ -15,7 +15,7 @@ module Invidious::Database::Statistics PG_DB.query_one(request, as: Int64) end - def count_users_active_1m : Int64 + def count_users_active_6m : Int64 request = <<-SQL SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '6 months' @@ -24,7 +24,7 @@ module Invidious::Database::Statistics PG_DB.query_one(request, as: Int64) end - def count_users_active_6m : Int64 + def count_users_active_1m : Int64 request = <<-SQL SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '1 month' diff --git a/src/invidious/frontend/comments_reddit.cr b/src/invidious/frontend/comments_reddit.cr index b5647bae..4dda683e 100644 --- a/src/invidious/frontend/comments_reddit.cr +++ b/src/invidious/frontend/comments_reddit.cr @@ -33,7 +33,7 @@ module Invidious::Frontend::Comments <a href="javascript:void(0)" data-onclick="toggle_parent">[ − ]</a> <b><a href="https://www.reddit.com/user/#{child.author}">#{child.author}</a></b> #{translate_count(locale, "comments_points_count", child.score, NumberFormatting::Separator)} - <span title="#{child.created_utc.to_s(translate(locale, "%a %B %-d %T %Y UTC"))}">#{translate(locale, "`x` ago", recode_date(child.created_utc, locale))}</span> + <span title="#{child.created_utc.to_s("%a %B %-d %T %Y UTC")}">#{translate(locale, "`x` ago", recode_date(child.created_utc, locale))}</span> <a href="https://www.reddit.com#{child.permalink}" title="#{translate(locale, "permalink")}">#{translate(locale, "permalink")}</a> </p> <div> diff --git a/src/invidious/frontend/comments_youtube.cr b/src/invidious/frontend/comments_youtube.cr index ecc0bc1b..aecac87f 100644 --- a/src/invidious/frontend/comments_youtube.cr +++ b/src/invidious/frontend/comments_youtube.cr @@ -107,6 +107,36 @@ module Invidious::Frontend::Comments </div> END_HTML end + when "multiImage" + html << <<-END_HTML + <section class="carousel"> + <a class="skip-link" href="#skip-#{child["commentId"]}">#{translate(locale, "carousel_skip")}</a> + <div class="slides"> + END_HTML + image_array = attachment["images"].as_a + + image_array.each_index do |i| + html << <<-END_HTML + <div class="slides-item slide-#{i + 1}" id="#{child["commentId"]}-slide-#{i + 1}" aria-label="#{translate(locale, "carousel_slide", {"current" => (i + 1).to_s, "total" => image_array.size.to_s})}" tabindex="0"> + <img loading="lazy" src="/ggpht#{URI.parse(image_array[i][1]["url"].as_s).request_target}" alt="" /> + </div> + END_HTML + end + + html << <<-END_HTML + </div> + <div class="carousel__nav"> + END_HTML + attachment["images"].as_a.each_index do |i| + html << <<-END_HTML + <a class="slider-nav" href="##{child["commentId"]}-slide-#{i + 1}" aria-label="#{translate(locale, "carousel_go_to", (i + 1).to_s)}" tabindex="-1" aria-hidden="true">#{i + 1}</a> + END_HTML + end + html << <<-END_HTML + </div> + <div id="skip-#{child["commentId"]}"></div> + </section> + END_HTML else nil # Ignore end end diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr index 6e5a975d..21b789bc 100644 --- a/src/invidious/helpers/errors.cr +++ b/src/invidious/helpers/errors.cr @@ -3,7 +3,7 @@ # ------------------- macro error_template(*args) - error_template_helper(env, {{*args}}) + error_template_helper(env, {{args.splat}}) end def github_details(summary : String, content : String) @@ -95,7 +95,7 @@ end # ------------------- macro error_atom(*args) - error_atom_helper(env, {{*args}}) + error_atom_helper(env, {{args.splat}}) end def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception) @@ -121,7 +121,7 @@ end # ------------------- macro error_json(*args) - error_json_helper(env, {{*args}}) + error_json_helper(env, {{args.splat}}) end def error_json_helper( diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index d140a858..174f620d 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -142,63 +142,8 @@ class APIHandler < Kemal::Handler exclude ["/api/v1/auth/notifications"], "POST" def call(env) - return call_next env unless only_match? env - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - # Since /api/v1/notifications is an event-stream, we don't want - # to wrap the response - return call_next env if exclude_match? env - - # Here we swap out the socket IO so we can modify the response as needed - output = env.response.output - env.response.output = IO::Memory.new - - begin - call_next env - - env.response.output.rewind - - if env.response.output.as(IO::Memory).size != 0 && - env.response.headers.includes_word?("Content-Type", "application/json") - response = JSON.parse(env.response.output) - - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - - if env.params.query["pretty"]?.try &.== "1" - response = response.to_pretty_json - else - response = response.to_json - end - else - response = env.response.output.gets_to_end - end - rescue ex - env.response.content_type = "application/json" if env.response.headers.includes_word?("Content-Type", "text/html") - env.response.status_code = 500 - - if env.response.headers.includes_word?("Content-Type", "application/json") - response = {"error" => ex.message || "Unspecified error"} - - if env.params.query["pretty"]?.try &.== "1" - response = response.to_pretty_json - else - response = response.to_json - end - end - ensure - env.response.output = output - env.response.print response - - env.response.flush - end + env.response.headers["Access-Control-Allow-Origin"] = "*" if only_match?(env) + call_next env end end diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 6dc9860e..6add0237 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -78,15 +78,6 @@ def create_notification_stream(env, topics, connection_channel) video.published = published response = JSON.parse(video.to_json(locale, nil)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -113,15 +104,6 @@ def create_notification_stream(env, topics, connection_channel) Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video| response = JSON.parse(video.to_json(locale)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -155,15 +137,6 @@ def create_notification_stream(env, topics, connection_channel) video.published = Time.unix(published) response = JSON.parse(video.to_json(locale, nil)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index 76e477a4..23a1aafc 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -78,7 +78,7 @@ def load_all_locales return locales end -def translate(locale : String?, key : String, text : String | Nil = nil) : String +def translate(locale : String?, key : String, text : String | Hash(String, String) | Nil = nil) : String # Log a warning if "key" doesn't exist in en-US locale and return # that key as the text, so this is more or less transparent to the user. if !LOCALES["en-US"].has_key?(key) @@ -101,10 +101,12 @@ def translate(locale : String?, key : String, text : String | Nil = nil) : Strin match_length = 0 raw_data.as_h.each do |hash_key, value| - if md = text.try &.match(/#{hash_key}/) - if md[0].size >= match_length - translation = value.as_s - match_length = md[0].size + if text.is_a?(String) + if md = text.try &.match(/#{hash_key}/) + if md[0].size >= match_length + translation = value.as_s + match_length = md[0].size + end end end end @@ -114,8 +116,13 @@ def translate(locale : String?, key : String, text : String | Nil = nil) : Strin raise "Invalid translation \"#{raw_data}\"" end - if text + if text.is_a?(String) translation = translation.gsub("`x`", text) + elsif text.is_a?(Hash(String, String)) + # adds support for multi string interpolation. Based on i18next https://www.i18next.com/translation-function/interpolation#basic + text.each_key do |hash_key| + translation = translation.gsub("{{#{hash_key}}}", text[hash_key]) + end end return translation diff --git a/src/invidious/helpers/i18next.cr b/src/invidious/helpers/i18next.cr index 252af6b9..9f4077e1 100644 --- a/src/invidious/helpers/i18next.cr +++ b/src/invidious/helpers/i18next.cr @@ -47,19 +47,19 @@ module I18next::Plurals private PLURAL_SETS = { PluralForms::Single_gt_one => [ - "ach", "ak", "am", "arn", "br", "fil", "gun", "ln", "mfe", "mg", - "mi", "oc", "pt", "tg", "tl", "ti", "tr", "uz", "wa", + "ach", "ak", "am", "arn", "br", "fa", "fil", "gun", "ln", "mfe", "mg", + "mi", "oc", "pt-PT", "tg", "tl", "ti", "tr", "uz", "wa", ], PluralForms::Single_not_one => [ "af", "an", "ast", "az", "bg", "bn", "ca", "da", "de", "dev", "el", "en", - "eo", "es", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi", + "eo", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi", "hu", "hy", "ia", "kk", "kn", "ku", "lb", "mai", "ml", "mn", "mr", "nah", "nap", "nb", "ne", "nl", "nn", "no", "nso", "pa", "pap", "pms", "ps", "rm", "sco", "se", "si", "so", "son", "sq", "sv", "sw", "ta", "te", "tk", "ur", "yo", ], PluralForms::None => [ - "ay", "bo", "cgg", "fa", "ht", "id", "ja", "jbo", "ka", "km", "ko", "ky", + "ay", "bo", "cgg", "ht", "id", "ja", "jbo", "ka", "km", "ko", "ky", "lo", "ms", "sah", "su", "th", "tt", "ug", "vi", "wo", "zh", ], PluralForms::Dual_Slavic => [ @@ -90,11 +90,13 @@ module I18next::Plurals "sk" => PluralForms::Special_Czech_Slovak, "sl" => PluralForms::Special_Slovenian, # Mixed v3/v4 rules - "fr" => PluralForms::Special_French_Portuguese, - "hr" => PluralForms::Special_Hungarian_Serbian, - "it" => PluralForms::Special_Spanish_Italian, - "pt-BR" => PluralForms::Special_French_Portuguese, - "sr" => PluralForms::Special_Hungarian_Serbian, + "es" => PluralForms::Special_Spanish_Italian, + "fr" => PluralForms::Special_French_Portuguese, + "hr" => PluralForms::Special_Hungarian_Serbian, + "it" => PluralForms::Special_Spanish_Italian, + "pt" => PluralForms::Special_French_Portuguese, + "pt" => PluralForms::Special_French_Portuguese, + "sr" => PluralForms::Special_Hungarian_Serbian, } # These are the v1 and v2 compatible suffixes. @@ -165,7 +167,7 @@ module I18next::Plurals def get_plural_form(locale : String) : PluralForms # Extract the ISO 639-1 or 639-2 code from an RFC 5646 language code - if !locale.matches?(/^pt-BR$/) + if !locale.matches?(/^pt-PT$/) locale = locale.split('-')[0] end diff --git a/src/invidious/helpers/json_filter.cr b/src/invidious/helpers/json_filter.cr deleted file mode 100644 index 3f4080ba..00000000 --- a/src/invidious/helpers/json_filter.cr +++ /dev/null @@ -1,248 +0,0 @@ -module JSONFilter - alias BracketIndex = Hash(Int64, Int64) - - alias GroupedFieldsValue = String | Array(GroupedFieldsValue) - alias GroupedFieldsList = Array(GroupedFieldsValue) - - class FieldsParser - class ParseError < Exception - end - - # Returns the `Regex` pattern used to match nest groups - def self.nest_group_pattern : Regex - # uses a '.' character to match json keys as they are allowed - # to contain any unicode codepoint - /(?:|,)(?<groupname>[^,\n]*?)\(/ - end - - # Returns the `Regex` pattern used to check if there are any empty nest groups - def self.unnamed_nest_group_pattern : Regex - /^\(|\(\(|\/\(/ - end - - def self.parse_fields(fields_text : String, &) : Nil - if fields_text.empty? - raise FieldsParser::ParseError.new "Fields is empty" - end - - opening_bracket_count = fields_text.count('(') - closing_bracket_count = fields_text.count(')') - - if opening_bracket_count != closing_bracket_count - bracket_type = opening_bracket_count > closing_bracket_count ? "opening" : "closing" - raise FieldsParser::ParseError.new "There are too many #{bracket_type} brackets (#{opening_bracket_count}:#{closing_bracket_count})" - elsif match_result = unnamed_nest_group_pattern.match(fields_text) - raise FieldsParser::ParseError.new "Unnamed nest group at position #{match_result.begin}" - end - - # first, handle top-level single nested properties: items/id, playlistItems/snippet, etc - parse_single_nests(fields_text) { |nest_list| yield nest_list } - - # next, handle nest groups: items(id, etag, etc) - parse_nest_groups(fields_text) { |nest_list| yield nest_list } - end - - def self.parse_single_nests(fields_text : String, &) : Nil - single_nests = remove_nest_groups(fields_text) - - if !single_nests.empty? - property_nests = single_nests.split(',') - - property_nests.each do |nest| - nest_list = nest.split('/') - if nest_list.includes? "" - raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list}" - end - yield nest_list - end - # else - # raise FieldsParser::ParseError.new "Empty key in nest list 22: #{fields_text} | #{single_nests}" - end - end - - def self.parse_nest_groups(fields_text : String, &) : Nil - nest_stack = [] of NamedTuple(group_name: String, closing_bracket_index: Int64) - bracket_pairs = get_bracket_pairs(fields_text, true) - - text_index = 0 - regex_index = 0 - - while regex_result = self.nest_group_pattern.match(fields_text, regex_index) - raw_match = regex_result[0] - group_name = regex_result["groupname"] - - text_index = regex_result.begin - regex_index = regex_result.end - - if text_index.nil? || regex_index.nil? - raise FieldsParser::ParseError.new "Received invalid index while parsing nest groups: text_index: #{text_index} | regex_index: #{regex_index}" - end - - offset = raw_match.starts_with?(',') ? 1 : 0 - - opening_bracket_index = (text_index + group_name.size) + offset - closing_bracket_index = bracket_pairs[opening_bracket_index] - content_start = opening_bracket_index + 1 - - content = fields_text[content_start...closing_bracket_index] - - if content.empty? - raise FieldsParser::ParseError.new "Empty nest group at position #{content_start}" - else - content = remove_nest_groups(content) - end - - while nest_stack.size > 0 && closing_bracket_index > nest_stack[nest_stack.size - 1][:closing_bracket_index] - if nest_stack.size - nest_stack.pop - end - end - - group_name.split('/').each do |name| - nest_stack.push({ - group_name: name, - closing_bracket_index: closing_bracket_index, - }) - end - - if !content.empty? - properties = content.split(',') - - properties.each do |prop| - nest_list = nest_stack.map { |nest_prop| nest_prop[:group_name] } - - if !prop.empty? - if prop.includes?('/') - parse_single_nests(prop) { |list| nest_list += list } - else - nest_list.push prop - end - else - raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list << prop}" - end - - yield nest_list - end - end - end - end - - def self.remove_nest_groups(text : String) : String - content_bracket_pairs = get_bracket_pairs(text, false) - - content_bracket_pairs.each_key.to_a.reverse.each do |opening_bracket| - closing_bracket = content_bracket_pairs[opening_bracket] - last_comma = text.rindex(',', opening_bracket) || 0 - - text = text[0...last_comma] + text[closing_bracket + 1...text.size] - end - - return text.starts_with?(',') ? text[1...text.size] : text - end - - def self.get_bracket_pairs(text : String, recursive = true) : BracketIndex - istart = [] of Int64 - bracket_index = BracketIndex.new - - text.each_char_with_index do |char, index| - if char == '(' - istart.push(index.to_i64) - end - - if char == ')' - begin - opening = istart.pop - if recursive || (!recursive && istart.size == 0) - bracket_index[opening] = index.to_i64 - end - rescue - raise FieldsParser::ParseError.new "No matching opening parenthesis at: #{index}" - end - end - end - - if istart.size != 0 - idx = istart.pop - raise FieldsParser::ParseError.new "No matching closing parenthesis at: #{idx}" - end - - return bracket_index - end - end - - class FieldsGrouper - alias SkeletonValue = Hash(String, SkeletonValue) - - def self.create_json_skeleton(fields_text : String) : SkeletonValue - root_hash = {} of String => SkeletonValue - - FieldsParser.parse_fields(fields_text) do |nest_list| - current_item = root_hash - nest_list.each do |key| - if current_item[key]? - current_item = current_item[key] - else - current_item[key] = {} of String => SkeletonValue - current_item = current_item[key] - end - end - end - root_hash - end - - def self.create_grouped_fields_list(json_skeleton : SkeletonValue) : GroupedFieldsList - grouped_fields_list = GroupedFieldsList.new - json_skeleton.each do |key, value| - grouped_fields_list.push key - - nested_keys = create_grouped_fields_list(value) - grouped_fields_list.push nested_keys unless nested_keys.empty? - end - return grouped_fields_list - end - end - - class FilterError < Exception - end - - def self.filter(item : JSON::Any, fields_text : String, in_place : Bool = true) - skeleton = FieldsGrouper.create_json_skeleton(fields_text) - grouped_fields_list = FieldsGrouper.create_grouped_fields_list(skeleton) - filter(item, grouped_fields_list, in_place) - end - - def self.filter(item : JSON::Any, grouped_fields_list : GroupedFieldsList, in_place : Bool = true) : JSON::Any - item = item.clone unless in_place - - if !item.as_h? && !item.as_a? - raise FilterError.new "Can't filter '#{item}' by #{grouped_fields_list}" - end - - top_level_keys = Array(String).new - grouped_fields_list.each do |value| - if value.is_a? String - top_level_keys.push value - elsif value.is_a? Array - if !top_level_keys.empty? - key_to_filter = top_level_keys.last - - if item.as_h? - filter(item[key_to_filter], value, in_place: true) - elsif item.as_a? - item.as_a.each { |arr_item| filter(arr_item[key_to_filter], value, in_place: true) } - end - else - raise FilterError.new "Tried to filter while top level keys list is empty" - end - end - end - - if item.as_h? - item.as_h.select! top_level_keys - elsif item.as_a? - item.as_a.map { |value| filter(value, top_level_keys, in_place: true) } - end - - item - end -end diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index a006d602..e438e3b9 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -262,7 +262,7 @@ def get_referer(env, fallback = "/", unroll = true) end referer = referer.request_target - referer = "/" + referer.gsub(/[^\/?@&%=\-_.:,0-9a-zA-Z]/, "").lstrip("/\\") + referer = "/" + referer.gsub(/[^\/?@&%=\-_.:,*0-9a-zA-Z]/, "").lstrip("/\\") if referer == env.request.path referer = fallback diff --git a/src/invidious/helpers/webvtt.cr b/src/invidious/helpers/webvtt.cr index 56f761ed..260d250f 100644 --- a/src/invidious/helpers/webvtt.cr +++ b/src/invidious/helpers/webvtt.cr @@ -4,13 +4,23 @@ module WebVTT # A WebVTT builder generates WebVTT files private class Builder + # See https://developer.mozilla.org/en-US/docs/Web/API/WebVTT_API#cue_payload + private ESCAPE_SUBSTITUTIONS = { + '&' => "&", + '<' => "<", + '>' => ">", + '\u200E' => "‎", + '\u200F' => "‏", + '\u00A0' => " ", + } + def initialize(@io : IO) end # Writes an vtt cue with the specified time stamp and contents def cue(start_time : Time::Span, end_time : Time::Span, text : String) timestamp(start_time, end_time) - @io << text + @io << self.escape(text) @io << "\n\n" end @@ -29,6 +39,10 @@ module WebVTT @io << '.' << timestamp.milliseconds.to_s.rjust(3, '0') end + private def escape(text : String) : String + return text.gsub(ESCAPE_SUBSTITUTIONS) + end + def document(setting_fields : Hash(String, String)? = nil, &) @io << "WEBVTT\n" diff --git a/src/invidious/jobs/bypass_captcha_job.cr b/src/invidious/jobs/bypass_captcha_job.cr deleted file mode 100644 index 71f8a938..00000000 --- a/src/invidious/jobs/bypass_captcha_job.cr +++ /dev/null @@ -1,135 +0,0 @@ -class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob - def begin - loop do - begin - random_video = PG_DB.query_one?("select id, ucid from (select id, ucid from channel_videos limit 1000) as s ORDER BY RANDOM() LIMIT 1", as: {id: String, ucid: String}) - if !random_video - random_video = {id: "zj82_v2R6ts", ucid: "UCK87Lox575O_HCHBWaBSyGA"} - end - {"/watch?v=#{random_video["id"]}&gl=US&hl=en&has_verified=1&bpctr=9999999999", produce_channel_videos_url(ucid: random_video["ucid"])}.each do |path| - response = YT_POOL.client &.get(path) - if response.body.includes?("To continue with your YouTube experience, please fill out the form below.") - html = XML.parse_html(response.body) - form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil! - site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"] - s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"] - - inputs = {} of String => String - form.xpath_nodes(%(.//input[@name])).map do |node| - inputs[node["name"]] = node["value"] - end - - headers = response.cookies.add_request_headers(HTTP::Headers.new) - - response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/createTask", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "task" => { - "type" => "NoCaptchaTaskProxyless", - "websiteURL" => "https://www.youtube.com#{path}", - "websiteKey" => site_key, - "recaptchaDataSValue" => s_value, - }, - }.to_json).body) - - raise response["error"].as_s if response["error"]? - task_id = response["taskId"].as_i - - loop do - sleep 10.seconds - - response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/getTaskResult", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "taskId" => task_id, - }.to_json).body) - - if response["status"]?.try &.== "ready" - break - elsif response["errorId"]?.try &.as_i != 0 - raise response["errorDescription"].as_s - end - end - - inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s - headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || "" - response = YT_POOL.client &.post("/das_captcha", headers, form: inputs) - - response.cookies - .select { |cookie| cookie.name != "PREF" } - .each { |cookie| CONFIG.cookies << cookie } - - # Persist cookies between runs - File.write("config/config.yml", CONFIG.to_yaml) - elsif response.headers["Location"]?.try &.includes?("/sorry/index") - location = response.headers["Location"].try { |u| URI.parse(u) } - headers = HTTP::Headers{":authority" => location.host.not_nil!} - response = YT_POOL.client &.get(location.request_target, headers) - - html = XML.parse_html(response.body) - form = html.xpath_node(%(//form[@action="index"])).not_nil! - site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"] - s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"] - - inputs = {} of String => String - form.xpath_nodes(%(.//input[@name])).map do |node| - inputs[node["name"]] = node["value"] - end - - captcha_client = HTTPClient.new(URI.parse(CONFIG.captcha_api_url)) - captcha_client.family = CONFIG.force_resolve || Socket::Family::INET - response = JSON.parse(captcha_client.post("/createTask", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "task" => { - "type" => "NoCaptchaTaskProxyless", - "websiteURL" => location.to_s, - "websiteKey" => site_key, - "recaptchaDataSValue" => s_value, - }, - }.to_json).body) - - captcha_client.close - - raise response["error"].as_s if response["error"]? - task_id = response["taskId"].as_i - - loop do - sleep 10.seconds - - response = JSON.parse(captcha_client.post("/getTaskResult", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "taskId" => task_id, - }.to_json).body) - - if response["status"]?.try &.== "ready" - break - elsif response["errorId"]?.try &.as_i != 0 - raise response["errorDescription"].as_s - end - end - - inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s - headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || "" - response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs) - headers = HTTP::Headers{ - "Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0], - } - cookies = HTTP::Cookies.from_client_headers(headers) - - cookies.each { |cookie| CONFIG.cookies << cookie } - - # Persist cookies between runs - File.write("config/config.yml", CONFIG.to_yaml) - end - end - rescue ex - LOGGER.error("BypassCaptchaJob: #{ex.message}") - ensure - sleep 1.minute - Fiber.yield - end - end - end -end diff --git a/src/invidious/jobs/statistics_refresh_job.cr b/src/invidious/jobs/statistics_refresh_job.cr index 72d1ce88..66c91ad5 100644 --- a/src/invidious/jobs/statistics_refresh_job.cr +++ b/src/invidious/jobs/statistics_refresh_job.cr @@ -56,8 +56,8 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob users = STATISTICS.dig("usage", "users").as(Hash(String, Int64)) users["total"] = Invidious::Database::Statistics.count_users_total - users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_1m - users["activeMonth"] = Invidious::Database::Statistics.count_users_active_6m + users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_6m + users["activeMonth"] = Invidious::Database::Statistics.count_users_active_1m STATISTICS["metadata"] = { "updatedAt" => Time.utc.to_unix, diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index 1651559a..0dced80b 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -62,6 +62,7 @@ module Invidious::JSONify::APIv1 json.field "rating", 0_i64 json.field "isListed", video.is_listed json.field "liveNow", video.live_now + json.field "isPostLiveDvr", video.post_live_dvr json.field "isUpcoming", video.is_upcoming if video.premiere_timestamp @@ -160,6 +161,8 @@ module Invidious::JSONify::APIv1 json.field "type", fmt["mimeType"] json.field "quality", fmt["quality"] + json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]? + fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) if fmt_info fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 @@ -227,6 +230,7 @@ module Invidious::JSONify::APIv1 json.field "author", rv["author"] json.field "authorUrl", "/channel/#{rv["ucid"]?}" json.field "authorId", rv["ucid"]? + json.field "authorVerified", rv["author_verified"] == "true" if rv["author_thumbnail"]? json.field "authorThumbnails" do json.array do diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index 662d1002..d89e752c 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -21,7 +21,13 @@ module Invidious::Routes::API::Manifest end if dashmpd = video.dash_manifest_url - manifest = YT_POOL.client &.get(URI.parse(dashmpd).request_target).body + response = YT_POOL.client &.get(URI.parse(dashmpd).request_target) + + if response.status_code != 200 + haltf env, status_code: response.status_code + end + + manifest = response.body manifest = manifest.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl| url = baseurl.lchop("<BaseURL>") diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 67018660..43a5c35b 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -90,6 +90,7 @@ module Invidious::Routes::API::V1::Channels json.field "allowedRegions", channel.allowed_regions json.field "tabs", channel.tabs + json.field "tags", channel.tags json.field "authorVerified", channel.verified json.field "latestVideos" do @@ -207,11 +208,12 @@ module Invidious::Routes::API::V1::Channels get_channel() # Retrieve continuation from URL parameters + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" continuation = env.params.query["continuation"]? begin videos, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation + channel, continuation: continuation, sort_by: sort_by ) rescue ex return error_json(500, ex) @@ -393,7 +395,7 @@ module Invidious::Routes::API::V1::Channels else comments = YoutubeAPI.browse(continuation: continuation) end - return Comments.parse_youtube(id, comments, format, locale, thin_mode, isPost: true) + return Comments.parse_youtube(id, comments, format, locale, thin_mode, is_post: true) end def self.channels(env) diff --git a/src/invidious/routes/api/v1/feeds.cr b/src/invidious/routes/api/v1/feeds.cr index 41865f34..fea2993c 100644 --- a/src/invidious/routes/api/v1/feeds.cr +++ b/src/invidious/routes/api/v1/feeds.cr @@ -31,7 +31,7 @@ module Invidious::Routes::API::V1::Feeds if !CONFIG.popular_enabled error_message = {"error" => "Administrator has disabled this endpoint."}.to_json - haltf env, 400, error_message + haltf env, 403, error_message end JSON.build do |json| diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index b42ecd1a..12942906 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -191,6 +191,8 @@ module Invidious::Routes::API::V1::Misc json.object do json.field "ucid", sub_endpoint["browseId"].as_s if sub_endpoint["browseId"]? json.field "videoId", sub_endpoint["videoId"].as_s if sub_endpoint["videoId"]? + json.field "playlistId", sub_endpoint["playlistId"].as_s if sub_endpoint["playlistId"]? + json.field "startTimeSeconds", sub_endpoint["startTimeSeconds"].as_i if sub_endpoint["startTimeSeconds"]? json.field "params", params.try &.as_s json.field "pageType", pageType end diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index 9fb283c2..2922b060 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -32,11 +32,14 @@ module Invidious::Routes::API::V1::Search begin client = HTTP::Client.new("suggestqueries-clients6.youtube.com") - url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&xssi=t&gs_ri=youtube&ds=yt" + client.before_request { |r| add_yt_headers(r) } + + url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt" response = client.get(url).body + client.close - body = JSON.parse(response[5..-1]).as_a + body = JSON.parse(response[19..-2]).as_a suggestions = body[1].as_a[0..-2] JSON.build do |json| diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 3bead06a..5e269923 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -89,9 +89,14 @@ module Invidious::Routes::API::V1::Videos if CONFIG.use_innertube_for_captions params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated) - initial_data = YoutubeAPI.get_transcript(params) - webvtt = Invidious::Videos::Transcript.convert_transcripts_to_vtt(initial_data, caption.language_code) + transcript = Invidious::Videos::Transcript.from_raw( + YoutubeAPI.get_transcript(params), + caption.language_code, + caption.auto_generated + ) + + webvtt = transcript.to_vtt else # Timedtext API handling url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target @@ -367,4 +372,47 @@ module Invidious::Routes::API::V1::Videos end end end + + def self.clips(env) + locale = env.get("preferences").as(Preferences).locale + + env.response.content_type = "application/json" + + clip_id = env.params.url["id"] + region = env.params.query["region"]? + proxy = {"1", "true"}.any? &.== env.params.query["local"]? + + response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}") + return error_json(400, "Invalid clip ID") if response["error"]? + + video_id = response.dig?("endpoint", "watchEndpoint", "videoId").try &.as_s + return error_json(400, "Invalid clip ID") if video_id.nil? + + start_time = nil + end_time = nil + clip_title = nil + + if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s + start_time, end_time, clip_title = parse_clip_parameters(params) + end + + begin + video = get_video(video_id, region: region) + rescue ex : NotFoundException + return error_json(404, ex) + rescue ex + return error_json(500, ex) + end + + return JSON.build do |json| + json.object do + json.field "startTime", start_time + json.field "endTime", end_time + json.field "clipTitle", clip_title + json.field "video" do + Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy) + end + end + end + end end diff --git a/src/invidious/routes/before_all.cr b/src/invidious/routes/before_all.cr index 396840a4..5695dee9 100644 --- a/src/invidious/routes/before_all.cr +++ b/src/invidious/routes/before_all.cr @@ -30,7 +30,7 @@ module Invidious::Routes::BeforeAll # Only allow the pages at /embed/* to be embedded if env.request.resource.starts_with?("/embed") - frame_ancestors = "'self' http: https:" + frame_ancestors = "'self' file: http: https:" else frame_ancestors = "'none'" end diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index d4d8b1c1..360af2cd 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -81,13 +81,12 @@ module Invidious::Routes::Channels return env.redirect "/channel/#{channel.ucid}" end - # TODO: support sort option for livestreams - sort_by = "" - sort_options = [] of String + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" + sort_options = {"newest", "oldest", "popular"} # Fetch items and continuation token items, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation + channel, continuation: continuation, sort_by: sort_by ) selected_tab = Frontend::ChannelPage::TabsAvailable::Streams @@ -231,7 +230,7 @@ module Invidious::Routes::Channels if nojs comments = Comments.fetch_community_post_comments(ucid, id) - comment_html = JSON.parse(Comments.parse_youtube(id, comments, "html", locale, thin_mode, isPost: true))["contentHtml"] + comment_html = JSON.parse(Comments.parse_youtube(id, comments, "html", locale, thin_mode, is_post: true))["contentHtml"] end templated "post" end diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 40bca008..e20a7139 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -407,14 +407,23 @@ module Invidious::Routes::Feeds end spawn do - rss = XML.parse_html(body) - rss.xpath_nodes("//feed/entry").each do |entry| - id = entry.xpath_node("videoid").not_nil!.content - author = entry.xpath_node("author/name").not_nil!.content - published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) - updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) - - video = get_video(id, force_refresh: true) + # TODO: unify this with the other almost identical looking parts in this and channels.cr somehow? + namespaces = { + "yt" => "http://www.youtube.com/xml/schemas/2015", + "default" => "http://www.w3.org/2005/Atom", + } + rss = XML.parse(body) + rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry| + id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content + author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content + published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content) + updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content) + + begin + video = get_video(id, force_refresh: true) + rescue + next # skip this video since it raised an exception (e.g. it is a scheduled live event) + end if CONFIG.enable_user_notifications # Deliver notifications to `/api/v1/auth/notifications` diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 1d5aa914..ec18f3b8 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -42,7 +42,7 @@ module Invidious::Routes::VideoPlayback headers["Range"] = "bytes=#{range_for_head}" end - client = make_client(URI.parse(host), region) + client = make_client(URI.parse(host), region, force_resolve = true) response = HTTP::Client::Response.new(500) error = "" 5.times do @@ -57,7 +57,7 @@ module Invidious::Routes::VideoPlayback if new_host != host host = new_host client.close - client = make_client(URI.parse(new_host), region) + client = make_client(URI.parse(new_host), region, force_resolve = true) end url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" @@ -71,7 +71,7 @@ module Invidious::Routes::VideoPlayback fvip = "3" host = "https://r#{fvip}---#{mn}.googlevideo.com" - client = make_client(URI.parse(host), region) + client = make_client(URI.parse(host), region, force_resolve = true) rescue ex error = ex.message end @@ -196,7 +196,7 @@ module Invidious::Routes::VideoPlayback break else client.close - client = make_client(URI.parse(host), region) + client = make_client(URI.parse(host), region, force_resolve = true) end end diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 3d935f0a..aabe8dfc 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -275,6 +275,12 @@ module Invidious::Routes::Watch return error_template(400, "Invalid clip ID") if response["error"]? if video_id = response.dig?("endpoint", "watchEndpoint", "videoId") + if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s + start_time, end_time, _ = parse_clip_parameters(params) + env.params.query["start"] = start_time.to_s if start_time != nil + env.params.query["end"] = end_time.to_s if end_time != nil + end + return env.redirect "/watch?v=#{video_id}&#{env.params.query}" else return error_template(404, "The requested clip doesn't exist") diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index d6bd991c..ba05da19 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -235,6 +235,7 @@ module Invidious::Routing get "/api/v1/captions/:id", {{namespace}}::Videos, :captions get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations get "/api/v1/comments/:id", {{namespace}}::Videos, :comments + get "/api/v1/clips/:id", {{namespace}}::Videos, :clips # Feeds get "/api/v1/trending", {{namespace}}::Feeds, :trending diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr index c2b5c758..bf968734 100644 --- a/src/invidious/search/filters.cr +++ b/src/invidious/search/filters.cr @@ -300,9 +300,9 @@ module Invidious::Search object["9:varint"] = ((page - 1) * 20).to_i64 end - # If the object is empty, return an empty string, - # otherwise encode to protobuf then to base64 - return "" if object.empty? + # Prevent censoring of self harm topics + # See https://github.com/iv-org/invidious/issues/4398 + object["30:varint"] = 1.to_i64 return object .try { |i| Protodec::Any.cast_json(i) } diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index 2d9f8a83..107d148d 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr @@ -22,12 +22,14 @@ def fetch_trending(trending_type, region, locale) extracted = [] of SearchItem + deduplicate = items.size > 1 + items.each do |itm| if itm.is_a?(Category) # Ignore the smaller categories, as they generally contain a sponsored # channel, which brings a lot of noise on the trending page. # See: https://github.com/iv-org/invidious/issues/2989 - next if itm.contents.size < 24 + next if (itm.contents.size < 24 && deduplicate) extracted.concat extract_category(itm) else diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 9fbd1374..c218b4ef 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -82,6 +82,10 @@ struct Video return (self.video_type == VideoType::Livestream) end + def post_live_dvr + return info["isPostLiveDvr"].as_bool + end + def premiere_timestamp : Time? info .dig?("microformat", "playerMicroformatRenderer", "liveBroadcastDetails", "startTimestamp") @@ -227,8 +231,22 @@ struct Video info.dig?("streamingData", "hlsManifestUrl").try &.as_s end - def dash_manifest_url - info.dig?("streamingData", "dashManifestUrl").try &.as_s + def dash_manifest_url : String? + raw_dash_url = info.dig?("streamingData", "dashManifestUrl").try &.as_s + return nil if raw_dash_url.nil? + + # Use manifest v5 parameter to reduce file size + # See https://github.com/iv-org/invidious/issues/4186 + dash_url = URI.parse(raw_dash_url) + dash_query = dash_url.query || "" + + if dash_query.empty? + dash_url.path = "#{dash_url.path}/mpd_version/5" + else + dash_url.query = "#{dash_query}&mpd_version=5" + end + + return dash_url.to_s end def genre_url : String? @@ -380,17 +398,6 @@ def fetch_video(id, region) .dig?("microformat", "playerMicroformatRenderer", "availableCountries") .try &.as_a.map &.as_s || [] of String - # Check for region-blocks - if info["reason"]?.try &.as_s.includes?("your country") - bypass_regions = PROXY_LIST.keys & allowed_regions - if !bypass_regions.empty? - region = bypass_regions[rand(bypass_regions.size)] - region_info = extract_video_info(video_id: id, proxy_region: region) - region_info["region"] = JSON::Any.new(region) if region - info = region_info if !region_info["reason"]? - end - end - if reason = info["reason"]? if reason == "Video unavailable" raise NotFoundException.new(reason.as_s || "") diff --git a/src/invidious/videos/clip.cr b/src/invidious/videos/clip.cr new file mode 100644 index 00000000..29c57182 --- /dev/null +++ b/src/invidious/videos/clip.cr @@ -0,0 +1,22 @@ +require "json" + +# returns start_time, end_time and clip_title +def parse_clip_parameters(params) : {Float64?, Float64?, String?} + decoded_protobuf = params.try { |i| URI.decode_www_form(i) } + .try { |i| Base64.decode(i) } + .try { |i| IO::Memory.new(i) } + .try { |i| Protodec::Any.parse(i) } + + start_time = decoded_protobuf + .try(&.["50:0:embedded"]["2:1:varint"].as_i64) + .try { |i| i/1000 } + + end_time = decoded_protobuf + .try(&.["50:0:embedded"]["3:2:varint"].as_i64) + .try { |i| i/1000 } + + clip_title = decoded_protobuf + .try(&.["50:0:embedded"]["4:3:string"].as_s) + + return start_time, end_time, clip_title +end diff --git a/src/invidious/videos/description.cr b/src/invidious/videos/description.cr index 542cb416..c7191dec 100644 --- a/src/invidious/videos/description.cr +++ b/src/invidious/videos/description.cr @@ -7,7 +7,19 @@ private def copy_string(str : String::Builder, iter : Iterator, count : Int) : I cp = iter.next break if cp.is_a?(Iterator::Stop) - str << cp.chr + if cp == 0x26 # Ampersand (&) + str << "&" + elsif cp == 0x27 # Single quote (') + str << "'" + elsif cp == 0x22 # Double quote (") + str << """ + elsif cp == 0x3C # Less-than (<) + str << "<" + elsif cp == 0x3E # Greater than (>) + str << ">" + else + str << cp.chr + end # A codepoint from the SMP counts twice copied += 1 if cp > 0xFFFF diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 77520dbe..0e1a947c 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -50,9 +50,9 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? } end -def extract_video_info(video_id : String, proxy_region : String? = nil) +def extract_video_info(video_id : String) # Init client config for the API - client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region) + client_config = YoutubeAPI::ClientConfig.new # Fetch data from the player endpoint player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) @@ -107,11 +107,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil) # decrypted URLs and maybe fix throttling issues (#2194). See the # following issue for an explanation about decrypted URLs: # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 - client_config.client_type = YoutubeAPI::ClientType::Android - new_player_response = try_fetch_streaming_data(video_id, client_config) - elsif !reason.includes?("your country") # Handled separately - # The Android embedded client could help here - client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed + client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite new_player_response = try_fetch_streaming_data(video_id, client_config) end @@ -123,8 +119,9 @@ def extract_video_info(video_id : String, proxy_region : String? = nil) # Replace player response and reset reason if !new_player_response.nil? - # Preserve storyboard data before replacement + # Preserve captions & storyboard data before replacement new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]? + new_player_response["captions"] = player_response["captions"] if player_response["captions"]? player_response = new_player_response params.delete("reason") @@ -142,8 +139,7 @@ end def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)? LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.") - # 2AMBCgIQBg is a workaround for streaming URLs that returns a 403. - response = YoutubeAPI.player(video_id: id, params: "2AMBCgIQBg", client_config: client_config) + response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config) playability_status = response["playabilityStatus"]["status"] LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.") @@ -151,7 +147,7 @@ def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConf if id != response.dig("videoDetails", "videoId") # YouTube may return a different video player response than expected. # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 - raise VideoNotAvailableException.new( + raise InfoException.new( "The video returned by YouTube isn't the requested one. (#{client_config.client_type} client)" ) elsif playability_status == "OK" @@ -215,6 +211,9 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow") .try &.as_bool || false + post_live_dvr = video_details.dig?("isPostLiveDvr") + .try &.as_bool || false + # Extra video infos allowed_regions = microformat["availableCountries"]? @@ -266,7 +265,18 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any .try &.dig?("videoActions", "menuRenderer", "topLevelButtons") if toplevel_buttons - likes_button = toplevel_buttons.try &.as_a + # New Format as of december 2023 + likes_button = toplevel_buttons.dig?(0, + "segmentedLikeDislikeButtonViewModel", + "likeButtonViewModel", + "likeButtonViewModel", + "toggleButtonViewModel", + "toggleButtonViewModel", + "defaultButtonViewModel", + "buttonViewModel" + ) + + likes_button ||= toplevel_buttons.try &.as_a .find(&.dig?("toggleButtonRenderer", "defaultIcon", "iconType").=== "LIKE") .try &.["toggleButtonRenderer"] @@ -279,9 +289,10 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any ) if likes_button + likes_txt = likes_button.dig?("accessibilityText") # Note: The like count from `toggledText` is off by one, as it would # represent the new like count in the event where the user clicks on "like". - likes_txt = (likes_button["defaultText"]? || likes_button["toggledText"]?) + likes_txt ||= (likes_button["defaultText"]? || likes_button["toggledText"]?) .try &.dig?("accessibility", "accessibilityData", "label") likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt @@ -404,6 +415,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any "isListed" => JSON::Any.new(is_listed || false), "isUpcoming" => JSON::Any.new(is_upcoming || false), "keywords" => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }), + "isPostLiveDvr" => JSON::Any.new(post_live_dvr), # Related videos "relatedVideos" => JSON::Any.new(related), # Description diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr index dac00eea..9cd064c5 100644 --- a/src/invidious/videos/transcript.cr +++ b/src/invidious/videos/transcript.cr @@ -1,8 +1,26 @@ module Invidious::Videos - # Namespace for methods primarily relating to Transcripts - module Transcript - record TranscriptLine, start_ms : Time::Span, end_ms : Time::Span, line : String + # A `Transcripts` struct encapsulates a sequence of lines that together forms the whole transcript for a given YouTube video. + # These lines can be categorized into two types: section headings and regular lines representing content from the video. + struct Transcript + # Types + record HeadingLine, start_ms : Time::Span, end_ms : Time::Span, line : String + record RegularLine, start_ms : Time::Span, end_ms : Time::Span, line : String + alias TranscriptLine = HeadingLine | RegularLine + property lines : Array(TranscriptLine) + + property language_code : String + property auto_generated : Bool + + # User friendly label for the current transcript. + # Example: "English (auto-generated)" + property label : String + + # Initializes a new Transcript struct with the contents and associated metadata describing it + def initialize(@lines : Array(TranscriptLine), @language_code : String, @auto_generated : Bool, @label : String) + end + + # Generates a protobuf string to fetch the requested transcript from YouTube def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String kind = auto_generated ? "asr" : "" @@ -30,48 +48,79 @@ module Invidious::Videos return params end - def self.convert_transcripts_to_vtt(initial_data : Hash(String, JSON::Any), target_language : String) : String - # Convert into array of TranscriptLine - lines = self.parse(initial_data) + # Constructs a Transcripts struct from the initial YouTube response + def self.from_raw(initial_data : Hash(String, JSON::Any), language_code : String, auto_generated : Bool) + transcript_panel = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", + "content", "transcriptSearchPanelRenderer") - settings_field = { - "Kind" => "captions", - "Language" => target_language, - } + segment_list = transcript_panel.dig("body", "transcriptSegmentListRenderer") - # Taken from Invidious::Videos::Captions::Metadata.timedtext_to_vtt() - vtt = WebVTT.build(settings_field) do |vtt| - lines.each do |line| - vtt.cue(line.start_ms, line.end_ms, line.line) - end + if !segment_list["initialSegments"]? + raise NotFoundException.new("Requested transcript does not exist") end - return vtt - end + # Extract user-friendly label for the current transcript + + footer_language_menu = transcript_panel.dig?( + "footer", "transcriptFooterRenderer", "languageMenu", "sortFilterSubMenuRenderer", "subMenuItems" + ) - private def self.parse(initial_data : Hash(String, JSON::Any)) - body = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", - "content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer", - "initialSegments").as_a + if footer_language_menu + label = footer_language_menu.as_a.select(&.["selected"].as_bool)[0]["title"].as_s + else + label = language_code + end + + # Extract transcript lines + + initial_segments = segment_list["initialSegments"].as_a lines = [] of TranscriptLine - body.each do |line| - # Transcript section headers. They are not apart of the captions and as such we can safely skip them. - if line.as_h.has_key?("transcriptSectionHeaderRenderer") - next + + initial_segments.each do |line| + if unpacked_line = line["transcriptSectionHeaderRenderer"]? + line_type = HeadingLine + else + unpacked_line = line["transcriptSegmentRenderer"] + line_type = RegularLine end - line = line["transcriptSegmentRenderer"] + start_ms = unpacked_line["startMs"].as_s.to_i.millisecond + end_ms = unpacked_line["endMs"].as_s.to_i.millisecond + text = extract_text(unpacked_line["snippet"]) || "" + + lines << line_type.new(start_ms, end_ms, text) + end + + return Transcript.new( + lines: lines, + language_code: language_code, + auto_generated: auto_generated, + label: label + ) + end - start_ms = line["startMs"].as_s.to_i.millisecond - end_ms = line["endMs"].as_s.to_i.millisecond + # Converts transcript lines to a WebVTT file + # + # This is used within Invidious to replace subtitles + # as to workaround YouTube's rate-limited timedtext endpoint. + def to_vtt + settings_field = { + "Kind" => "captions", + "Language" => @language_code, + } - text = extract_text(line["snippet"]) || "" + vtt = WebVTT.build(settings_field) do |vtt| + @lines.each do |line| + # Section headers are excluded from the VTT conversion as to + # match the regular captions returned from YouTube as much as possible + next if line.is_a? HeadingLine - lines << TranscriptLine.new(start_ms, end_ms, text) + vtt.cue(line.start_ms, line.end_ms, line.line) + end end - return lines + return vtt end end end diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr index 09df106d..a84e44bc 100644 --- a/src/invidious/views/channel.ecr +++ b/src/invidious/views/channel.ecr @@ -30,13 +30,13 @@ <meta property="og:site_name" content="Invidious"> <meta property="og:url" content="<%= HOST_URL %>/channel/<%= ucid %>"> <meta property="og:title" content="<%= author %>"> -<meta property="og:image" content="/ggpht<%= channel_profile_pic %>"> +<meta property="og:image" content="<%= HOST_URL %>/ggpht<%= channel_profile_pic %>"> <meta property="og:description" content="<%= channel.description %>"> <meta name="twitter:card" content="summary"> <meta name="twitter:url" content="<%= HOST_URL %>/channel/<%= ucid %>"> <meta name="twitter:title" content="<%= author %>"> <meta name="twitter:description" content="<%= channel.description %>"> -<meta name="twitter:image" content="/ggpht<%= channel_profile_pic %>"> +<meta name="twitter:image" content="<%= HOST_URL %>/ggpht<%= channel_profile_pic %>"> <link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/channel/<%= ucid %>" /> <%- end -%> diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index 031b46da..6d227cfc 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -82,11 +82,19 @@ </div> <div class="video-card-row flexible"> - <div class="flex-left"><a href="/channel/<%= item.ucid %>"> - <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> - <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> - </p> - </a></div> + <div class="flex-left"> + <% if !item.ucid.to_s.empty? %> + <a href="/channel/<%= item.ucid %>"> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + </a> + <% else %> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + <% end %> + </div> </div> <% when Category %> <% else %> @@ -160,11 +168,19 @@ </div> <div class="video-card-row flexible"> - <div class="flex-left"><a href="/channel/<%= item.ucid %>"> - <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> - <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> - </p> - </a></div> + <div class="flex-left"> + <% if !item.ucid.to_s.empty? %> + <a href="/channel/<%= item.ucid %>"> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + </a> + <% else %> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + <% end %> + </div> <%= rendered "components/video-context-buttons" %> </div> diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index 77265679..9904b4fc 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -1,5 +1,9 @@ +<% + locale = env.get("preferences").as(Preferences).locale + dark_mode = env.get("preferences").as(Preferences).dark_mode +%> <!DOCTYPE html> -<html lang="<%= env.get("preferences").as(Preferences).locale %>"> +<html lang="<%= locale %>"> <head> <meta charset="utf-8"> @@ -17,19 +21,14 @@ <link rel="stylesheet" href="/css/grids-responsive-min.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/ionicons.min.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/default.css?v=<%= ASSET_COMMIT %>"> + <link rel="stylesheet" href="/css/carousel.css?v=<%= ASSET_COMMIT %>"> <script src="/js/_helpers.js?v=<%= ASSET_COMMIT %>"></script> </head> -<% - locale = env.get("preferences").as(Preferences).locale - dark_mode = env.get("preferences").as(Preferences).dark_mode -%> - <body class="<%= dark_mode.blank? ? "no" : dark_mode %>-theme"> - <span style="display:none" id="dark_mode_pref"><%= env.get("preferences").as(Preferences).dark_mode %></span> + <span style="display:none" id="dark_mode_pref"><%= dark_mode %></span> <div class="pure-g"> - <div class="pure-u-1 pure-u-md-2-24"></div> - <div class="pure-u-1 pure-u-md-20-24" id="contents"> + <div class="pure-u-1 pure-u-xl-20-24" id="contents"> <div class="pure-g navbar h-box"> <% if navbar_search %> <div class="pure-u-1 pure-u-md-4-24"> @@ -43,8 +42,8 @@ <div class="pure-u-1 pure-u-md-8-24 user-field"> <% if env.get? "user" %> <div class="pure-u-1-4"> - <a id="toggle_theme" href="/toggle_theme?referer=<%= env.get?("current_page") %>" class="pure-menu-heading"> - <% if env.get("preferences").as(Preferences).dark_mode == "dark" %> + <a id="toggle_theme" href="/toggle_theme?referer=<%= env.get?("current_page") %>" class="pure-menu-heading" title="<%= translate(locale, "toggle_theme") %>"> + <% if dark_mode == "dark" %> <i class="icon ion-ios-sunny"></i> <% else %> <i class="icon ion-ios-moon"></i> @@ -81,8 +80,8 @@ </div> <% else %> <div class="pure-u-1-3"> - <a id="toggle_theme" href="/toggle_theme?referer=<%= env.get?("current_page") %>" class="pure-menu-heading"> - <% if env.get("preferences").as(Preferences).dark_mode == "dark" %> + <a id="toggle_theme" href="/toggle_theme?referer=<%= env.get?("current_page") %>" class="pure-menu-heading" title="<%= translate(locale, "toggle_theme") %>"> + <% if dark_mode == "dark" %> <i class="icon ion-ios-sunny"></i> <% else %> <i class="icon ion-ios-moon"></i> @@ -156,7 +155,6 @@ </footer> </div> - <div class="pure-u-1 pure-u-md-2-24"></div> </div> <script src="/js/handlers.js?v=<%= ASSET_COMMIT %>"></script> <script src="/js/themes.js?v=<%= ASSET_COMMIT %>"></script> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 07474896..9e7467dd 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -10,7 +10,7 @@ <meta property="og:site_name" content="<%= author %> | Invidious"> <meta property="og:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>"> <meta property="og:title" content="<%= title %>"> -<meta property="og:image" content="/vi/<%= video.id %>/maxres.jpg"> +<meta property="og:image" content="<%= HOST_URL %>/vi/<%= video.id %>/maxres.jpg"> <meta property="og:description" content="<%= HTML.escape(video.short_description) %>"> <meta property="og:type" content="video.other"> <meta property="og:video:url" content="<%= HOST_URL %>/embed/<%= video.id %>"> @@ -118,7 +118,7 @@ we're going to need to do it here in order to allow for translations. link_yt_embed = URI.new(scheme: "https", host: "www.youtube.com", path: "/embed/#{video.id}") if !plid.nil? && !continuation.nil? - link_yt_param = URI::Params{"plid" => [plid], "index" => [continuation.to_s]} + link_yt_param = URI::Params{"list" => [plid], "index" => [continuation.to_s]} link_yt_watch = IV::HttpServer::Utils.add_params_to_url(link_yt_watch, link_yt_param) link_yt_embed = IV::HttpServer::Utils.add_params_to_url(link_yt_embed, link_yt_param) end @@ -346,7 +346,7 @@ we're going to need to do it here in order to allow for translations. <h5 class="pure-g"> <div class="pure-u-14-24"> - <% if rv["ucid"]? %> + <% if !rv["ucid"].empty? %> <b style="width:100%"><a href="/channel/<%= rv["ucid"] %>"><%= rv["author"]? %><% if rv["author_verified"]? == "true" %> <i class="icon ion ion-md-checkmark-circle"></i><% end %></a></b> <% else %> <b style="width:100%"><%= rv["author"]? %><% if rv["author_verified"]? == "true" %> <i class="icon ion ion-md-checkmark-circle"></i><% end %></b> diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 03ff0ee4..d3dbcc0e 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -1,7 +1,6 @@ def add_yt_headers(request) - if request.headers["User-Agent"] == "Crystal" - request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" - end + request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" + request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" @@ -25,25 +24,20 @@ struct YoutubeConnectionPool @pool = build_pool() end - def client(region = nil, &block) - if region - conn = make_client(url, region) + def client(&block) + conn = pool.checkout + begin response = yield conn - else - conn = pool.checkout - begin - response = yield conn - rescue ex - conn.close - conn = HTTP::Client.new(url) + rescue ex + conn.close + conn = HTTP::Client.new(url) - conn.family = CONFIG.force_resolve - conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC - conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" - response = yield conn - ensure - pool.release(conn) - end + conn.family = CONFIG.force_resolve + conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC + conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" + response = yield conn + ensure + pool.release(conn) end response @@ -60,29 +54,23 @@ struct YoutubeConnectionPool end end -def make_client(url : URI, region = nil) - client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure) - client.family = CONFIG.force_resolve +def make_client(url : URI, region = nil, force_resolve : Bool = false) + client = HTTP::Client.new(url) + + # Force the usage of a specific configured IP Family + if force_resolve + client.family = CONFIG.force_resolve + end + client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" client.read_timeout = 10.seconds client.connect_timeout = 10.seconds - if region - PROXY_LIST[region]?.try &.sample(40).each do |proxy| - begin - proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) - client.set_proxy(proxy) - break - rescue ex - end - end - end - return client end -def make_client(url : URI, region = nil, &block) - client = make_client(url, region) +def make_client(url : URI, region = nil, force_resolve : Bool = false, &block) + client = make_client(url, region, force_resolve) begin yield client ensure diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 56325cf7..0e72957e 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -822,9 +822,9 @@ module HelperExtractors end # Retrieves the ID required for querying the InnerTube browse endpoint. - # Raises when it's unable to do so + # Returns an empty string when it's unable to do so def self.get_browse_id(container) - return container.dig("navigationEndpoint", "browseEndpoint", "browseId").as_s + return container.dig?("navigationEndpoint", "browseEndpoint", "browseId").try &.as_s || "" end end diff --git a/src/invidious/yt_backend/proxy.cr b/src/invidious/yt_backend/proxy.cr deleted file mode 100644 index 2d0fd4ba..00000000 --- a/src/invidious/yt_backend/proxy.cr +++ /dev/null @@ -1,316 +0,0 @@ -# See https://github.com/crystal-lang/crystal/issues/2963 -class HTTPProxy - getter proxy_host : String - getter proxy_port : Int32 - getter options : Hash(Symbol, String) - getter tls : OpenSSL::SSL::Context::Client? - - def initialize(@proxy_host, @proxy_port = 80, @options = {} of Symbol => String) - end - - def open(host, port, tls = nil, connection_options = {} of Symbol => Float64 | Nil) - dns_timeout = connection_options.fetch(:dns_timeout, nil) - connect_timeout = connection_options.fetch(:connect_timeout, nil) - read_timeout = connection_options.fetch(:read_timeout, nil) - - socket = TCPSocket.new @proxy_host, @proxy_port, dns_timeout, connect_timeout - socket.read_timeout = read_timeout if read_timeout - socket.sync = true - - socket << "CONNECT #{host}:#{port} HTTP/1.1\r\n" - - if options[:user]? - credentials = Base64.strict_encode("#{options[:user]}:#{options[:password]}") - credentials = "#{credentials}\n".gsub(/\s/, "") - socket << "Proxy-Authorization: Basic #{credentials}\r\n" - end - - socket << "\r\n" - - resp = parse_response(socket) - - if resp[:code]? == 200 - {% if !flag?(:without_openssl) %} - if tls - tls_socket = OpenSSL::SSL::Socket::Client.new(socket, context: tls, sync_close: true, hostname: host) - socket = tls_socket - end - {% end %} - - return socket - else - socket.close - raise IO::Error.new(resp.inspect) - end - end - - private def parse_response(socket) - resp = {} of Symbol => Int32 | String | Hash(String, String) - - begin - version, code, reason = socket.gets.as(String).chomp.split(/ /, 3) - - headers = {} of String => String - - while (line = socket.gets.as(String)) && (line.chomp != "") - name, value = line.split(/:/, 2) - headers[name.strip] = value.strip - end - - resp[:version] = version - resp[:code] = code.to_i - resp[:reason] = reason - resp[:headers] = headers - rescue - end - - return resp - end -end - -class HTTPClient < HTTP::Client - def set_proxy(proxy : HTTPProxy) - begin - @io = proxy.open(host: @host, port: @port, tls: @tls, connection_options: proxy_connection_options) - rescue IO::Error - @io = nil - end - end - - def unset_proxy - @io = nil - end - - def proxy_connection_options - opts = {} of Symbol => Float64 | Nil - - opts[:dns_timeout] = @dns_timeout - opts[:connect_timeout] = @connect_timeout - opts[:read_timeout] = @read_timeout - - return opts - end -end - -def get_proxies(country_code = "US") - # return get_spys_proxies(country_code) - return get_nova_proxies(country_code) -end - -def filter_proxies(proxies) - proxies.select! do |proxy| - begin - client = HTTPClient.new(YT_URL) - client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" - client.read_timeout = 10.seconds - client.connect_timeout = 10.seconds - - proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) - client.set_proxy(proxy) - - status_ok = client.head("/").status_code == 200 - client.close - status_ok - rescue ex - false - end - end - - return proxies -end - -def get_nova_proxies(country_code = "US") - country_code = country_code.downcase - client = HTTP::Client.new(URI.parse("https://www.proxynova.com")) - client.read_timeout = 10.seconds - client.connect_timeout = 10.seconds - - headers = HTTP::Headers.new - headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36" - headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8" - headers["Accept-Language"] = "Accept-Language: en-US,en;q=0.9" - headers["Host"] = "www.proxynova.com" - headers["Origin"] = "https://www.proxynova.com" - headers["Referer"] = "https://www.proxynova.com/proxy-server-list/country-#{country_code}/" - - response = client.get("/proxy-server-list/country-#{country_code}/", headers) - client.close - document = XML.parse_html(response.body) - - proxies = [] of {ip: String, port: Int32, score: Float64} - document.xpath_nodes(%q(//tr[@data-proxy-id])).each do |node| - ip = node.xpath_node(%q(.//td/abbr/script)).not_nil!.content - ip = ip.match(/document\.write\('(?<sub1>[^']+)'.substr\(8\) \+ '(?<sub2>[^']+)'/).not_nil! - ip = "#{ip["sub1"][8..-1]}#{ip["sub2"]}" - port = node.xpath_node(%q(.//td[2])).not_nil!.content.strip.to_i - - anchor = node.xpath_node(%q(.//td[4]/div)).not_nil! - speed = anchor["data-value"].to_f - latency = anchor["title"].to_f - uptime = node.xpath_node(%q(.//td[5]/span)).not_nil!.content.rchop("%").to_f - - # TODO: Tweak me - score = (uptime*4 + speed*2 + latency)/7 - proxies << {ip: ip, port: port, score: score} - end - - # proxies = proxies.sort_by { |proxy| proxy[:score] }.reverse - return proxies -end - -def get_spys_proxies(country_code = "US") - client = HTTP::Client.new(URI.parse("http://spys.one")) - client.read_timeout = 10.seconds - client.connect_timeout = 10.seconds - - headers = HTTP::Headers.new - headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36" - headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8" - headers["Accept-Language"] = "Accept-Language: en-US,en;q=0.9" - headers["Host"] = "spys.one" - headers["Origin"] = "http://spys.one" - headers["Referer"] = "http://spys.one/free-proxy-list/#{country_code}/" - headers["Content-Type"] = "application/x-www-form-urlencoded" - body = { - "xpp" => "5", - "xf1" => "0", - "xf2" => "0", - "xf4" => "0", - "xf5" => "1", - } - - response = client.post("/free-proxy-list/#{country_code}/", headers, form: body) - client.close - 20.times do - if response.status_code == 200 - break - end - response = client.post("/free-proxy-list/#{country_code}/", headers, form: body) - end - - response = XML.parse_html(response.body) - - mapping = response.xpath_node(%q(.//body/script)).not_nil!.content - mapping = mapping.match(/\}\('(?<p>[^']+)',\d+,\d+,'(?<x>[^']+)'/).not_nil! - p = mapping["p"].not_nil! - x = mapping["x"].not_nil! - mapping = decrypt_port(p, x) - - proxies = [] of {ip: String, port: Int32, score: Float64} - response = response.xpath_node(%q(//tr/td/table)).not_nil! - response.xpath_nodes(%q(.//tr)).each do |node| - if !node["onmouseover"]? - next - end - - ip = node.xpath_node(%q(.//td[1]/font[2])).to_s.match(/<font class="spy14">(?<address>[^<]+)</).not_nil!["address"] - encrypted_port = node.xpath_node(%q(.//td[1]/font[2]/script)).not_nil!.content - encrypted_port = encrypted_port.match(/<\\\/font>"\+(?<encrypted_port>[\d\D]+)\)$/).not_nil!["encrypted_port"] - - port = "" - encrypted_port.split("+").each do |number| - number = number.delete("()") - left_side, right_side = number.split("^") - result = mapping[left_side] ^ mapping[right_side] - port = "#{port}#{result}" - end - port = port.to_i - - latency = node.xpath_node(%q(.//td[6])).not_nil!.content.to_f - speed = node.xpath_node(%q(.//td[7]/font/table)).not_nil!["width"].to_f - uptime = node.xpath_node(%q(.//td[8]/font/acronym)).not_nil! - - # Skip proxies that are down - if uptime["title"].ends_with? "?" - next - end - - if md = uptime.content.match(/^\d+/) - uptime = md[0].to_f - else - next - end - - score = (uptime*4 + speed*2 + latency)/7 - - proxies << {ip: ip, port: port, score: score} - end - - proxies = proxies.sort_by!(&.[:score]).reverse! - return proxies -end - -def decrypt_port(p, x) - x = x.split("^") - s = {} of String => String - - 60.times do |i| - if x[i]?.try &.empty? - s[y_func(i)] = y_func(i) - else - s[y_func(i)] = x[i] - end - end - - x = s - p = p.gsub(/\b\w+\b/, x) - - p = p.split(";") - p = p.map(&.split("=")) - - mapping = {} of String => Int32 - p.each do |item| - if item == [""] - next - end - - key = item[0] - value = item[1] - value = value.split("^") - - if value.size == 1 - value = value[0].to_i - else - left_side = value[0].to_i? - left_side ||= mapping[value[0]] - right_side = value[1].to_i? - right_side ||= mapping[value[1]] - - value = left_side ^ right_side - end - - mapping[key] = value - end - - return mapping -end - -def y_func(c) - return (c < 60 ? "" : y_func((c/60).to_i)) + ((c = c % 60) > 35 ? ((c.to_u8 + 29).unsafe_chr) : c.to_s(36)) -end - -PROXY_LIST = { - "GB" => [{ip: "147.135.206.233", port: 3128}, {ip: "167.114.180.102", port: 8080}, {ip: "176.35.250.108", port: 8080}, {ip: "5.148.128.44", port: 80}, {ip: "62.7.85.234", port: 8080}, {ip: "88.150.135.10", port: 36624}], - "DE" => [{ip: "138.201.223.250", port: 31288}, {ip: "138.68.73.59", port: 32574}, {ip: "159.69.211.173", port: 3128}, {ip: "173.249.43.105", port: 3128}, {ip: "212.202.244.90", port: 8080}, {ip: "5.56.18.35", port: 38827}], - "FR" => [{ip: "137.74.254.242", port: 3128}, {ip: "151.80.143.155", port: 53281}, {ip: "178.33.150.97", port: 3128}, {ip: "37.187.2.31", port: 3128}, {ip: "5.135.164.72", port: 3128}, {ip: "5.39.91.73", port: 3128}, {ip: "51.38.162.2", port: 32231}, {ip: "51.38.217.121", port: 808}, {ip: "51.75.109.81", port: 3128}, {ip: "51.75.109.82", port: 3128}, {ip: "51.75.109.83", port: 3128}, {ip: "51.75.109.84", port: 3128}, {ip: "51.75.109.86", port: 3128}, {ip: "51.75.109.88", port: 3128}, {ip: "51.75.109.90", port: 3128}, {ip: "62.210.167.3", port: 3128}, {ip: "90.63.218.232", port: 8080}, {ip: "91.134.165.198", port: 9999}], - "IN" => [{ip: "1.186.151.206", port: 36253}, {ip: "1.186.63.130", port: 39142}, {ip: "103.105.40.1", port: 16538}, {ip: "103.105.40.153", port: 16538}, {ip: "103.106.148.203", port: 60227}, {ip: "103.106.148.207", port: 51451}, {ip: "103.12.246.12", port: 8080}, {ip: "103.14.235.109", port: 8080}, {ip: "103.14.235.26", port: 8080}, {ip: "103.198.172.4", port: 50820}, {ip: "103.205.112.1", port: 23500}, {ip: "103.209.64.19", port: 6666}, {ip: "103.211.76.5", port: 8080}, {ip: "103.216.82.19", port: 6666}, {ip: "103.216.82.190", port: 6666}, {ip: "103.216.82.209", port: 54806}, {ip: "103.216.82.214", port: 6666}, {ip: "103.216.82.37", port: 6666}, {ip: "103.216.82.44", port: 8080}, {ip: "103.216.82.50", port: 53281}, {ip: "103.22.173.230", port: 8080}, {ip: "103.224.38.2", port: 83}, {ip: "103.226.142.90", port: 41386}, {ip: "103.236.114.38", port: 49638}, {ip: "103.240.161.107", port: 6666}, {ip: "103.240.161.108", port: 6666}, {ip: "103.240.161.109", port: 6666}, {ip: "103.240.161.59", port: 48809}, {ip: "103.245.198.101", port: 8080}, {ip: "103.250.148.82", port: 6666}, {ip: "103.251.58.51", port: 61489}, {ip: "103.253.169.115", port: 32731}, {ip: "103.253.211.182", port: 8080}, {ip: "103.253.211.182", port: 80}, {ip: "103.255.234.169", port: 39847}, {ip: "103.42.161.118", port: 8080}, {ip: "103.42.162.30", port: 8080}, {ip: "103.42.162.50", port: 8080}, {ip: "103.42.162.58", port: 8080}, {ip: "103.46.233.12", port: 83}, {ip: "103.46.233.13", port: 83}, {ip: "103.46.233.16", port: 83}, {ip: "103.46.233.17", port: 83}, {ip: "103.46.233.21", port: 83}, {ip: "103.46.233.23", port: 83}, {ip: "103.46.233.29", port: 81}, {ip: "103.46.233.29", port: 83}, {ip: "103.46.233.50", port: 83}, {ip: "103.47.153.87", port: 8080}, {ip: "103.47.66.2", port: 39804}, {ip: "103.49.53.1", port: 81}, {ip: "103.52.220.1", port: 49068}, {ip: "103.56.228.166", port: 53281}, {ip: "103.56.30.128", port: 8080}, {ip: "103.65.193.17", port: 50862}, {ip: "103.65.195.1", port: 33960}, {ip: "103.69.220.14", port: 3128}, {ip: "103.70.128.84", port: 8080}, {ip: "103.70.128.86", port: 8080}, {ip: "103.70.131.74", port: 8080}, {ip: "103.70.146.250", port: 59563}, {ip: "103.72.216.194", port: 38345}, {ip: "103.75.161.38", port: 21776}, {ip: "103.76.253.155", port: 3128}, {ip: "103.87.104.137", port: 8080}, {ip: "110.235.198.3", port: 57660}, {ip: "114.69.229.161", port: 8080}, {ip: "117.196.231.201", port: 37769}, {ip: "117.211.166.214", port: 3128}, {ip: "117.240.175.51", port: 3128}, {ip: "117.240.210.155", port: 53281}, {ip: "117.240.59.115", port: 36127}, {ip: "117.242.154.73", port: 33889}, {ip: "117.244.15.243", port: 3128}, {ip: "119.235.54.3", port: 8080}, {ip: "120.138.117.102", port: 59308}, {ip: "123.108.200.185", port: 83}, {ip: "123.108.200.217", port: 82}, {ip: "123.176.43.218", port: 40524}, {ip: "125.21.43.82", port: 8080}, {ip: "125.62.192.225", port: 82}, {ip: "125.62.192.33", port: 84}, {ip: "125.62.194.1", port: 83}, {ip: "125.62.213.134", port: 82}, {ip: "125.62.213.18", port: 83}, {ip: "125.62.213.201", port: 84}, {ip: "125.62.213.242", port: 83}, {ip: "125.62.214.185", port: 84}, {ip: "139.5.26.27", port: 53281}, {ip: "14.102.67.101", port: 30337}, {ip: "14.142.122.134", port: 8080}, {ip: "150.129.114.194", port: 6666}, {ip: "150.129.151.62", port: 6666}, {ip: "150.129.171.115", port: 6666}, {ip: "150.129.201.30", port: 6666}, {ip: "157.119.207.38", port: 53281}, {ip: "175.100.185.151", port: 53281}, {ip: "182.18.177.114", port: 56173}, {ip: "182.73.194.170", port: 8080}, {ip: "182.74.85.230", port: 51214}, {ip: "183.82.116.56", port: 8080}, {ip: "183.82.32.56", port: 49551}, {ip: "183.87.14.229", port: 53281}, {ip: "183.87.14.250", port: 44915}, {ip: "202.134.160.168", port: 8080}, {ip: "202.134.166.1", port: 8080}, {ip: "202.134.180.50", port: 8080}, {ip: "202.62.84.210", port: 53281}, {ip: "203.192.193.225", port: 8080}, {ip: "203.192.195.14", port: 31062}, {ip: "203.192.217.11", port: 8080}, {ip: "223.196.83.182", port: 53281}, {ip: "27.116.20.169", port: 36630}, {ip: "27.116.20.209", port: 36630}, {ip: "27.116.51.21", port: 36033}, {ip: "43.224.8.114", port: 50333}, {ip: "43.224.8.116", port: 6666}, {ip: "43.224.8.124", port: 6666}, {ip: "43.224.8.86", port: 6666}, {ip: "43.225.20.73", port: 8080}, {ip: "43.225.23.26", port: 8080}, {ip: "43.230.196.98", port: 36569}, {ip: "43.240.5.225", port: 31777}, {ip: "43.241.28.248", port: 8080}, {ip: "43.242.209.201", port: 8080}, {ip: "43.246.139.82", port: 8080}, {ip: "43.248.73.86", port: 53281}, {ip: "43.251.170.145", port: 54059}, {ip: "45.112.57.230", port: 61222}, {ip: "45.115.171.30", port: 47949}, {ip: "45.121.29.254", port: 54858}, {ip: "45.123.26.146", port: 53281}, {ip: "45.125.61.193", port: 32804}, {ip: "45.125.61.209", port: 32804}, {ip: "45.127.121.194", port: 53281}, {ip: "45.250.226.14", port: 3128}, {ip: "45.250.226.38", port: 8080}, {ip: "45.250.226.47", port: 8080}, {ip: "45.250.226.55", port: 8080}, {ip: "49.249.251.86", port: 53281}], - "CN" => [{ip: "182.61.170.45", port: 3128}], - "RU" => [{ip: "109.106.139.225", port: 45689}, {ip: "109.161.48.228", port: 53281}, {ip: "109.167.224.198", port: 51919}, {ip: "109.172.57.250", port: 23500}, {ip: "109.194.2.126", port: 61822}, {ip: "109.195.150.128", port: 37564}, {ip: "109.201.96.171", port: 31773}, {ip: "109.201.97.204", port: 41258}, {ip: "109.201.97.235", port: 39125}, {ip: "109.206.140.74", port: 45991}, {ip: "109.206.148.31", port: 30797}, {ip: "109.69.75.5", port: 46347}, {ip: "109.71.181.170", port: 53983}, {ip: "109.74.132.190", port: 42663}, {ip: "109.74.143.45", port: 36529}, {ip: "109.75.140.158", port: 59916}, {ip: "109.95.84.114", port: 52125}, {ip: "130.255.12.24", port: 31004}, {ip: "134.19.147.72", port: 44812}, {ip: "134.90.181.7", port: 54353}, {ip: "145.255.6.171", port: 31252}, {ip: "146.120.227.3", port: 8080}, {ip: "149.255.112.194", port: 48968}, {ip: "158.46.127.222", port: 52574}, {ip: "158.46.43.144", port: 39120}, {ip: "158.58.130.185", port: 50016}, {ip: "158.58.132.12", port: 56962}, {ip: "158.58.133.106", port: 41258}, {ip: "158.58.133.13", port: 21213}, {ip: "176.101.0.47", port: 34471}, {ip: "176.101.89.226", port: 33470}, {ip: "176.106.12.65", port: 30120}, {ip: "176.107.80.110", port: 58901}, {ip: "176.110.121.9", port: 46322}, {ip: "176.110.121.90", port: 21776}, {ip: "176.111.97.18", port: 8080}, {ip: "176.112.106.230", port: 33996}, {ip: "176.112.110.40", port: 61142}, {ip: "176.113.116.70", port: 55589}, {ip: "176.113.27.192", port: 47337}, {ip: "176.115.197.118", port: 8080}, {ip: "176.117.255.182", port: 53100}, {ip: "176.120.200.69", port: 44331}, {ip: "176.124.123.93", port: 41258}, {ip: "176.192.124.98", port: 60787}, {ip: "176.192.5.238", port: 61227}, {ip: "176.192.8.206", port: 39422}, {ip: "176.193.15.94", port: 8080}, {ip: "176.196.195.170", port: 48129}, {ip: "176.196.198.154", port: 35252}, {ip: "176.196.238.234", port: 44648}, {ip: "176.196.239.46", port: 35656}, {ip: "176.196.246.6", port: 53281}, {ip: "176.196.84.138", port: 51336}, {ip: "176.197.145.246", port: 32649}, {ip: "176.197.99.142", port: 47278}, {ip: "176.215.1.108", port: 60339}, {ip: "176.215.170.147", port: 35604}, {ip: "176.56.23.14", port: 35340}, {ip: "176.62.185.54", port: 53883}, {ip: "176.74.13.110", port: 8080}, {ip: "178.130.29.226", port: 53295}, {ip: "178.170.254.178", port: 46788}, {ip: "178.213.13.136", port: 53281}, {ip: "178.218.104.8", port: 49707}, {ip: "178.219.183.163", port: 8080}, {ip: "178.237.180.34", port: 57307}, {ip: "178.57.101.212", port: 38020}, {ip: "178.57.101.235", port: 31309}, {ip: "178.64.190.133", port: 46688}, {ip: "178.75.1.111", port: 50411}, {ip: "178.75.27.131", port: 41879}, {ip: "185.13.35.178", port: 40654}, {ip: "185.15.189.67", port: 30215}, {ip: "185.175.119.137", port: 41258}, {ip: "185.18.111.194", port: 41258}, {ip: "185.19.176.237", port: 53281}, {ip: "185.190.40.115", port: 31747}, {ip: "185.216.195.134", port: 61287}, {ip: "185.22.172.94", port: 10010}, {ip: "185.22.172.94", port: 1448}, {ip: "185.22.174.65", port: 10010}, {ip: "185.22.174.65", port: 1448}, {ip: "185.23.64.100", port: 3130}, {ip: "185.23.82.39", port: 59248}, {ip: "185.233.94.105", port: 59288}, {ip: "185.233.94.146", port: 57736}, {ip: "185.3.68.54", port: 53500}, {ip: "185.32.120.177", port: 60724}, {ip: "185.34.20.164", port: 53700}, {ip: "185.34.23.43", port: 63238}, {ip: "185.51.60.141", port: 39935}, {ip: "185.61.92.228", port: 33060}, {ip: "185.61.93.67", port: 49107}, {ip: "185.7.233.66", port: 53504}, {ip: "185.72.225.10", port: 56285}, {ip: "185.75.5.158", port: 60819}, {ip: "185.9.86.186", port: 39345}, {ip: "188.133.136.10", port: 47113}, {ip: "188.168.75.254", port: 56899}, {ip: "188.170.41.6", port: 60332}, {ip: "188.187.189.142", port: 38264}, {ip: "188.234.151.103", port: 8080}, {ip: "188.235.11.88", port: 57143}, {ip: "188.235.137.196", port: 23500}, {ip: "188.244.175.2", port: 8080}, {ip: "188.255.82.136", port: 53281}, {ip: "188.43.4.117", port: 60577}, {ip: "188.68.95.166", port: 41258}, {ip: "188.92.242.180", port: 52048}, {ip: "188.93.242.213", port: 49774}, {ip: "192.162.193.243", port: 36910}, {ip: "192.162.214.11", port: 41258}, {ip: "193.106.170.133", port: 38591}, {ip: "193.232.113.244", port: 40412}, {ip: "193.232.234.130", port: 61932}, {ip: "193.242.177.105", port: 53281}, {ip: "193.242.178.50", port: 52376}, {ip: "193.242.178.90", port: 8080}, {ip: "193.33.101.152", port: 34611}, {ip: "194.114.128.149", port: 61213}, {ip: "194.135.15.146", port: 59328}, {ip: "194.135.216.178", port: 56805}, {ip: "194.135.75.74", port: 41258}, {ip: "194.146.201.67", port: 53281}, {ip: "194.186.18.46", port: 56408}, {ip: "194.186.20.62", port: 21231}, {ip: "194.190.171.214", port: 43960}, {ip: "194.9.27.82", port: 42720}, {ip: "195.133.232.58", port: 41733}, {ip: "195.14.114.116", port: 59530}, {ip: "195.14.114.24", port: 56897}, {ip: "195.158.250.97", port: 41582}, {ip: "195.16.48.142", port: 36083}, {ip: "195.191.183.169", port: 47238}, {ip: "195.206.45.112", port: 53281}, {ip: "195.208.172.70", port: 8080}, {ip: "195.209.141.67", port: 31927}, {ip: "195.209.176.2", port: 8080}, {ip: "195.210.144.166", port: 30088}, {ip: "195.211.160.88", port: 44464}, {ip: "195.218.144.182", port: 31705}, {ip: "195.46.168.147", port: 8080}, {ip: "195.9.188.78", port: 53281}, {ip: "195.9.209.10", port: 35242}, {ip: "195.9.223.246", port: 52098}, {ip: "195.9.237.66", port: 8080}, {ip: "195.9.91.66", port: 33199}, {ip: "195.91.132.20", port: 19600}, {ip: "195.98.183.82", port: 30953}, {ip: "212.104.82.246", port: 36495}, {ip: "212.119.229.18", port: 33852}, {ip: "212.13.97.122", port: 30466}, {ip: "212.19.21.19", port: 53264}, {ip: "212.19.5.157", port: 58442}, {ip: "212.19.8.223", port: 30281}, {ip: "212.19.8.239", port: 55602}, {ip: "212.192.202.207", port: 4550}, {ip: "212.22.80.224", port: 34822}, {ip: "212.26.247.178", port: 38418}, {ip: "212.33.228.161", port: 37971}, {ip: "212.33.243.83", port: 38605}, {ip: "212.34.53.126", port: 44369}, {ip: "212.5.107.81", port: 56481}, {ip: "212.7.230.7", port: 51405}, {ip: "212.77.138.161", port: 41258}, {ip: "213.108.221.201", port: 32800}, {ip: "213.109.7.135", port: 59918}, {ip: "213.128.9.204", port: 35549}, {ip: "213.134.196.12", port: 38723}, {ip: "213.168.37.86", port: 8080}, {ip: "213.187.118.184", port: 53281}, {ip: "213.21.23.98", port: 53281}, {ip: "213.210.67.166", port: 53281}, {ip: "213.234.0.242", port: 56503}, {ip: "213.247.192.131", port: 41258}, {ip: "213.251.226.208", port: 56900}, {ip: "213.33.155.80", port: 44387}, {ip: "213.33.199.194", port: 36411}, {ip: "213.33.224.82", port: 8080}, {ip: "213.59.153.19", port: 53281}, {ip: "217.10.45.103", port: 8080}, {ip: "217.107.197.39", port: 33628}, {ip: "217.116.60.66", port: 21231}, {ip: "217.195.87.58", port: 41258}, {ip: "217.197.239.54", port: 34463}, {ip: "217.74.161.42", port: 34175}, {ip: "217.8.84.76", port: 46378}, {ip: "31.131.67.14", port: 8080}, {ip: "31.132.127.142", port: 35432}, {ip: "31.132.218.252", port: 32423}, {ip: "31.173.17.118", port: 51317}, {ip: "31.193.124.70", port: 53281}, {ip: "31.210.211.147", port: 8080}, {ip: "31.220.183.217", port: 53281}, {ip: "31.29.212.82", port: 35066}, {ip: "31.42.254.24", port: 30912}, {ip: "31.47.189.14", port: 38473}, {ip: "37.113.129.98", port: 41665}, {ip: "37.192.103.164", port: 34835}, {ip: "37.192.194.50", port: 50165}, {ip: "37.192.99.151", port: 51417}, {ip: "37.205.83.91", port: 35888}, {ip: "37.233.85.155", port: 53281}, {ip: "37.235.167.66", port: 53281}, {ip: "37.235.65.2", port: 47816}, {ip: "37.235.67.178", port: 34450}, {ip: "37.9.134.133", port: 41262}, {ip: "46.150.174.90", port: 53281}, {ip: "46.151.156.198", port: 56013}, {ip: "46.16.226.10", port: 8080}, {ip: "46.163.131.55", port: 48306}, {ip: "46.173.191.51", port: 53281}, {ip: "46.174.222.61", port: 34977}, {ip: "46.180.96.79", port: 42319}, {ip: "46.181.151.79", port: 39386}, {ip: "46.21.74.130", port: 8080}, {ip: "46.227.162.98", port: 51558}, {ip: "46.229.187.169", port: 53281}, {ip: "46.229.67.198", port: 47437}, {ip: "46.243.179.221", port: 41598}, {ip: "46.254.217.54", port: 53281}, {ip: "46.32.68.188", port: 39707}, {ip: "46.39.224.112", port: 36765}, {ip: "46.63.162.171", port: 8080}, {ip: "46.73.33.253", port: 8080}, {ip: "5.128.32.12", port: 51959}, {ip: "5.129.155.3", port: 51390}, {ip: "5.129.16.27", port: 48935}, {ip: "5.141.81.65", port: 61853}, {ip: "5.16.15.234", port: 8080}, {ip: "5.167.51.235", port: 8080}, {ip: "5.167.96.238", port: 3128}, {ip: "5.19.165.235", port: 30793}, {ip: "5.35.93.157", port: 31773}, {ip: "5.59.137.90", port: 8888}, {ip: "5.8.207.160", port: 57192}, {ip: "62.122.97.66", port: 59143}, {ip: "62.148.151.253", port: 53570}, {ip: "62.152.85.158", port: 31156}, {ip: "62.165.54.153", port: 55522}, {ip: "62.173.140.14", port: 8080}, {ip: "62.173.155.206", port: 41258}, {ip: "62.182.206.19", port: 37715}, {ip: "62.213.14.166", port: 8080}, {ip: "62.76.123.224", port: 8080}, {ip: "77.221.220.133", port: 44331}, {ip: "77.232.153.248", port: 60950}, {ip: "77.233.10.37", port: 54210}, {ip: "77.244.27.109", port: 47554}, {ip: "77.37.142.203", port: 53281}, {ip: "77.39.29.29", port: 49243}, {ip: "77.75.6.34", port: 8080}, {ip: "77.87.102.7", port: 42601}, {ip: "77.94.121.212", port: 36896}, {ip: "77.94.121.51", port: 45293}, {ip: "78.110.154.177", port: 59888}, {ip: "78.140.201.226", port: 8090}, {ip: "78.153.4.122", port: 9001}, {ip: "78.156.225.170", port: 41258}, {ip: "78.156.243.146", port: 59730}, {ip: "78.29.14.201", port: 39001}, {ip: "78.81.24.112", port: 8080}, {ip: "78.85.36.203", port: 8080}, {ip: "79.104.219.125", port: 3128}, {ip: "79.104.55.134", port: 8080}, {ip: "79.137.181.170", port: 8080}, {ip: "79.173.124.194", port: 47832}, {ip: "79.173.124.207", port: 53281}, {ip: "79.174.186.168", port: 45710}, {ip: "79.175.51.13", port: 54853}, {ip: "79.175.57.77", port: 55477}, {ip: "80.234.107.118", port: 56952}, {ip: "80.237.6.1", port: 34880}, {ip: "80.243.14.182", port: 49320}, {ip: "80.251.48.215", port: 45157}, {ip: "80.254.121.66", port: 41055}, {ip: "80.254.125.236", port: 80}, {ip: "80.72.121.185", port: 52379}, {ip: "80.89.133.210", port: 3128}, {ip: "80.91.17.113", port: 41258}, {ip: "81.162.61.166", port: 40392}, {ip: "81.163.57.121", port: 41258}, {ip: "81.163.57.46", port: 41258}, {ip: "81.163.62.136", port: 41258}, {ip: "81.23.112.98", port: 55269}, {ip: "81.23.118.106", port: 60427}, {ip: "81.23.177.245", port: 8080}, {ip: "81.24.126.166", port: 8080}, {ip: "81.30.216.147", port: 41258}, {ip: "81.95.131.10", port: 44292}, {ip: "82.114.125.22", port: 8080}, {ip: "82.151.208.20", port: 8080}, {ip: "83.221.216.110", port: 47326}, {ip: "83.246.139.24", port: 8080}, {ip: "83.97.108.8", port: 41258}, {ip: "84.22.154.76", port: 8080}, {ip: "84.52.110.36", port: 38674}, {ip: "84.52.74.194", port: 8080}, {ip: "84.52.77.227", port: 41806}, {ip: "84.52.79.166", port: 43548}, {ip: "84.52.84.157", port: 44331}, {ip: "84.52.88.125", port: 32666}, {ip: "85.113.48.148", port: 8080}, {ip: "85.113.49.220", port: 8080}, {ip: "85.12.193.210", port: 58470}, {ip: "85.15.179.5", port: 8080}, {ip: "85.173.244.102", port: 53281}, {ip: "85.174.227.52", port: 59280}, {ip: "85.192.184.133", port: 8080}, {ip: "85.192.184.133", port: 80}, {ip: "85.21.240.193", port: 55820}, {ip: "85.21.63.219", port: 53281}, {ip: "85.235.190.18", port: 42494}, {ip: "85.237.56.193", port: 8080}, {ip: "85.91.119.6", port: 8080}, {ip: "86.102.116.30", port: 8080}, {ip: "86.110.30.146", port: 38109}, {ip: "87.117.3.129", port: 3128}, {ip: "87.225.108.195", port: 8080}, {ip: "87.228.103.111", port: 8080}, {ip: "87.228.103.43", port: 8080}, {ip: "87.229.143.10", port: 48872}, {ip: "87.249.205.103", port: 8080}, {ip: "87.249.21.193", port: 43079}, {ip: "87.255.13.217", port: 8080}, {ip: "88.147.159.167", port: 53281}, {ip: "88.200.225.32", port: 38583}, {ip: "88.204.59.177", port: 32666}, {ip: "88.84.209.69", port: 30819}, {ip: "88.87.72.72", port: 8080}, {ip: "88.87.79.20", port: 8080}, {ip: "88.87.91.163", port: 48513}, {ip: "88.87.93.20", port: 33277}, {ip: "89.109.12.82", port: 47972}, {ip: "89.109.21.43", port: 9090}, {ip: "89.109.239.183", port: 41041}, {ip: "89.109.54.137", port: 36469}, {ip: "89.17.37.218", port: 52957}, {ip: "89.189.130.103", port: 32626}, {ip: "89.189.159.214", port: 42530}, {ip: "89.189.174.121", port: 52636}, {ip: "89.23.18.29", port: 53281}, {ip: "89.249.251.21", port: 3128}, {ip: "89.250.149.114", port: 60981}, {ip: "89.250.17.209", port: 8080}, {ip: "89.250.19.173", port: 8080}, {ip: "90.150.87.172", port: 81}, {ip: "90.154.125.173", port: 33078}, {ip: "90.188.38.81", port: 60585}, {ip: "90.189.151.183", port: 32601}, {ip: "91.103.208.114", port: 57063}, {ip: "91.122.100.222", port: 44331}, {ip: "91.122.207.229", port: 8080}, {ip: "91.144.139.93", port: 3128}, {ip: "91.144.142.19", port: 44617}, {ip: "91.146.16.54", port: 57902}, {ip: "91.190.116.194", port: 38783}, {ip: "91.190.80.100", port: 31659}, {ip: "91.190.85.97", port: 34286}, {ip: "91.203.36.188", port: 8080}, {ip: "91.205.131.102", port: 8080}, {ip: "91.205.146.25", port: 37501}, {ip: "91.210.94.212", port: 52635}, {ip: "91.213.23.110", port: 8080}, {ip: "91.215.22.51", port: 53305}, {ip: "91.217.42.3", port: 8080}, {ip: "91.217.42.4", port: 8080}, {ip: "91.220.135.146", port: 41258}, {ip: "91.222.167.213", port: 38057}, {ip: "91.226.140.71", port: 33199}, {ip: "91.235.7.216", port: 59067}, {ip: "92.124.195.22", port: 3128}, {ip: "92.126.193.180", port: 8080}, {ip: "92.241.110.223", port: 53281}, {ip: "92.252.240.1", port: 53281}, {ip: "92.255.164.187", port: 3128}, {ip: "92.255.195.57", port: 53281}, {ip: "92.255.229.146", port: 55785}, {ip: "92.255.5.2", port: 41012}, {ip: "92.38.32.36", port: 56113}, {ip: "92.39.138.98", port: 31150}, {ip: "92.51.16.155", port: 46202}, {ip: "92.55.59.63", port: 33030}, {ip: "93.170.112.200", port: 47995}, {ip: "93.183.86.185", port: 53281}, {ip: "93.188.45.157", port: 8080}, {ip: "93.81.246.5", port: 53281}, {ip: "93.91.112.247", port: 41258}, {ip: "94.127.217.66", port: 40115}, {ip: "94.154.85.214", port: 8080}, {ip: "94.180.106.94", port: 32767}, {ip: "94.180.249.187", port: 38051}, {ip: "94.230.243.6", port: 8080}, {ip: "94.232.57.231", port: 51064}, {ip: "94.24.244.170", port: 48936}, {ip: "94.242.55.108", port: 10010}, {ip: "94.242.55.108", port: 1448}, {ip: "94.242.57.136", port: 10010}, {ip: "94.242.57.136", port: 1448}, {ip: "94.242.58.108", port: 10010}, {ip: "94.242.58.108", port: 1448}, {ip: "94.242.58.14", port: 10010}, {ip: "94.242.58.14", port: 1448}, {ip: "94.242.58.142", port: 10010}, {ip: "94.242.58.142", port: 1448}, {ip: "94.242.59.245", port: 10010}, {ip: "94.242.59.245", port: 1448}, {ip: "94.247.241.70", port: 53640}, {ip: "94.247.62.165", port: 33176}, {ip: "94.253.13.228", port: 54935}, {ip: "94.253.14.187", port: 55045}, {ip: "94.28.94.154", port: 46966}, {ip: "94.73.217.125", port: 40858}, {ip: "95.140.19.9", port: 8080}, {ip: "95.140.20.94", port: 33994}, {ip: "95.154.137.66", port: 41258}, {ip: "95.154.159.119", port: 44242}, {ip: "95.154.82.254", port: 52484}, {ip: "95.161.157.227", port: 43170}, {ip: "95.161.182.146", port: 33877}, {ip: "95.161.189.26", port: 61522}, {ip: "95.165.163.146", port: 8888}, {ip: "95.165.172.90", port: 60496}, {ip: "95.165.182.18", port: 38950}, {ip: "95.165.203.222", port: 33805}, {ip: "95.165.244.122", port: 58162}, {ip: "95.167.123.54", port: 58664}, {ip: "95.167.241.242", port: 49636}, {ip: "95.171.1.92", port: 35956}, {ip: "95.172.52.230", port: 35989}, {ip: "95.181.35.30", port: 40804}, {ip: "95.181.56.178", port: 39144}, {ip: "95.181.75.228", port: 53281}, {ip: "95.188.74.194", port: 57122}, {ip: "95.189.112.214", port: 35508}, {ip: "95.31.10.247", port: 30711}, {ip: "95.31.197.77", port: 41651}, {ip: "95.31.2.199", port: 33632}, {ip: "95.71.125.50", port: 49882}, {ip: "95.73.62.13", port: 32185}, {ip: "95.79.36.55", port: 44861}, {ip: "95.79.55.196", port: 53281}, {ip: "95.79.99.148", port: 3128}, {ip: "95.80.65.39", port: 43555}, {ip: "95.80.93.44", port: 41258}, {ip: "95.80.98.41", port: 8080}, {ip: "95.83.156.250", port: 58438}, {ip: "95.84.128.25", port: 33765}, {ip: "95.84.154.73", port: 57423}], - "CA" => [{ip: "144.217.161.149", port: 8080}, {ip: "24.37.9.6", port: 54154}, {ip: "54.39.138.144", port: 3128}, {ip: "54.39.138.145", port: 3128}, {ip: "54.39.138.151", port: 3128}, {ip: "54.39.138.152", port: 3128}, {ip: "54.39.138.153", port: 3128}, {ip: "54.39.138.154", port: 3128}, {ip: "54.39.138.155", port: 3128}, {ip: "54.39.138.156", port: 3128}, {ip: "54.39.138.157", port: 3128}, {ip: "54.39.53.104", port: 3128}, {ip: "66.70.167.113", port: 3128}, {ip: "66.70.167.116", port: 3128}, {ip: "66.70.167.117", port: 3128}, {ip: "66.70.167.119", port: 3128}, {ip: "66.70.167.120", port: 3128}, {ip: "66.70.167.125", port: 3128}, {ip: "66.70.188.148", port: 3128}, {ip: "70.35.213.229", port: 36127}, {ip: "70.65.233.174", port: 8080}, {ip: "72.139.24.66", port: 38861}, {ip: "74.15.191.160", port: 41564}], - "JP" => [{ip: "47.91.20.67", port: 8080}, {ip: "61.118.35.94", port: 55725}], - "IT" => [{ip: "109.70.201.97", port: 53517}, {ip: "176.31.82.212", port: 8080}, {ip: "185.132.228.118", port: 55583}, {ip: "185.49.58.88", port: 56006}, {ip: "185.94.89.179", port: 41258}, {ip: "213.203.134.10", port: 41258}, {ip: "217.61.172.12", port: 41369}, {ip: "46.232.143.126", port: 41258}, {ip: "46.232.143.253", port: 41258}, {ip: "93.67.154.125", port: 8080}, {ip: "93.67.154.125", port: 80}, {ip: "95.169.95.242", port: 53803}], - "TH" => [{ip: "1.10.184.166", port: 57330}, {ip: "1.10.186.100", port: 55011}, {ip: "1.10.186.209", port: 32431}, {ip: "1.10.186.245", port: 34360}, {ip: "1.10.186.93", port: 53711}, {ip: "1.10.187.118", port: 62000}, {ip: "1.10.187.34", port: 51635}, {ip: "1.10.187.43", port: 38715}, {ip: "1.10.188.181", port: 51093}, {ip: "1.10.188.83", port: 31940}, {ip: "1.10.188.95", port: 30593}, {ip: "1.10.189.58", port: 48564}, {ip: "1.179.157.237", port: 46178}, {ip: "1.179.164.213", port: 8080}, {ip: "1.179.198.37", port: 8080}, {ip: "1.20.100.99", port: 53794}, {ip: "1.20.101.221", port: 55707}, {ip: "1.20.101.254", port: 35394}, {ip: "1.20.101.80", port: 36234}, {ip: "1.20.102.133", port: 40296}, {ip: "1.20.103.13", port: 40544}, {ip: "1.20.103.56", port: 55422}, {ip: "1.20.96.234", port: 53142}, {ip: "1.20.97.54", port: 60122}, {ip: "1.20.99.63", port: 32123}, {ip: "101.108.92.20", port: 8080}, {ip: "101.109.143.71", port: 36127}, {ip: "101.51.141.110", port: 42860}, {ip: "101.51.141.60", port: 60417}, {ip: "103.246.17.237", port: 3128}, {ip: "110.164.73.131", port: 8080}, {ip: "110.164.87.80", port: 35844}, {ip: "110.77.134.106", port: 8080}, {ip: "113.53.29.92", port: 47297}, {ip: "113.53.83.192", port: 32780}, {ip: "113.53.83.195", port: 35686}, {ip: "113.53.91.214", port: 8080}, {ip: "115.87.27.0", port: 53276}, {ip: "118.172.211.3", port: 58535}, {ip: "118.172.211.40", port: 30430}, {ip: "118.174.196.174", port: 23500}, {ip: "118.174.196.203", port: 23500}, {ip: "118.174.220.107", port: 41222}, {ip: "118.174.220.110", port: 39025}, {ip: "118.174.220.115", port: 41011}, {ip: "118.174.220.118", port: 59556}, {ip: "118.174.220.136", port: 55041}, {ip: "118.174.220.163", port: 31561}, {ip: "118.174.220.168", port: 47455}, {ip: "118.174.220.231", port: 40924}, {ip: "118.174.220.238", port: 46326}, {ip: "118.174.234.13", port: 53084}, {ip: "118.174.234.26", port: 41926}, {ip: "118.174.234.32", port: 57403}, {ip: "118.174.234.59", port: 59149}, {ip: "118.174.234.68", port: 42626}, {ip: "118.174.234.83", port: 38006}, {ip: "118.175.207.104", port: 38959}, {ip: "118.175.244.111", port: 8080}, {ip: "118.175.93.207", port: 50738}, {ip: "122.154.38.53", port: 8080}, {ip: "122.154.59.6", port: 8080}, {ip: "122.154.72.102", port: 8080}, {ip: "122.155.222.98", port: 3128}, {ip: "124.121.22.121", port: 61699}, {ip: "125.24.156.16", port: 44321}, {ip: "125.25.165.105", port: 33850}, {ip: "125.25.165.111", port: 40808}, {ip: "125.25.165.42", port: 47221}, {ip: "125.25.201.14", port: 30100}, {ip: "125.26.99.135", port: 55637}, {ip: "125.26.99.141", port: 38537}, {ip: "125.26.99.148", port: 31818}, {ip: "134.236.247.137", port: 8080}, {ip: "159.192.98.224", port: 3128}, {ip: "171.100.2.154", port: 8080}, {ip: "171.100.9.126", port: 49163}, {ip: "180.180.156.116", port: 48431}, {ip: "180.180.156.46", port: 48507}, {ip: "180.180.156.87", port: 36628}, {ip: "180.180.218.204", port: 51565}, {ip: "180.180.8.34", port: 8080}, {ip: "182.52.238.125", port: 58861}, {ip: "182.52.74.73", port: 36286}, {ip: "182.52.74.76", port: 34084}, {ip: "182.52.74.77", port: 34825}, {ip: "182.52.74.78", port: 48708}, {ip: "182.52.90.45", port: 53799}, {ip: "182.53.206.155", port: 34307}, {ip: "182.53.206.43", port: 45330}, {ip: "182.53.206.49", port: 54228}, {ip: "183.88.212.141", port: 8080}, {ip: "183.88.212.184", port: 8080}, {ip: "183.88.213.85", port: 8080}, {ip: "183.88.214.47", port: 8080}, {ip: "184.82.128.211", port: 8080}, {ip: "202.183.201.13", port: 8081}, {ip: "202.29.20.151", port: 43083}, {ip: "203.150.172.151", port: 8080}, {ip: "27.131.157.94", port: 8080}, {ip: "27.145.100.22", port: 8080}, {ip: "27.145.100.243", port: 8080}, {ip: "49.231.196.114", port: 53281}, {ip: "58.97.72.83", port: 8080}, {ip: "61.19.145.66", port: 8080}], - "ES" => [{ip: "185.198.184.14", port: 48122}, {ip: "185.26.226.241", port: 36012}, {ip: "194.224.188.82", port: 3128}, {ip: "195.235.68.61", port: 3128}, {ip: "195.53.237.122", port: 3128}, {ip: "195.53.86.82", port: 3128}, {ip: "213.96.245.47", port: 8080}, {ip: "217.125.71.214", port: 33950}, {ip: "62.14.178.72", port: 53281}, {ip: "80.35.254.42", port: 53281}, {ip: "81.33.4.214", port: 61711}, {ip: "83.175.238.170", port: 53281}, {ip: "85.217.137.77", port: 3128}, {ip: "90.170.205.178", port: 33680}, {ip: "93.156.177.91", port: 53281}, {ip: "95.60.152.139", port: 37995}], - "AE" => [{ip: "178.32.5.90", port: 36159}], - "KR" => [{ip: "112.217.219.179", port: 3128}, {ip: "114.141.229.2", port: 58115}, {ip: "121.139.218.165", port: 31409}, {ip: "122.49.112.2", port: 38592}, {ip: "61.42.18.132", port: 53281}], - "BR" => [{ip: "128.201.97.157", port: 53281}, {ip: "128.201.97.158", port: 53281}, {ip: "131.0.246.157", port: 35252}, {ip: "131.161.26.90", port: 8080}, {ip: "131.72.143.100", port: 41396}, {ip: "138.0.24.66", port: 53281}, {ip: "138.121.130.50", port: 50600}, {ip: "138.121.155.127", port: 61932}, {ip: "138.121.32.133", port: 23492}, {ip: "138.185.176.63", port: 53281}, {ip: "138.204.233.190", port: 53281}, {ip: "138.204.233.242", port: 53281}, {ip: "138.219.71.74", port: 52688}, {ip: "138.36.107.24", port: 41184}, {ip: "138.94.115.166", port: 8080}, {ip: "143.0.188.161", port: 53281}, {ip: "143.202.218.135", port: 8080}, {ip: "143.208.2.42", port: 53281}, {ip: "143.208.79.223", port: 8080}, {ip: "143.255.52.102", port: 40687}, {ip: "143.255.52.116", port: 57856}, {ip: "143.255.52.117", port: 37279}, {ip: "144.217.22.128", port: 8080}, {ip: "168.0.8.225", port: 8080}, {ip: "168.0.8.55", port: 8080}, {ip: "168.121.139.54", port: 40056}, {ip: "168.181.168.23", port: 53281}, {ip: "168.181.170.198", port: 31935}, {ip: "168.232.198.25", port: 32009}, {ip: "168.232.198.35", port: 42267}, {ip: "168.232.207.145", port: 46342}, {ip: "170.0.104.107", port: 60337}, {ip: "170.0.112.2", port: 50359}, {ip: "170.0.112.229", port: 50359}, {ip: "170.238.118.107", port: 34314}, {ip: "170.239.144.9", port: 3128}, {ip: "170.247.29.138", port: 8080}, {ip: "170.81.237.36", port: 37124}, {ip: "170.84.51.74", port: 53281}, {ip: "170.84.60.222", port: 42981}, {ip: "177.10.202.67", port: 8080}, {ip: "177.101.60.86", port: 80}, {ip: "177.103.231.211", port: 55091}, {ip: "177.12.80.50", port: 50556}, {ip: "177.131.13.9", port: 20183}, {ip: "177.135.178.115", port: 42510}, {ip: "177.135.248.75", port: 20183}, {ip: "177.184.206.238", port: 39508}, {ip: "177.185.148.46", port: 58623}, {ip: "177.200.83.238", port: 8080}, {ip: "177.21.24.146", port: 666}, {ip: "177.220.188.120", port: 47556}, {ip: "177.220.188.213", port: 8080}, {ip: "177.222.229.243", port: 23500}, {ip: "177.234.161.42", port: 8080}, {ip: "177.36.11.241", port: 3128}, {ip: "177.36.12.193", port: 23500}, {ip: "177.37.199.175", port: 49608}, {ip: "177.39.187.70", port: 37315}, {ip: "177.44.175.199", port: 8080}, {ip: "177.46.148.126", port: 3128}, {ip: "177.46.148.142", port: 3128}, {ip: "177.47.194.98", port: 21231}, {ip: "177.5.98.58", port: 20183}, {ip: "177.52.55.19", port: 60901}, {ip: "177.54.200.66", port: 57526}, {ip: "177.55.255.74", port: 37147}, {ip: "177.67.217.94", port: 53281}, {ip: "177.73.248.6", port: 54381}, {ip: "177.73.4.234", port: 23500}, {ip: "177.75.143.211", port: 35955}, {ip: "177.75.161.206", port: 3128}, {ip: "177.75.86.49", port: 20183}, {ip: "177.8.216.106", port: 8080}, {ip: "177.8.216.114", port: 8080}, {ip: "177.8.37.247", port: 56052}, {ip: "177.84.216.17", port: 50569}, {ip: "177.85.200.254", port: 53095}, {ip: "177.87.169.1", port: 53281}, {ip: "179.107.97.178", port: 3128}, {ip: "179.109.144.25", port: 8080}, {ip: "179.109.193.137", port: 53281}, {ip: "179.189.125.206", port: 8080}, {ip: "179.97.30.46", port: 53100}, {ip: "186.192.195.220", port: 38983}, {ip: "186.193.11.226", port: 48999}, {ip: "186.193.26.106", port: 3128}, {ip: "186.208.220.248", port: 3128}, {ip: "186.209.243.142", port: 3128}, {ip: "186.209.243.233", port: 3128}, {ip: "186.211.106.227", port: 34334}, {ip: "186.211.160.178", port: 36756}, {ip: "186.215.133.170", port: 20183}, {ip: "186.216.81.21", port: 31773}, {ip: "186.219.214.13", port: 32708}, {ip: "186.224.94.6", port: 48957}, {ip: "186.225.97.246", port: 43082}, {ip: "186.226.171.163", port: 48698}, {ip: "186.226.179.2", port: 56089}, {ip: "186.226.234.67", port: 33834}, {ip: "186.228.147.58", port: 20183}, {ip: "186.233.97.163", port: 8888}, {ip: "186.248.170.82", port: 53281}, {ip: "186.249.213.101", port: 53482}, {ip: "186.249.213.65", port: 52018}, {ip: "186.250.213.225", port: 60774}, {ip: "186.250.96.70", port: 8080}, {ip: "186.250.96.77", port: 8080}, {ip: "187.1.43.246", port: 53396}, {ip: "187.108.36.250", port: 20183}, {ip: "187.108.38.10", port: 20183}, {ip: "187.109.36.251", port: 20183}, {ip: "187.109.40.9", port: 20183}, {ip: "187.109.56.101", port: 20183}, {ip: "187.111.90.89", port: 53281}, {ip: "187.115.10.50", port: 20183}, {ip: "187.19.62.7", port: 59010}, {ip: "187.33.79.61", port: 33469}, {ip: "187.35.158.150", port: 38872}, {ip: "187.44.1.167", port: 8080}, {ip: "187.45.127.87", port: 20183}, {ip: "187.45.156.109", port: 8080}, {ip: "187.5.218.215", port: 20183}, {ip: "187.58.65.225", port: 3128}, {ip: "187.63.111.37", port: 3128}, {ip: "187.72.166.10", port: 8080}, {ip: "187.73.68.14", port: 53281}, {ip: "187.84.177.6", port: 45903}, {ip: "187.84.191.170", port: 43936}, {ip: "187.87.204.210", port: 45597}, {ip: "187.87.39.247", port: 31793}, {ip: "189.1.16.162", port: 23500}, {ip: "189.113.124.162", port: 8080}, {ip: "189.124.195.185", port: 37318}, {ip: "189.3.196.18", port: 61595}, {ip: "189.37.33.59", port: 35532}, {ip: "189.7.49.66", port: 42700}, {ip: "189.90.194.35", port: 30843}, {ip: "189.90.248.75", port: 8080}, {ip: "189.91.231.43", port: 3128}, {ip: "191.239.243.156", port: 3128}, {ip: "191.240.154.246", port: 23500}, {ip: "191.240.156.154", port: 36127}, {ip: "191.240.99.142", port: 9090}, {ip: "191.241.226.230", port: 53281}, {ip: "191.241.228.74", port: 20183}, {ip: "191.241.228.78", port: 20183}, {ip: "191.241.33.238", port: 39188}, {ip: "191.241.36.170", port: 8080}, {ip: "191.241.36.218", port: 3128}, {ip: "191.242.182.132", port: 8081}, {ip: "191.243.221.130", port: 3128}, {ip: "191.255.207.231", port: 20183}, {ip: "191.36.192.196", port: 3128}, {ip: "191.36.244.230", port: 51377}, {ip: "191.5.0.79", port: 53281}, {ip: "191.6.228.6", port: 53281}, {ip: "191.7.193.18", port: 38133}, {ip: "191.7.20.134", port: 3128}, {ip: "192.140.91.173", port: 20183}, {ip: "200.150.86.138", port: 44677}, {ip: "200.155.36.185", port: 3128}, {ip: "200.155.36.188", port: 3128}, {ip: "200.155.39.41", port: 3128}, {ip: "200.174.158.26", port: 34112}, {ip: "200.187.177.105", port: 20183}, {ip: "200.187.87.138", port: 20183}, {ip: "200.192.252.201", port: 8080}, {ip: "200.192.255.102", port: 8080}, {ip: "200.203.144.2", port: 50262}, {ip: "200.229.238.42", port: 20183}, {ip: "200.233.134.85", port: 43172}, {ip: "200.233.136.177", port: 20183}, {ip: "200.241.44.3", port: 20183}, {ip: "200.255.122.170", port: 8080}, {ip: "200.255.122.174", port: 8080}, {ip: "201.12.21.57", port: 8080}, {ip: "201.131.224.21", port: 56200}, {ip: "201.182.223.16", port: 37492}, {ip: "201.20.89.126", port: 8080}, {ip: "201.22.95.10", port: 8080}, {ip: "201.57.167.34", port: 8080}, {ip: "201.59.200.246", port: 80}, {ip: "201.6.167.178", port: 3128}, {ip: "201.90.36.194", port: 3128}, {ip: "45.226.20.6", port: 8080}, {ip: "45.234.139.129", port: 20183}, {ip: "45.234.200.18", port: 53281}, {ip: "45.235.87.4", port: 51996}, {ip: "45.6.136.38", port: 53281}, {ip: "45.6.80.131", port: 52080}, {ip: "45.6.93.10", port: 8080}, {ip: "45.71.108.162", port: 53281}], - "PK" => [{ip: "103.18.243.154", port: 8080}, {ip: "110.36.218.126", port: 36651}, {ip: "110.36.234.210", port: 8080}, {ip: "110.39.162.74", port: 53281}, {ip: "110.39.174.58", port: 8080}, {ip: "111.68.108.34", port: 8080}, {ip: "125.209.116.182", port: 31653}, {ip: "125.209.78.21", port: 8080}, {ip: "125.209.82.78", port: 35087}, {ip: "180.92.156.150", port: 8080}, {ip: "202.142.158.114", port: 8080}, {ip: "202.147.173.10", port: 8080}, {ip: "202.147.173.10", port: 80}, {ip: "202.69.38.82", port: 8080}, {ip: "203.128.16.126", port: 59538}, {ip: "203.128.16.154", port: 33002}, {ip: "27.255.4.170", port: 8080}], - "ID" => [{ip: "101.128.68.113", port: 8080}, {ip: "101.255.116.113", port: 53281}, {ip: "101.255.120.170", port: 6969}, {ip: "101.255.121.74", port: 8080}, {ip: "101.255.124.242", port: 8080}, {ip: "101.255.124.242", port: 80}, {ip: "101.255.56.138", port: 53560}, {ip: "103.10.171.132", port: 41043}, {ip: "103.10.81.172", port: 80}, {ip: "103.108.158.3", port: 48196}, {ip: "103.111.219.159", port: 53281}, {ip: "103.111.54.26", port: 49781}, {ip: "103.111.54.74", port: 8080}, {ip: "103.19.110.177", port: 8080}, {ip: "103.2.146.66", port: 49089}, {ip: "103.206.168.177", port: 53281}, {ip: "103.206.253.58", port: 49573}, {ip: "103.21.92.254", port: 33929}, {ip: "103.226.49.83", port: 23500}, {ip: "103.227.147.142", port: 37581}, {ip: "103.23.101.58", port: 8080}, {ip: "103.24.107.2", port: 8181}, {ip: "103.245.19.222", port: 53281}, {ip: "103.247.122.38", port: 8080}, {ip: "103.247.218.166", port: 3128}, {ip: "103.248.219.26", port: 53634}, {ip: "103.253.2.165", port: 33543}, {ip: "103.253.2.168", port: 51229}, {ip: "103.253.2.174", port: 30827}, {ip: "103.28.114.134", port: 8080}, {ip: "103.28.220.73", port: 53281}, {ip: "103.30.246.47", port: 3128}, {ip: "103.31.45.169", port: 57655}, {ip: "103.41.122.14", port: 53281}, {ip: "103.75.101.97", port: 8080}, {ip: "103.76.17.151", port: 23500}, {ip: "103.76.50.181", port: 8080}, {ip: "103.76.50.181", port: 80}, {ip: "103.76.50.182", port: 8080}, {ip: "103.78.74.170", port: 3128}, {ip: "103.78.80.194", port: 33442}, {ip: "103.8.122.5", port: 53297}, {ip: "103.80.236.107", port: 53281}, {ip: "103.80.238.203", port: 53281}, {ip: "103.86.140.74", port: 59538}, {ip: "103.94.122.254", port: 8080}, {ip: "103.94.125.244", port: 41508}, {ip: "103.94.169.19", port: 8080}, {ip: "103.94.7.254", port: 53281}, {ip: "106.0.51.50", port: 17385}, {ip: "110.93.13.202", port: 34881}, {ip: "112.78.37.6", port: 54791}, {ip: "114.199.110.58", port: 55898}, {ip: "114.199.112.170", port: 23500}, {ip: "114.199.123.194", port: 8080}, {ip: "114.57.33.162", port: 46935}, {ip: "114.57.33.214", port: 8080}, {ip: "114.6.197.254", port: 8080}, {ip: "114.7.15.146", port: 8080}, {ip: "114.7.162.254", port: 53281}, {ip: "115.124.75.226", port: 53990}, {ip: "115.124.75.228", port: 3128}, {ip: "117.102.78.42", port: 8080}, {ip: "117.102.93.251", port: 8080}, {ip: "117.102.94.186", port: 8080}, {ip: "117.102.94.186", port: 80}, {ip: "117.103.2.249", port: 58276}, {ip: "117.54.13.174", port: 34190}, {ip: "117.74.124.129", port: 8088}, {ip: "118.97.100.83", port: 35220}, {ip: "118.97.191.162", port: 80}, {ip: "118.97.191.203", port: 8080}, {ip: "118.97.36.18", port: 8080}, {ip: "118.97.73.85", port: 53281}, {ip: "118.99.105.226", port: 8080}, {ip: "119.252.168.53", port: 53281}, {ip: "122.248.45.35", port: 53281}, {ip: "122.50.6.186", port: 8080}, {ip: "122.50.6.186", port: 80}, {ip: "123.231.226.114", port: 47562}, {ip: "123.255.202.83", port: 32523}, {ip: "124.158.164.195", port: 8080}, {ip: "124.81.99.30", port: 3128}, {ip: "137.59.162.10", port: 3128}, {ip: "139.0.29.20", port: 59532}, {ip: "139.255.123.194", port: 4550}, {ip: "139.255.16.171", port: 31773}, {ip: "139.255.17.2", port: 47421}, {ip: "139.255.19.162", port: 42371}, {ip: "139.255.7.81", port: 53281}, {ip: "139.255.91.115", port: 8080}, {ip: "139.255.92.26", port: 53281}, {ip: "158.140.181.140", port: 54041}, {ip: "160.202.40.20", port: 55655}, {ip: "175.103.42.147", port: 8080}, {ip: "180.178.98.198", port: 8080}, {ip: "180.250.101.146", port: 8080}, {ip: "182.23.107.212", port: 3128}, {ip: "182.23.2.101", port: 49833}, {ip: "182.23.7.226", port: 8080}, {ip: "182.253.209.203", port: 3128}, {ip: "183.91.66.210", port: 80}, {ip: "202.137.10.179", port: 57338}, {ip: "202.137.25.53", port: 3128}, {ip: "202.137.25.8", port: 8080}, {ip: "202.138.242.76", port: 4550}, {ip: "202.138.249.202", port: 43108}, {ip: "202.148.2.254", port: 8000}, {ip: "202.162.201.94", port: 53281}, {ip: "202.165.47.26", port: 8080}, {ip: "202.43.167.130", port: 8080}, {ip: "202.51.126.10", port: 53281}, {ip: "202.59.171.164", port: 58567}, {ip: "202.93.128.98", port: 3128}, {ip: "203.142.72.114", port: 808}, {ip: "203.153.117.65", port: 54144}, {ip: "203.189.89.1", port: 53281}, {ip: "203.77.239.18", port: 37002}, {ip: "203.99.123.25", port: 61502}, {ip: "220.247.168.163", port: 53281}, {ip: "220.247.173.154", port: 53281}, {ip: "220.247.174.206", port: 53445}, {ip: "222.124.131.211", port: 47343}, {ip: "222.124.173.146", port: 53281}, {ip: "222.124.2.131", port: 8080}, {ip: "222.124.2.186", port: 8080}, {ip: "222.124.215.187", port: 38913}, {ip: "222.124.221.179", port: 53281}, {ip: "223.25.101.242", port: 59504}, {ip: "223.25.97.62", port: 8080}, {ip: "223.25.99.38", port: 80}, {ip: "27.111.44.202", port: 80}, {ip: "27.111.47.3", port: 51144}, {ip: "36.37.124.234", port: 36179}, {ip: "36.37.124.235", port: 36179}, {ip: "36.37.81.135", port: 8080}, {ip: "36.37.89.98", port: 32323}, {ip: "36.66.217.179", port: 8080}, {ip: "36.66.98.6", port: 53281}, {ip: "36.67.143.183", port: 48746}, {ip: "36.67.206.187", port: 8080}, {ip: "36.67.32.87", port: 8080}, {ip: "36.67.93.220", port: 3128}, {ip: "36.67.93.220", port: 80}, {ip: "36.89.10.51", port: 34115}, {ip: "36.89.119.149", port: 8080}, {ip: "36.89.157.23", port: 37728}, {ip: "36.89.181.155", port: 60165}, {ip: "36.89.188.11", port: 39507}, {ip: "36.89.194.113", port: 37811}, {ip: "36.89.226.254", port: 8081}, {ip: "36.89.232.138", port: 23500}, {ip: "36.89.39.10", port: 3128}, {ip: "36.89.65.253", port: 60997}, {ip: "43.243.141.114", port: 8080}, {ip: "43.245.184.202", port: 41102}, {ip: "43.245.184.238", port: 80}, {ip: "66.96.233.225", port: 35053}, {ip: "66.96.237.253", port: 8080}], - "BD" => [{ip: "103.103.88.91", port: 8080}, {ip: "103.106.119.154", port: 8080}, {ip: "103.106.236.1", port: 8080}, {ip: "103.106.236.41", port: 8080}, {ip: "103.108.144.139", port: 53281}, {ip: "103.109.57.218", port: 8080}, {ip: "103.109.58.242", port: 8080}, {ip: "103.112.129.106", port: 31094}, {ip: "103.112.129.82", port: 53281}, {ip: "103.114.10.177", port: 8080}, {ip: "103.114.10.250", port: 8080}, {ip: "103.15.245.26", port: 8080}, {ip: "103.195.204.73", port: 21776}, {ip: "103.197.49.106", port: 49688}, {ip: "103.198.168.29", port: 21776}, {ip: "103.214.200.6", port: 59008}, {ip: "103.218.25.161", port: 8080}, {ip: "103.218.25.41", port: 8080}, {ip: "103.218.26.204", port: 8080}, {ip: "103.218.27.221", port: 8080}, {ip: "103.231.229.90", port: 53281}, {ip: "103.239.252.233", port: 8080}, {ip: "103.239.252.50", port: 8080}, {ip: "103.239.253.193", port: 8080}, {ip: "103.250.68.193", port: 51370}, {ip: "103.5.232.146", port: 8080}, {ip: "103.73.224.53", port: 23500}, {ip: "103.9.134.73", port: 65301}, {ip: "113.11.47.242", port: 40071}, {ip: "113.11.5.67", port: 40071}, {ip: "114.31.5.34", port: 52606}, {ip: "115.127.51.226", port: 42764}, {ip: "115.127.64.62", port: 39611}, {ip: "115.127.91.106", port: 8080}, {ip: "119.40.85.198", port: 36899}, {ip: "123.200.29.110", port: 23500}, {ip: "123.49.51.42", port: 55124}, {ip: "163.47.36.90", port: 3128}, {ip: "180.211.134.158", port: 23500}, {ip: "180.211.193.74", port: 40536}, {ip: "180.92.238.226", port: 53451}, {ip: "182.160.104.213", port: 8080}, {ip: "202.191.126.58", port: 23500}, {ip: "202.4.126.170", port: 8080}, {ip: "202.5.37.241", port: 33623}, {ip: "202.5.57.5", port: 61729}, {ip: "202.79.17.65", port: 60122}, {ip: "203.188.248.52", port: 23500}, {ip: "27.147.146.78", port: 52220}, {ip: "27.147.164.10", port: 52344}, {ip: "27.147.212.38", port: 53281}, {ip: "27.147.217.154", port: 43252}, {ip: "27.147.219.102", port: 49464}, {ip: "43.239.74.137", port: 8080}, {ip: "43.240.103.252", port: 8080}, {ip: "45.125.223.57", port: 8080}, {ip: "45.125.223.81", port: 8080}, {ip: "45.251.228.122", port: 41418}, {ip: "45.64.132.137", port: 8080}, {ip: "45.64.132.137", port: 80}, {ip: "61.247.186.137", port: 8080}], - "MX" => [{ip: "148.217.94.54", port: 3128}, {ip: "177.244.28.77", port: 53281}, {ip: "187.141.73.147", port: 53281}, {ip: "187.185.15.35", port: 53281}, {ip: "187.188.46.172", port: 53455}, {ip: "187.216.83.185", port: 8080}, {ip: "187.216.90.46", port: 53281}, {ip: "187.243.253.182", port: 33796}, {ip: "189.195.132.86", port: 43286}, {ip: "189.204.158.161", port: 8080}, {ip: "200.79.180.115", port: 8080}, {ip: "201.140.113.90", port: 37193}, {ip: "201.144.14.229", port: 53281}, {ip: "201.163.73.93", port: 53281}], - "PH" => [{ip: "103.86.187.242", port: 23500}, {ip: "122.54.101.69", port: 8080}, {ip: "122.54.65.150", port: 8080}, {ip: "125.5.20.134", port: 53281}, {ip: "146.88.77.51", port: 8080}, {ip: "182.18.200.92", port: 8080}, {ip: "219.90.87.91", port: 53281}, {ip: "58.69.12.210", port: 8080}], - "EG" => [{ip: "41.65.0.167", port: 8080}], - "VN" => [{ip: "1.55.240.156", port: 53281}, {ip: "101.99.23.136", port: 3128}, {ip: "103.15.51.160", port: 8080}, {ip: "113.161.128.169", port: 60427}, {ip: "113.161.161.143", port: 57967}, {ip: "113.161.173.10", port: 3128}, {ip: "113.161.35.108", port: 30028}, {ip: "113.164.79.177", port: 46281}, {ip: "113.190.235.50", port: 34619}, {ip: "115.78.160.247", port: 8080}, {ip: "117.2.155.29", port: 47228}, {ip: "117.2.17.26", port: 53281}, {ip: "117.2.22.41", port: 41973}, {ip: "117.4.145.16", port: 51487}, {ip: "118.69.219.185", port: 55184}, {ip: "118.69.61.212", port: 53281}, {ip: "118.70.116.227", port: 61651}, {ip: "118.70.219.124", port: 53281}, {ip: "221.121.12.238", port: 36077}, {ip: "27.2.7.59", port: 52148}], - "CD" => [{ip: "41.79.233.45", port: 8080}], - "TR" => [{ip: "151.80.65.175", port: 3128}, {ip: "176.235.186.242", port: 37043}, {ip: "178.250.92.18", port: 8080}, {ip: "185.203.170.92", port: 8080}, {ip: "185.203.170.94", port: 8080}, {ip: "185.203.170.95", port: 8080}, {ip: "185.51.36.152", port: 41258}, {ip: "195.137.223.50", port: 41336}, {ip: "195.155.98.70", port: 52598}, {ip: "212.156.146.22", port: 40080}, {ip: "213.14.31.122", port: 44621}, {ip: "31.145.137.139", port: 31871}, {ip: "31.145.138.129", port: 31871}, {ip: "31.145.138.146", port: 34159}, {ip: "31.145.187.172", port: 30636}, {ip: "78.188.4.124", port: 34514}, {ip: "88.248.23.216", port: 36426}, {ip: "93.182.72.36", port: 8080}, {ip: "95.0.194.241", port: 9090}], -} diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index a5e621f2..c8b037c8 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -5,19 +5,21 @@ module YoutubeAPI extend self - private DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" - - private ANDROID_APP_VERSION = "18.20.38" - # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1308 - private ANDROID_USER_AGENT = "com.google.android.youtube/18.20.38 (Linux; U; Android 12; US) gzip" + # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history + private ANDROID_APP_VERSION = "19.14.42" + private ANDROID_USER_AGENT = "com.google.android.youtube/19.14.42 (Linux; U; Android 12; US) gzip" private ANDROID_SDK_VERSION = 31_i64 private ANDROID_VERSION = "12" - private IOS_APP_VERSION = "18.21.3" - # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1330 - private IOS_USER_AGENT = "com.google.ios.youtube/18.21.3 (iPhone14,5; U; CPU iOS 15_6 like Mac OS X;)" - # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1224 - private IOS_VERSION = "15.6.0.19G71" + private ANDROID_TS_APP_VERSION = "1.9" + private ANDROID_TS_USER_AGENT = "com.google.android.youtube/1.9 (Linux; U; Android 12; US) gzip" + + # For Apple device names, see https://gist.github.com/adamawolf/3048717 + # For iOS versions, see https://en.wikipedia.org/wiki/IOS_version_history#Releases, + # then go to the dedicated article of the major version you want. + private IOS_APP_VERSION = "19.16.3" + private IOS_USER_AGENT = "com.google.ios.youtube/19.16.3 (iPhone14,5; U; CPU iOS 17_4 like Mac OS X;)" + private IOS_VERSION = "17.4.0.21E219" # Major.Minor.Patch.Build private WINDOWS_VERSION = "10.0" @@ -31,6 +33,7 @@ module YoutubeAPI Android AndroidEmbeddedPlayer AndroidScreenEmbed + AndroidTestSuite IOS IOSEmbedded @@ -45,8 +48,7 @@ module YoutubeAPI ClientType::Web => { name: "WEB", name_proto: "1", - version: "2.20230602.01.00", - api_key: DEFAULT_API_KEY, + version: "2.20240304.00.00", screen: "WATCH_FULL_SCREEN", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -55,8 +57,7 @@ module YoutubeAPI ClientType::WebEmbeddedPlayer => { name: "WEB_EMBEDDED_PLAYER", name_proto: "56", - version: "1.20220803.01.00", - api_key: DEFAULT_API_KEY, + version: "1.20240303.00.00", screen: "EMBED", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -65,8 +66,7 @@ module YoutubeAPI ClientType::WebMobile => { name: "MWEB", name_proto: "2", - version: "2.20230531.05.00", - api_key: DEFAULT_API_KEY, + version: "2.20240304.08.00", os_name: "Android", os_version: ANDROID_VERSION, platform: "MOBILE", @@ -74,8 +74,7 @@ module YoutubeAPI ClientType::WebScreenEmbed => { name: "WEB", name_proto: "1", - version: "2.20220804.00.00", - api_key: DEFAULT_API_KEY, + version: "2.20240304.00.00", screen: "EMBED", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -88,7 +87,6 @@ module YoutubeAPI name: "ANDROID", name_proto: "3", version: ANDROID_APP_VERSION, - api_key: "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w", android_sdk_version: ANDROID_SDK_VERSION, user_agent: ANDROID_USER_AGENT, os_name: "Android", @@ -99,13 +97,11 @@ module YoutubeAPI name: "ANDROID_EMBEDDED_PLAYER", name_proto: "55", version: ANDROID_APP_VERSION, - api_key: DEFAULT_API_KEY, }, ClientType::AndroidScreenEmbed => { name: "ANDROID", name_proto: "3", version: ANDROID_APP_VERSION, - api_key: DEFAULT_API_KEY, screen: "EMBED", android_sdk_version: ANDROID_SDK_VERSION, user_agent: ANDROID_USER_AGENT, @@ -113,6 +109,16 @@ module YoutubeAPI os_version: ANDROID_VERSION, platform: "MOBILE", }, + ClientType::AndroidTestSuite => { + name: "ANDROID_TESTSUITE", + name_proto: "30", + version: ANDROID_TS_APP_VERSION, + android_sdk_version: ANDROID_SDK_VERSION, + user_agent: ANDROID_TS_USER_AGENT, + os_name: "Android", + os_version: ANDROID_VERSION, + platform: "MOBILE", + }, # IOS @@ -120,7 +126,6 @@ module YoutubeAPI name: "IOS", name_proto: "5", version: IOS_APP_VERSION, - api_key: "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc", user_agent: IOS_USER_AGENT, device_make: "Apple", device_model: "iPhone14,5", @@ -132,7 +137,6 @@ module YoutubeAPI name: "IOS_MESSAGES_EXTENSION", name_proto: "66", version: IOS_APP_VERSION, - api_key: DEFAULT_API_KEY, user_agent: IOS_USER_AGENT, device_make: "Apple", device_model: "iPhone14,5", @@ -143,9 +147,8 @@ module YoutubeAPI ClientType::IOSMusic => { name: "IOS_MUSIC", name_proto: "26", - version: "5.21", - api_key: "AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s", - user_agent: "com.google.ios.youtubemusic/5.21 (iPhone14,5; U; CPU iOS 15_6 like Mac OS X;)", + version: "6.42", + user_agent: "com.google.ios.youtubemusic/6.42 (iPhone14,5; U; CPU iOS 17_4 like Mac OS X;)", device_make: "Apple", device_model: "iPhone14,5", os_name: "iPhone", @@ -158,14 +161,12 @@ module YoutubeAPI ClientType::TvHtml5 => { name: "TVHTML5", name_proto: "7", - version: "7.20220325", - api_key: DEFAULT_API_KEY, + version: "7.20240304.10.00", }, ClientType::TvHtml5ScreenEmbed => { name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER", name_proto: "85", version: "2.0", - api_key: DEFAULT_API_KEY, screen: "EMBED", }, } @@ -187,10 +188,6 @@ module YoutubeAPI # conf_2 = ClientConfig.new(client_type: ClientType::Android) # YoutubeAPI::player(video_id: "dQw4w9WgXcQ", client_config: conf_2) # - # # Proxy request through russian proxies - # conf_3 = ClientConfig.new(proxy_region: "RU") - # YoutubeAPI::next({video_id: "dQw4w9WgXcQ"}, client_config: conf_3) - # ``` # struct ClientConfig # Type of client to emulate. @@ -201,16 +198,11 @@ module YoutubeAPI # (this is passed as the `gl` parameter). property region : String | Nil - # ISO code of country where the proxy is located. - # Used in case of geo-restricted videos. - property proxy_region : String | Nil - # Initialization function def initialize( *, @client_type = ClientType::Web, - @region = "US", - @proxy_region = nil + @region = "US" ) end @@ -230,11 +222,6 @@ module YoutubeAPI end # :ditto: - def api_key : String - HARDCODED_CLIENTS[@client_type][:api_key] - end - - # :ditto: def screen : String HARDCODED_CLIENTS[@client_type][:screen]? || "" end @@ -270,9 +257,8 @@ module YoutubeAPI # Convert to string, for logging purposes def to_s return { - client_type: self.name, - region: @region, - proxy_region: @proxy_region, + client_type: self.name, + region: @region, }.to_s end end @@ -599,7 +585,7 @@ module YoutubeAPI client_config ||= DEFAULT_CLIENT_CONFIG # Query parameters - url = "#{endpoint}?key=#{client_config.api_key}&prettyPrint=false" + url = "#{endpoint}?prettyPrint=false" headers = HTTP::Headers{ "Content-Type" => "application/json; charset=UTF-8", @@ -619,7 +605,7 @@ module YoutubeAPI LOGGER.trace("YoutubeAPI: POST data: #{data}") # Send the POST request - body = YT_POOL.client(client_config.proxy_region) do |client| + body = YT_POOL.client() do |client| client.post(url, headers: headers, body: data.to_json) do |response| self._decompress(response.body_io, response.headers["Content-Encoding"]?) end |