diff options
Diffstat (limited to 'src')
33 files changed, 276 insertions, 578 deletions
diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index c3d6124f..be739673 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -93,7 +93,7 @@ struct ChannelVideo def to_tuple {% begin %} { - {{*@type.instance_vars.map(&.name)}} + {{@type.instance_vars.map(&.name).splat}} } {% end %} end diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index beb86e08..351790d7 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -62,12 +62,6 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so return continuation end -# Used in bypass_captcha_job.cr -def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2) - return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" -end - module Invidious::Channel::Tabs extend self diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 429d9246..09c2168b 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -48,7 +48,7 @@ struct ConfigPreferences def to_tuple {% begin %} { - {{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}} + {{(@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }).splat}} } {% end %} end @@ -133,10 +133,6 @@ class Config # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] property cookies : HTTP::Cookies = HTTP::Cookies.new - # Key for Anti-Captcha - property captcha_key : String? = nil - # API URL for Anti-Captcha - property captcha_api_url : String = "https://api.anti-captcha.com" # Playlist length limit property playlist_length_limit : Int32 = 500 diff --git a/src/invidious/database/statistics.cr b/src/invidious/database/statistics.cr index 1df549e2..9e4963fd 100644 --- a/src/invidious/database/statistics.cr +++ b/src/invidious/database/statistics.cr @@ -15,7 +15,7 @@ module Invidious::Database::Statistics PG_DB.query_one(request, as: Int64) end - def count_users_active_1m : Int64 + def count_users_active_6m : Int64 request = <<-SQL SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '6 months' @@ -24,7 +24,7 @@ module Invidious::Database::Statistics PG_DB.query_one(request, as: Int64) end - def count_users_active_6m : Int64 + def count_users_active_1m : Int64 request = <<-SQL SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '1 month' diff --git a/src/invidious/frontend/comments_reddit.cr b/src/invidious/frontend/comments_reddit.cr index b5647bae..4dda683e 100644 --- a/src/invidious/frontend/comments_reddit.cr +++ b/src/invidious/frontend/comments_reddit.cr @@ -33,7 +33,7 @@ module Invidious::Frontend::Comments <a href="javascript:void(0)" data-onclick="toggle_parent">[ − ]</a> <b><a href="https://www.reddit.com/user/#{child.author}">#{child.author}</a></b> #{translate_count(locale, "comments_points_count", child.score, NumberFormatting::Separator)} - <span title="#{child.created_utc.to_s(translate(locale, "%a %B %-d %T %Y UTC"))}">#{translate(locale, "`x` ago", recode_date(child.created_utc, locale))}</span> + <span title="#{child.created_utc.to_s("%a %B %-d %T %Y UTC")}">#{translate(locale, "`x` ago", recode_date(child.created_utc, locale))}</span> <a href="https://www.reddit.com#{child.permalink}" title="#{translate(locale, "permalink")}">#{translate(locale, "permalink")}</a> </p> <div> diff --git a/src/invidious/frontend/comments_youtube.cr b/src/invidious/frontend/comments_youtube.cr index ecc0bc1b..aecac87f 100644 --- a/src/invidious/frontend/comments_youtube.cr +++ b/src/invidious/frontend/comments_youtube.cr @@ -107,6 +107,36 @@ module Invidious::Frontend::Comments </div> END_HTML end + when "multiImage" + html << <<-END_HTML + <section class="carousel"> + <a class="skip-link" href="#skip-#{child["commentId"]}">#{translate(locale, "carousel_skip")}</a> + <div class="slides"> + END_HTML + image_array = attachment["images"].as_a + + image_array.each_index do |i| + html << <<-END_HTML + <div class="slides-item slide-#{i + 1}" id="#{child["commentId"]}-slide-#{i + 1}" aria-label="#{translate(locale, "carousel_slide", {"current" => (i + 1).to_s, "total" => image_array.size.to_s})}" tabindex="0"> + <img loading="lazy" src="/ggpht#{URI.parse(image_array[i][1]["url"].as_s).request_target}" alt="" /> + </div> + END_HTML + end + + html << <<-END_HTML + </div> + <div class="carousel__nav"> + END_HTML + attachment["images"].as_a.each_index do |i| + html << <<-END_HTML + <a class="slider-nav" href="##{child["commentId"]}-slide-#{i + 1}" aria-label="#{translate(locale, "carousel_go_to", (i + 1).to_s)}" tabindex="-1" aria-hidden="true">#{i + 1}</a> + END_HTML + end + html << <<-END_HTML + </div> + <div id="skip-#{child["commentId"]}"></div> + </section> + END_HTML else nil # Ignore end end diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr index 6e5a975d..21b789bc 100644 --- a/src/invidious/helpers/errors.cr +++ b/src/invidious/helpers/errors.cr @@ -3,7 +3,7 @@ # ------------------- macro error_template(*args) - error_template_helper(env, {{*args}}) + error_template_helper(env, {{args.splat}}) end def github_details(summary : String, content : String) @@ -95,7 +95,7 @@ end # ------------------- macro error_atom(*args) - error_atom_helper(env, {{*args}}) + error_atom_helper(env, {{args.splat}}) end def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception) @@ -121,7 +121,7 @@ end # ------------------- macro error_json(*args) - error_json_helper(env, {{*args}}) + error_json_helper(env, {{args.splat}}) end def error_json_helper( diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index d140a858..174f620d 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -142,63 +142,8 @@ class APIHandler < Kemal::Handler exclude ["/api/v1/auth/notifications"], "POST" def call(env) - return call_next env unless only_match? env - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - # Since /api/v1/notifications is an event-stream, we don't want - # to wrap the response - return call_next env if exclude_match? env - - # Here we swap out the socket IO so we can modify the response as needed - output = env.response.output - env.response.output = IO::Memory.new - - begin - call_next env - - env.response.output.rewind - - if env.response.output.as(IO::Memory).size != 0 && - env.response.headers.includes_word?("Content-Type", "application/json") - response = JSON.parse(env.response.output) - - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - - if env.params.query["pretty"]?.try &.== "1" - response = response.to_pretty_json - else - response = response.to_json - end - else - response = env.response.output.gets_to_end - end - rescue ex - env.response.content_type = "application/json" if env.response.headers.includes_word?("Content-Type", "text/html") - env.response.status_code = 500 - - if env.response.headers.includes_word?("Content-Type", "application/json") - response = {"error" => ex.message || "Unspecified error"} - - if env.params.query["pretty"]?.try &.== "1" - response = response.to_pretty_json - else - response = response.to_json - end - end - ensure - env.response.output = output - env.response.print response - - env.response.flush - end + env.response.headers["Access-Control-Allow-Origin"] = "*" if only_match?(env) + call_next env end end diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 6dc9860e..6add0237 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -78,15 +78,6 @@ def create_notification_stream(env, topics, connection_channel) video.published = published response = JSON.parse(video.to_json(locale, nil)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -113,15 +104,6 @@ def create_notification_stream(env, topics, connection_channel) Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video| response = JSON.parse(video.to_json(locale)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -155,15 +137,6 @@ def create_notification_stream(env, topics, connection_channel) video.published = Time.unix(published) response = JSON.parse(video.to_json(locale, nil)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index 76e477a4..23a1aafc 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -78,7 +78,7 @@ def load_all_locales return locales end -def translate(locale : String?, key : String, text : String | Nil = nil) : String +def translate(locale : String?, key : String, text : String | Hash(String, String) | Nil = nil) : String # Log a warning if "key" doesn't exist in en-US locale and return # that key as the text, so this is more or less transparent to the user. if !LOCALES["en-US"].has_key?(key) @@ -101,10 +101,12 @@ def translate(locale : String?, key : String, text : String | Nil = nil) : Strin match_length = 0 raw_data.as_h.each do |hash_key, value| - if md = text.try &.match(/#{hash_key}/) - if md[0].size >= match_length - translation = value.as_s - match_length = md[0].size + if text.is_a?(String) + if md = text.try &.match(/#{hash_key}/) + if md[0].size >= match_length + translation = value.as_s + match_length = md[0].size + end end end end @@ -114,8 +116,13 @@ def translate(locale : String?, key : String, text : String | Nil = nil) : Strin raise "Invalid translation \"#{raw_data}\"" end - if text + if text.is_a?(String) translation = translation.gsub("`x`", text) + elsif text.is_a?(Hash(String, String)) + # adds support for multi string interpolation. Based on i18next https://www.i18next.com/translation-function/interpolation#basic + text.each_key do |hash_key| + translation = translation.gsub("{{#{hash_key}}}", text[hash_key]) + end end return translation diff --git a/src/invidious/helpers/i18next.cr b/src/invidious/helpers/i18next.cr index 252af6b9..9f4077e1 100644 --- a/src/invidious/helpers/i18next.cr +++ b/src/invidious/helpers/i18next.cr @@ -47,19 +47,19 @@ module I18next::Plurals private PLURAL_SETS = { PluralForms::Single_gt_one => [ - "ach", "ak", "am", "arn", "br", "fil", "gun", "ln", "mfe", "mg", - "mi", "oc", "pt", "tg", "tl", "ti", "tr", "uz", "wa", + "ach", "ak", "am", "arn", "br", "fa", "fil", "gun", "ln", "mfe", "mg", + "mi", "oc", "pt-PT", "tg", "tl", "ti", "tr", "uz", "wa", ], PluralForms::Single_not_one => [ "af", "an", "ast", "az", "bg", "bn", "ca", "da", "de", "dev", "el", "en", - "eo", "es", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi", + "eo", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi", "hu", "hy", "ia", "kk", "kn", "ku", "lb", "mai", "ml", "mn", "mr", "nah", "nap", "nb", "ne", "nl", "nn", "no", "nso", "pa", "pap", "pms", "ps", "rm", "sco", "se", "si", "so", "son", "sq", "sv", "sw", "ta", "te", "tk", "ur", "yo", ], PluralForms::None => [ - "ay", "bo", "cgg", "fa", "ht", "id", "ja", "jbo", "ka", "km", "ko", "ky", + "ay", "bo", "cgg", "ht", "id", "ja", "jbo", "ka", "km", "ko", "ky", "lo", "ms", "sah", "su", "th", "tt", "ug", "vi", "wo", "zh", ], PluralForms::Dual_Slavic => [ @@ -90,11 +90,13 @@ module I18next::Plurals "sk" => PluralForms::Special_Czech_Slovak, "sl" => PluralForms::Special_Slovenian, # Mixed v3/v4 rules - "fr" => PluralForms::Special_French_Portuguese, - "hr" => PluralForms::Special_Hungarian_Serbian, - "it" => PluralForms::Special_Spanish_Italian, - "pt-BR" => PluralForms::Special_French_Portuguese, - "sr" => PluralForms::Special_Hungarian_Serbian, + "es" => PluralForms::Special_Spanish_Italian, + "fr" => PluralForms::Special_French_Portuguese, + "hr" => PluralForms::Special_Hungarian_Serbian, + "it" => PluralForms::Special_Spanish_Italian, + "pt" => PluralForms::Special_French_Portuguese, + "pt" => PluralForms::Special_French_Portuguese, + "sr" => PluralForms::Special_Hungarian_Serbian, } # These are the v1 and v2 compatible suffixes. @@ -165,7 +167,7 @@ module I18next::Plurals def get_plural_form(locale : String) : PluralForms # Extract the ISO 639-1 or 639-2 code from an RFC 5646 language code - if !locale.matches?(/^pt-BR$/) + if !locale.matches?(/^pt-PT$/) locale = locale.split('-')[0] end diff --git a/src/invidious/helpers/json_filter.cr b/src/invidious/helpers/json_filter.cr deleted file mode 100644 index 3f4080ba..00000000 --- a/src/invidious/helpers/json_filter.cr +++ /dev/null @@ -1,248 +0,0 @@ -module JSONFilter - alias BracketIndex = Hash(Int64, Int64) - - alias GroupedFieldsValue = String | Array(GroupedFieldsValue) - alias GroupedFieldsList = Array(GroupedFieldsValue) - - class FieldsParser - class ParseError < Exception - end - - # Returns the `Regex` pattern used to match nest groups - def self.nest_group_pattern : Regex - # uses a '.' character to match json keys as they are allowed - # to contain any unicode codepoint - /(?:|,)(?<groupname>[^,\n]*?)\(/ - end - - # Returns the `Regex` pattern used to check if there are any empty nest groups - def self.unnamed_nest_group_pattern : Regex - /^\(|\(\(|\/\(/ - end - - def self.parse_fields(fields_text : String, &) : Nil - if fields_text.empty? - raise FieldsParser::ParseError.new "Fields is empty" - end - - opening_bracket_count = fields_text.count('(') - closing_bracket_count = fields_text.count(')') - - if opening_bracket_count != closing_bracket_count - bracket_type = opening_bracket_count > closing_bracket_count ? "opening" : "closing" - raise FieldsParser::ParseError.new "There are too many #{bracket_type} brackets (#{opening_bracket_count}:#{closing_bracket_count})" - elsif match_result = unnamed_nest_group_pattern.match(fields_text) - raise FieldsParser::ParseError.new "Unnamed nest group at position #{match_result.begin}" - end - - # first, handle top-level single nested properties: items/id, playlistItems/snippet, etc - parse_single_nests(fields_text) { |nest_list| yield nest_list } - - # next, handle nest groups: items(id, etag, etc) - parse_nest_groups(fields_text) { |nest_list| yield nest_list } - end - - def self.parse_single_nests(fields_text : String, &) : Nil - single_nests = remove_nest_groups(fields_text) - - if !single_nests.empty? - property_nests = single_nests.split(',') - - property_nests.each do |nest| - nest_list = nest.split('/') - if nest_list.includes? "" - raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list}" - end - yield nest_list - end - # else - # raise FieldsParser::ParseError.new "Empty key in nest list 22: #{fields_text} | #{single_nests}" - end - end - - def self.parse_nest_groups(fields_text : String, &) : Nil - nest_stack = [] of NamedTuple(group_name: String, closing_bracket_index: Int64) - bracket_pairs = get_bracket_pairs(fields_text, true) - - text_index = 0 - regex_index = 0 - - while regex_result = self.nest_group_pattern.match(fields_text, regex_index) - raw_match = regex_result[0] - group_name = regex_result["groupname"] - - text_index = regex_result.begin - regex_index = regex_result.end - - if text_index.nil? || regex_index.nil? - raise FieldsParser::ParseError.new "Received invalid index while parsing nest groups: text_index: #{text_index} | regex_index: #{regex_index}" - end - - offset = raw_match.starts_with?(',') ? 1 : 0 - - opening_bracket_index = (text_index + group_name.size) + offset - closing_bracket_index = bracket_pairs[opening_bracket_index] - content_start = opening_bracket_index + 1 - - content = fields_text[content_start...closing_bracket_index] - - if content.empty? - raise FieldsParser::ParseError.new "Empty nest group at position #{content_start}" - else - content = remove_nest_groups(content) - end - - while nest_stack.size > 0 && closing_bracket_index > nest_stack[nest_stack.size - 1][:closing_bracket_index] - if nest_stack.size - nest_stack.pop - end - end - - group_name.split('/').each do |name| - nest_stack.push({ - group_name: name, - closing_bracket_index: closing_bracket_index, - }) - end - - if !content.empty? - properties = content.split(',') - - properties.each do |prop| - nest_list = nest_stack.map { |nest_prop| nest_prop[:group_name] } - - if !prop.empty? - if prop.includes?('/') - parse_single_nests(prop) { |list| nest_list += list } - else - nest_list.push prop - end - else - raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list << prop}" - end - - yield nest_list - end - end - end - end - - def self.remove_nest_groups(text : String) : String - content_bracket_pairs = get_bracket_pairs(text, false) - - content_bracket_pairs.each_key.to_a.reverse.each do |opening_bracket| - closing_bracket = content_bracket_pairs[opening_bracket] - last_comma = text.rindex(',', opening_bracket) || 0 - - text = text[0...last_comma] + text[closing_bracket + 1...text.size] - end - - return text.starts_with?(',') ? text[1...text.size] : text - end - - def self.get_bracket_pairs(text : String, recursive = true) : BracketIndex - istart = [] of Int64 - bracket_index = BracketIndex.new - - text.each_char_with_index do |char, index| - if char == '(' - istart.push(index.to_i64) - end - - if char == ')' - begin - opening = istart.pop - if recursive || (!recursive && istart.size == 0) - bracket_index[opening] = index.to_i64 - end - rescue - raise FieldsParser::ParseError.new "No matching opening parenthesis at: #{index}" - end - end - end - - if istart.size != 0 - idx = istart.pop - raise FieldsParser::ParseError.new "No matching closing parenthesis at: #{idx}" - end - - return bracket_index - end - end - - class FieldsGrouper - alias SkeletonValue = Hash(String, SkeletonValue) - - def self.create_json_skeleton(fields_text : String) : SkeletonValue - root_hash = {} of String => SkeletonValue - - FieldsParser.parse_fields(fields_text) do |nest_list| - current_item = root_hash - nest_list.each do |key| - if current_item[key]? - current_item = current_item[key] - else - current_item[key] = {} of String => SkeletonValue - current_item = current_item[key] - end - end - end - root_hash - end - - def self.create_grouped_fields_list(json_skeleton : SkeletonValue) : GroupedFieldsList - grouped_fields_list = GroupedFieldsList.new - json_skeleton.each do |key, value| - grouped_fields_list.push key - - nested_keys = create_grouped_fields_list(value) - grouped_fields_list.push nested_keys unless nested_keys.empty? - end - return grouped_fields_list - end - end - - class FilterError < Exception - end - - def self.filter(item : JSON::Any, fields_text : String, in_place : Bool = true) - skeleton = FieldsGrouper.create_json_skeleton(fields_text) - grouped_fields_list = FieldsGrouper.create_grouped_fields_list(skeleton) - filter(item, grouped_fields_list, in_place) - end - - def self.filter(item : JSON::Any, grouped_fields_list : GroupedFieldsList, in_place : Bool = true) : JSON::Any - item = item.clone unless in_place - - if !item.as_h? && !item.as_a? - raise FilterError.new "Can't filter '#{item}' by #{grouped_fields_list}" - end - - top_level_keys = Array(String).new - grouped_fields_list.each do |value| - if value.is_a? String - top_level_keys.push value - elsif value.is_a? Array - if !top_level_keys.empty? - key_to_filter = top_level_keys.last - - if item.as_h? - filter(item[key_to_filter], value, in_place: true) - elsif item.as_a? - item.as_a.each { |arr_item| filter(arr_item[key_to_filter], value, in_place: true) } - end - else - raise FilterError.new "Tried to filter while top level keys list is empty" - end - end - end - - if item.as_h? - item.as_h.select! top_level_keys - elsif item.as_a? - item.as_a.map { |value| filter(value, top_level_keys, in_place: true) } - end - - item - end -end diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index a006d602..e438e3b9 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -262,7 +262,7 @@ def get_referer(env, fallback = "/", unroll = true) end referer = referer.request_target - referer = "/" + referer.gsub(/[^\/?@&%=\-_.:,0-9a-zA-Z]/, "").lstrip("/\\") + referer = "/" + referer.gsub(/[^\/?@&%=\-_.:,*0-9a-zA-Z]/, "").lstrip("/\\") if referer == env.request.path referer = fallback diff --git a/src/invidious/helpers/webvtt.cr b/src/invidious/helpers/webvtt.cr index 56f761ed..260d250f 100644 --- a/src/invidious/helpers/webvtt.cr +++ b/src/invidious/helpers/webvtt.cr @@ -4,13 +4,23 @@ module WebVTT # A WebVTT builder generates WebVTT files private class Builder + # See https://developer.mozilla.org/en-US/docs/Web/API/WebVTT_API#cue_payload + private ESCAPE_SUBSTITUTIONS = { + '&' => "&", + '<' => "<", + '>' => ">", + '\u200E' => "‎", + '\u200F' => "‏", + '\u00A0' => " ", + } + def initialize(@io : IO) end # Writes an vtt cue with the specified time stamp and contents def cue(start_time : Time::Span, end_time : Time::Span, text : String) timestamp(start_time, end_time) - @io << text + @io << self.escape(text) @io << "\n\n" end @@ -29,6 +39,10 @@ module WebVTT @io << '.' << timestamp.milliseconds.to_s.rjust(3, '0') end + private def escape(text : String) : String + return text.gsub(ESCAPE_SUBSTITUTIONS) + end + def document(setting_fields : Hash(String, String)? = nil, &) @io << "WEBVTT\n" diff --git a/src/invidious/jobs/bypass_captcha_job.cr b/src/invidious/jobs/bypass_captcha_job.cr deleted file mode 100644 index 71f8a938..00000000 --- a/src/invidious/jobs/bypass_captcha_job.cr +++ /dev/null @@ -1,135 +0,0 @@ -class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob - def begin - loop do - begin - random_video = PG_DB.query_one?("select id, ucid from (select id, ucid from channel_videos limit 1000) as s ORDER BY RANDOM() LIMIT 1", as: {id: String, ucid: String}) - if !random_video - random_video = {id: "zj82_v2R6ts", ucid: "UCK87Lox575O_HCHBWaBSyGA"} - end - {"/watch?v=#{random_video["id"]}&gl=US&hl=en&has_verified=1&bpctr=9999999999", produce_channel_videos_url(ucid: random_video["ucid"])}.each do |path| - response = YT_POOL.client &.get(path) - if response.body.includes?("To continue with your YouTube experience, please fill out the form below.") - html = XML.parse_html(response.body) - form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil! - site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"] - s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"] - - inputs = {} of String => String - form.xpath_nodes(%(.//input[@name])).map do |node| - inputs[node["name"]] = node["value"] - end - - headers = response.cookies.add_request_headers(HTTP::Headers.new) - - response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/createTask", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "task" => { - "type" => "NoCaptchaTaskProxyless", - "websiteURL" => "https://www.youtube.com#{path}", - "websiteKey" => site_key, - "recaptchaDataSValue" => s_value, - }, - }.to_json).body) - - raise response["error"].as_s if response["error"]? - task_id = response["taskId"].as_i - - loop do - sleep 10.seconds - - response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/getTaskResult", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "taskId" => task_id, - }.to_json).body) - - if response["status"]?.try &.== "ready" - break - elsif response["errorId"]?.try &.as_i != 0 - raise response["errorDescription"].as_s - end - end - - inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s - headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || "" - response = YT_POOL.client &.post("/das_captcha", headers, form: inputs) - - response.cookies - .select { |cookie| cookie.name != "PREF" } - .each { |cookie| CONFIG.cookies << cookie } - - # Persist cookies between runs - File.write("config/config.yml", CONFIG.to_yaml) - elsif response.headers["Location"]?.try &.includes?("/sorry/index") - location = response.headers["Location"].try { |u| URI.parse(u) } - headers = HTTP::Headers{":authority" => location.host.not_nil!} - response = YT_POOL.client &.get(location.request_target, headers) - - html = XML.parse_html(response.body) - form = html.xpath_node(%(//form[@action="index"])).not_nil! - site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"] - s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"] - - inputs = {} of String => String - form.xpath_nodes(%(.//input[@name])).map do |node| - inputs[node["name"]] = node["value"] - end - - captcha_client = HTTPClient.new(URI.parse(CONFIG.captcha_api_url)) - captcha_client.family = CONFIG.force_resolve || Socket::Family::INET - response = JSON.parse(captcha_client.post("/createTask", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "task" => { - "type" => "NoCaptchaTaskProxyless", - "websiteURL" => location.to_s, - "websiteKey" => site_key, - "recaptchaDataSValue" => s_value, - }, - }.to_json).body) - - captcha_client.close - - raise response["error"].as_s if response["error"]? - task_id = response["taskId"].as_i - - loop do - sleep 10.seconds - - response = JSON.parse(captcha_client.post("/getTaskResult", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "taskId" => task_id, - }.to_json).body) - - if response["status"]?.try &.== "ready" - break - elsif response["errorId"]?.try &.as_i != 0 - raise response["errorDescription"].as_s - end - end - - inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s - headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || "" - response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs) - headers = HTTP::Headers{ - "Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0], - } - cookies = HTTP::Cookies.from_client_headers(headers) - - cookies.each { |cookie| CONFIG.cookies << cookie } - - # Persist cookies between runs - File.write("config/config.yml", CONFIG.to_yaml) - end - end - rescue ex - LOGGER.error("BypassCaptchaJob: #{ex.message}") - ensure - sleep 1.minute - Fiber.yield - end - end - end -end diff --git a/src/invidious/jobs/statistics_refresh_job.cr b/src/invidious/jobs/statistics_refresh_job.cr index 72d1ce88..66c91ad5 100644 --- a/src/invidious/jobs/statistics_refresh_job.cr +++ b/src/invidious/jobs/statistics_refresh_job.cr @@ -56,8 +56,8 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob users = STATISTICS.dig("usage", "users").as(Hash(String, Int64)) users["total"] = Invidious::Database::Statistics.count_users_total - users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_1m - users["activeMonth"] = Invidious::Database::Statistics.count_users_active_6m + users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_6m + users["activeMonth"] = Invidious::Database::Statistics.count_users_active_1m STATISTICS["metadata"] = { "updatedAt" => Time.utc.to_unix, diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index b42ecd1a..12942906 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -191,6 +191,8 @@ module Invidious::Routes::API::V1::Misc json.object do json.field "ucid", sub_endpoint["browseId"].as_s if sub_endpoint["browseId"]? json.field "videoId", sub_endpoint["videoId"].as_s if sub_endpoint["videoId"]? + json.field "playlistId", sub_endpoint["playlistId"].as_s if sub_endpoint["playlistId"]? + json.field "startTimeSeconds", sub_endpoint["startTimeSeconds"].as_i if sub_endpoint["startTimeSeconds"]? json.field "params", params.try &.as_s json.field "pageType", pageType end diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index 9fb283c2..2922b060 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -32,11 +32,14 @@ module Invidious::Routes::API::V1::Search begin client = HTTP::Client.new("suggestqueries-clients6.youtube.com") - url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&xssi=t&gs_ri=youtube&ds=yt" + client.before_request { |r| add_yt_headers(r) } + + url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt" response = client.get(url).body + client.close - body = JSON.parse(response[5..-1]).as_a + body = JSON.parse(response[19..-2]).as_a suggestions = body[1].as_a[0..-2] JSON.build do |json| diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 1017ac9d..9281f4dd 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -363,4 +363,47 @@ module Invidious::Routes::API::V1::Videos end end end + + def self.clips(env) + locale = env.get("preferences").as(Preferences).locale + + env.response.content_type = "application/json" + + clip_id = env.params.url["id"] + region = env.params.query["region"]? + proxy = {"1", "true"}.any? &.== env.params.query["local"]? + + response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}") + return error_json(400, "Invalid clip ID") if response["error"]? + + video_id = response.dig?("endpoint", "watchEndpoint", "videoId").try &.as_s + return error_json(400, "Invalid clip ID") if video_id.nil? + + start_time = nil + end_time = nil + clip_title = nil + + if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s + start_time, end_time, clip_title = parse_clip_parameters(params) + end + + begin + video = get_video(video_id, region: region) + rescue ex : NotFoundException + return error_json(404, ex) + rescue ex + return error_json(500, ex) + end + + return JSON.build do |json| + json.object do + json.field "startTime", start_time + json.field "endTime", end_time + json.field "clipTitle", clip_title + json.field "video" do + Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy) + end + end + end + end end diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 40bca008..e20a7139 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -407,14 +407,23 @@ module Invidious::Routes::Feeds end spawn do - rss = XML.parse_html(body) - rss.xpath_nodes("//feed/entry").each do |entry| - id = entry.xpath_node("videoid").not_nil!.content - author = entry.xpath_node("author/name").not_nil!.content - published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) - updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) - - video = get_video(id, force_refresh: true) + # TODO: unify this with the other almost identical looking parts in this and channels.cr somehow? + namespaces = { + "yt" => "http://www.youtube.com/xml/schemas/2015", + "default" => "http://www.w3.org/2005/Atom", + } + rss = XML.parse(body) + rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry| + id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content + author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content + published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content) + updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content) + + begin + video = get_video(id, force_refresh: true) + rescue + next # skip this video since it raised an exception (e.g. it is a scheduled live event) + end if CONFIG.enable_user_notifications # Deliver notifications to `/api/v1/auth/notifications` diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 1d5aa914..ec18f3b8 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -42,7 +42,7 @@ module Invidious::Routes::VideoPlayback headers["Range"] = "bytes=#{range_for_head}" end - client = make_client(URI.parse(host), region) + client = make_client(URI.parse(host), region, force_resolve = true) response = HTTP::Client::Response.new(500) error = "" 5.times do @@ -57,7 +57,7 @@ module Invidious::Routes::VideoPlayback if new_host != host host = new_host client.close - client = make_client(URI.parse(new_host), region) + client = make_client(URI.parse(new_host), region, force_resolve = true) end url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" @@ -71,7 +71,7 @@ module Invidious::Routes::VideoPlayback fvip = "3" host = "https://r#{fvip}---#{mn}.googlevideo.com" - client = make_client(URI.parse(host), region) + client = make_client(URI.parse(host), region, force_resolve = true) rescue ex error = ex.message end @@ -196,7 +196,7 @@ module Invidious::Routes::VideoPlayback break else client.close - client = make_client(URI.parse(host), region) + client = make_client(URI.parse(host), region, force_resolve = true) end end diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 3d935f0a..aabe8dfc 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -275,6 +275,12 @@ module Invidious::Routes::Watch return error_template(400, "Invalid clip ID") if response["error"]? if video_id = response.dig?("endpoint", "watchEndpoint", "videoId") + if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s + start_time, end_time, _ = parse_clip_parameters(params) + env.params.query["start"] = start_time.to_s if start_time != nil + env.params.query["end"] = end_time.to_s if end_time != nil + end + return env.redirect "/watch?v=#{video_id}&#{env.params.query}" else return error_template(404, "The requested clip doesn't exist") diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index d6bd991c..ba05da19 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -235,6 +235,7 @@ module Invidious::Routing get "/api/v1/captions/:id", {{namespace}}::Videos, :captions get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations get "/api/v1/comments/:id", {{namespace}}::Videos, :comments + get "/api/v1/clips/:id", {{namespace}}::Videos, :clips # Feeds get "/api/v1/trending", {{namespace}}::Feeds, :trending diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr index c2b5c758..bf968734 100644 --- a/src/invidious/search/filters.cr +++ b/src/invidious/search/filters.cr @@ -300,9 +300,9 @@ module Invidious::Search object["9:varint"] = ((page - 1) * 20).to_i64 end - # If the object is empty, return an empty string, - # otherwise encode to protobuf then to base64 - return "" if object.empty? + # Prevent censoring of self harm topics + # See https://github.com/iv-org/invidious/issues/4398 + object["30:varint"] = 1.to_i64 return object .try { |i| Protodec::Any.cast_json(i) } diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 9fbd1374..a8f02056 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -227,8 +227,22 @@ struct Video info.dig?("streamingData", "hlsManifestUrl").try &.as_s end - def dash_manifest_url - info.dig?("streamingData", "dashManifestUrl").try &.as_s + def dash_manifest_url : String? + raw_dash_url = info.dig?("streamingData", "dashManifestUrl").try &.as_s + return nil if raw_dash_url.nil? + + # Use manifest v5 parameter to reduce file size + # See https://github.com/iv-org/invidious/issues/4186 + dash_url = URI.parse(raw_dash_url) + dash_query = dash_url.query || "" + + if dash_query.empty? + dash_url.path = "#{dash_url.path}/mpd_version/5" + else + dash_url.query = "#{dash_query}&mpd_version=5" + end + + return dash_url.to_s end def genre_url : String? diff --git a/src/invidious/videos/clip.cr b/src/invidious/videos/clip.cr new file mode 100644 index 00000000..29c57182 --- /dev/null +++ b/src/invidious/videos/clip.cr @@ -0,0 +1,22 @@ +require "json" + +# returns start_time, end_time and clip_title +def parse_clip_parameters(params) : {Float64?, Float64?, String?} + decoded_protobuf = params.try { |i| URI.decode_www_form(i) } + .try { |i| Base64.decode(i) } + .try { |i| IO::Memory.new(i) } + .try { |i| Protodec::Any.parse(i) } + + start_time = decoded_protobuf + .try(&.["50:0:embedded"]["2:1:varint"].as_i64) + .try { |i| i/1000 } + + end_time = decoded_protobuf + .try(&.["50:0:embedded"]["3:2:varint"].as_i64) + .try { |i| i/1000 } + + clip_title = decoded_protobuf + .try(&.["50:0:embedded"]["4:3:string"].as_s) + + return start_time, end_time, clip_title +end diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 77520dbe..75fe4a36 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -142,8 +142,9 @@ end def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)? LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.") - # 2AMBCgIQBg is a workaround for streaming URLs that returns a 403. - response = YoutubeAPI.player(video_id: id, params: "2AMBCgIQBg", client_config: client_config) + # CgIIAdgDAQ%3D%3D is a workaround for streaming URLs that returns a 403. + # https://github.com/LuanRT/YouTube.js/pull/624 + response = YoutubeAPI.player(video_id: id, params: "CgIIAdgDAQ%3D%3D", client_config: client_config) playability_status = response["playabilityStatus"]["status"] LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.") diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index 031b46da..6d227cfc 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -82,11 +82,19 @@ </div> <div class="video-card-row flexible"> - <div class="flex-left"><a href="/channel/<%= item.ucid %>"> - <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> - <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> - </p> - </a></div> + <div class="flex-left"> + <% if !item.ucid.to_s.empty? %> + <a href="/channel/<%= item.ucid %>"> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + </a> + <% else %> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + <% end %> + </div> </div> <% when Category %> <% else %> @@ -160,11 +168,19 @@ </div> <div class="video-card-row flexible"> - <div class="flex-left"><a href="/channel/<%= item.ucid %>"> - <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> - <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> - </p> - </a></div> + <div class="flex-left"> + <% if !item.ucid.to_s.empty? %> + <a href="/channel/<%= item.ucid %>"> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + </a> + <% else %> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + <% end %> + </div> <%= rendered "components/video-context-buttons" %> </div> diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index 77265679..9904b4fc 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -1,5 +1,9 @@ +<% + locale = env.get("preferences").as(Preferences).locale + dark_mode = env.get("preferences").as(Preferences).dark_mode +%> <!DOCTYPE html> -<html lang="<%= env.get("preferences").as(Preferences).locale %>"> +<html lang="<%= locale %>"> <head> <meta charset="utf-8"> @@ -17,19 +21,14 @@ <link rel="stylesheet" href="/css/grids-responsive-min.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/ionicons.min.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/default.css?v=<%= ASSET_COMMIT %>"> + <link rel="stylesheet" href="/css/carousel.css?v=<%= ASSET_COMMIT %>"> <script src="/js/_helpers.js?v=<%= ASSET_COMMIT %>"></script> </head> -<% - locale = env.get("preferences").as(Preferences).locale - dark_mode = env.get("preferences").as(Preferences).dark_mode -%> - <body class="<%= dark_mode.blank? ? "no" : dark_mode %>-theme"> - <span style="display:none" id="dark_mode_pref"><%= env.get("preferences").as(Preferences).dark_mode %></span> + <span style="display:none" id="dark_mode_pref"><%= dark_mode %></span> <div class="pure-g"> - <div class="pure-u-1 pure-u-md-2-24"></div> - <div class="pure-u-1 pure-u-md-20-24" id="contents"> + <div class="pure-u-1 pure-u-xl-20-24" id="contents"> <div class="pure-g navbar h-box"> <% if navbar_search %> <div class="pure-u-1 pure-u-md-4-24"> @@ -43,8 +42,8 @@ <div class="pure-u-1 pure-u-md-8-24 user-field"> <% if env.get? "user" %> <div class="pure-u-1-4"> - <a id="toggle_theme" href="/toggle_theme?referer=<%= env.get?("current_page") %>" class="pure-menu-heading"> - <% if env.get("preferences").as(Preferences).dark_mode == "dark" %> + <a id="toggle_theme" href="/toggle_theme?referer=<%= env.get?("current_page") %>" class="pure-menu-heading" title="<%= translate(locale, "toggle_theme") %>"> + <% if dark_mode == "dark" %> <i class="icon ion-ios-sunny"></i> <% else %> <i class="icon ion-ios-moon"></i> @@ -81,8 +80,8 @@ </div> <% else %> <div class="pure-u-1-3"> - <a id="toggle_theme" href="/toggle_theme?referer=<%= env.get?("current_page") %>" class="pure-menu-heading"> - <% if env.get("preferences").as(Preferences).dark_mode == "dark" %> + <a id="toggle_theme" href="/toggle_theme?referer=<%= env.get?("current_page") %>" class="pure-menu-heading" title="<%= translate(locale, "toggle_theme") %>"> + <% if dark_mode == "dark" %> <i class="icon ion-ios-sunny"></i> <% else %> <i class="icon ion-ios-moon"></i> @@ -156,7 +155,6 @@ </footer> </div> - <div class="pure-u-1 pure-u-md-2-24"></div> </div> <script src="/js/handlers.js?v=<%= ASSET_COMMIT %>"></script> <script src="/js/themes.js?v=<%= ASSET_COMMIT %>"></script> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 07474896..7a1cf2c3 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -118,7 +118,7 @@ we're going to need to do it here in order to allow for translations. link_yt_embed = URI.new(scheme: "https", host: "www.youtube.com", path: "/embed/#{video.id}") if !plid.nil? && !continuation.nil? - link_yt_param = URI::Params{"plid" => [plid], "index" => [continuation.to_s]} + link_yt_param = URI::Params{"list" => [plid], "index" => [continuation.to_s]} link_yt_watch = IV::HttpServer::Utils.add_params_to_url(link_yt_watch, link_yt_param) link_yt_embed = IV::HttpServer::Utils.add_params_to_url(link_yt_embed, link_yt_param) end @@ -346,7 +346,7 @@ we're going to need to do it here in order to allow for translations. <h5 class="pure-g"> <div class="pure-u-14-24"> - <% if rv["ucid"]? %> + <% if !rv["ucid"].empty? %> <b style="width:100%"><a href="/channel/<%= rv["ucid"] %>"><%= rv["author"]? %><% if rv["author_verified"]? == "true" %> <i class="icon ion ion-md-checkmark-circle"></i><% end %></a></b> <% else %> <b style="width:100%"><%= rv["author"]? %><% if rv["author_verified"]? == "true" %> <i class="icon ion ion-md-checkmark-circle"></i><% end %></b> diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 03ff0ee4..81cfb272 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -1,7 +1,6 @@ def add_yt_headers(request) - if request.headers["User-Agent"] == "Crystal" - request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" - end + request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" + request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" @@ -27,7 +26,7 @@ struct YoutubeConnectionPool def client(region = nil, &block) if region - conn = make_client(url, region) + conn = make_client(url, region, force_resolve = true) response = yield conn else conn = pool.checkout @@ -60,9 +59,14 @@ struct YoutubeConnectionPool end end -def make_client(url : URI, region = nil) +def make_client(url : URI, region = nil, force_resolve : Bool = false) client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure) - client.family = CONFIG.force_resolve + + # Some services do not support IPv6. + if force_resolve + client.family = CONFIG.force_resolve + end + client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" client.read_timeout = 10.seconds client.connect_timeout = 10.seconds @@ -81,8 +85,8 @@ def make_client(url : URI, region = nil) return client end -def make_client(url : URI, region = nil, &block) - client = make_client(url, region) +def make_client(url : URI, region = nil, force_resolve : Bool = false, &block) + client = make_client(url, region, force_resolve) begin yield client ensure diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 56325cf7..0e72957e 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -822,9 +822,9 @@ module HelperExtractors end # Retrieves the ID required for querying the InnerTube browse endpoint. - # Raises when it's unable to do so + # Returns an empty string when it's unable to do so def self.get_browse_id(container) - return container.dig("navigationEndpoint", "browseEndpoint", "browseId").as_s + return container.dig?("navigationEndpoint", "browseEndpoint", "browseId").try &.as_s || "" end end diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index a5e621f2..9e0631f6 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -7,17 +7,18 @@ module YoutubeAPI private DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" - private ANDROID_APP_VERSION = "18.20.38" - # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1308 - private ANDROID_USER_AGENT = "com.google.android.youtube/18.20.38 (Linux; U; Android 12; US) gzip" + # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history + private ANDROID_APP_VERSION = "19.09.36" + private ANDROID_USER_AGENT = "com.google.android.youtube/19.09.36 (Linux; U; Android 12; US) gzip" private ANDROID_SDK_VERSION = 31_i64 private ANDROID_VERSION = "12" - private IOS_APP_VERSION = "18.21.3" - # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1330 - private IOS_USER_AGENT = "com.google.ios.youtube/18.21.3 (iPhone14,5; U; CPU iOS 15_6 like Mac OS X;)" - # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1224 - private IOS_VERSION = "15.6.0.19G71" + # For Apple device names, see https://gist.github.com/adamawolf/3048717 + # For iOS versions, see https://en.wikipedia.org/wiki/IOS_version_history#Releases, + # then go to the dedicated article of the major version you want. + private IOS_APP_VERSION = "19.09.3" + private IOS_USER_AGENT = "com.google.ios.youtube/19.09.3 (iPhone14,5; U; CPU iOS 17_4 like Mac OS X;)" + private IOS_VERSION = "17.4.0.21E219" # Major.Minor.Patch.Build private WINDOWS_VERSION = "10.0" @@ -45,7 +46,7 @@ module YoutubeAPI ClientType::Web => { name: "WEB", name_proto: "1", - version: "2.20230602.01.00", + version: "2.20240304.00.00", api_key: DEFAULT_API_KEY, screen: "WATCH_FULL_SCREEN", os_name: "Windows", @@ -55,7 +56,7 @@ module YoutubeAPI ClientType::WebEmbeddedPlayer => { name: "WEB_EMBEDDED_PLAYER", name_proto: "56", - version: "1.20220803.01.00", + version: "1.20240303.00.00", api_key: DEFAULT_API_KEY, screen: "EMBED", os_name: "Windows", @@ -65,7 +66,7 @@ module YoutubeAPI ClientType::WebMobile => { name: "MWEB", name_proto: "2", - version: "2.20230531.05.00", + version: "2.20240304.08.00", api_key: DEFAULT_API_KEY, os_name: "Android", os_version: ANDROID_VERSION, @@ -74,7 +75,7 @@ module YoutubeAPI ClientType::WebScreenEmbed => { name: "WEB", name_proto: "1", - version: "2.20220804.00.00", + version: "2.20240304.00.00", api_key: DEFAULT_API_KEY, screen: "EMBED", os_name: "Windows", @@ -99,7 +100,7 @@ module YoutubeAPI name: "ANDROID_EMBEDDED_PLAYER", name_proto: "55", version: ANDROID_APP_VERSION, - api_key: DEFAULT_API_KEY, + api_key: "AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw", }, ClientType::AndroidScreenEmbed => { name: "ANDROID", @@ -143,9 +144,9 @@ module YoutubeAPI ClientType::IOSMusic => { name: "IOS_MUSIC", name_proto: "26", - version: "5.21", + version: "6.42", api_key: "AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s", - user_agent: "com.google.ios.youtubemusic/5.21 (iPhone14,5; U; CPU iOS 15_6 like Mac OS X;)", + user_agent: "com.google.ios.youtubemusic/6.42 (iPhone14,5; U; CPU iOS 17_4 like Mac OS X;)", device_make: "Apple", device_model: "iPhone14,5", os_name: "iPhone", @@ -158,7 +159,7 @@ module YoutubeAPI ClientType::TvHtml5 => { name: "TVHTML5", name_proto: "7", - version: "7.20220325", + version: "7.20240304.10.00", api_key: DEFAULT_API_KEY, }, ClientType::TvHtml5ScreenEmbed => { |