diff options
Diffstat (limited to 'src')
25 files changed, 234 insertions, 405 deletions
diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index 8b60a728..b5a27667 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -14,6 +14,7 @@ record AboutChannel, is_family_friendly : Bool, allowed_regions : Array(String), tabs : Array(String), + tags : Array(String), verified : Bool def get_about_info(ucid, locale) : AboutChannel @@ -43,6 +44,8 @@ def get_about_info(ucid, locale) : AboutChannel auto_generated = true end + tags = [] of String + if auto_generated author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s @@ -52,7 +55,13 @@ def get_about_info(ucid, locale) : AboutChannel banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? banner = banners.try &.[-1]?.try &.["url"].as_s? - description_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] + description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] + # some channels have the description in a simpleText + # ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/ + description_node = description_base_node.dig?("simpleText") || description_base_node + + tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges") + .try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String else author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s @@ -70,6 +79,7 @@ def get_about_info(ucid, locale) : AboutChannel # end description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]? + tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String end is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool @@ -131,7 +141,8 @@ def get_about_info(ucid, locale) : AboutChannel # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] auto_generated = ( (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \ - extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" + extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" || + channel_about_meta.dig?("links", 0, "channelExternalLinkViewModel", "title", "content").try &.as_s == "Auto-generated by YouTube" ) end end @@ -155,6 +166,7 @@ def get_about_info(ucid, locale) : AboutChannel is_family_friendly: is_family_friendly, allowed_regions: allowed_regions, tabs: tab_names, + tags: tags, verified: author_verified || false, ) end diff --git a/src/invidious/database/statistics.cr b/src/invidious/database/statistics.cr index 1df549e2..9e4963fd 100644 --- a/src/invidious/database/statistics.cr +++ b/src/invidious/database/statistics.cr @@ -15,7 +15,7 @@ module Invidious::Database::Statistics PG_DB.query_one(request, as: Int64) end - def count_users_active_1m : Int64 + def count_users_active_6m : Int64 request = <<-SQL SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '6 months' @@ -24,7 +24,7 @@ module Invidious::Database::Statistics PG_DB.query_one(request, as: Int64) end - def count_users_active_6m : Int64 + def count_users_active_1m : Int64 request = <<-SQL SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '1 month' diff --git a/src/invidious/frontend/comments_reddit.cr b/src/invidious/frontend/comments_reddit.cr index b5647bae..4dda683e 100644 --- a/src/invidious/frontend/comments_reddit.cr +++ b/src/invidious/frontend/comments_reddit.cr @@ -33,7 +33,7 @@ module Invidious::Frontend::Comments <a href="javascript:void(0)" data-onclick="toggle_parent">[ − ]</a> <b><a href="https://www.reddit.com/user/#{child.author}">#{child.author}</a></b> #{translate_count(locale, "comments_points_count", child.score, NumberFormatting::Separator)} - <span title="#{child.created_utc.to_s(translate(locale, "%a %B %-d %T %Y UTC"))}">#{translate(locale, "`x` ago", recode_date(child.created_utc, locale))}</span> + <span title="#{child.created_utc.to_s("%a %B %-d %T %Y UTC")}">#{translate(locale, "`x` ago", recode_date(child.created_utc, locale))}</span> <a href="https://www.reddit.com#{child.permalink}" title="#{translate(locale, "permalink")}">#{translate(locale, "permalink")}</a> </p> <div> diff --git a/src/invidious/frontend/comments_youtube.cr b/src/invidious/frontend/comments_youtube.cr index ecc0bc1b..aecac87f 100644 --- a/src/invidious/frontend/comments_youtube.cr +++ b/src/invidious/frontend/comments_youtube.cr @@ -107,6 +107,36 @@ module Invidious::Frontend::Comments </div> END_HTML end + when "multiImage" + html << <<-END_HTML + <section class="carousel"> + <a class="skip-link" href="#skip-#{child["commentId"]}">#{translate(locale, "carousel_skip")}</a> + <div class="slides"> + END_HTML + image_array = attachment["images"].as_a + + image_array.each_index do |i| + html << <<-END_HTML + <div class="slides-item slide-#{i + 1}" id="#{child["commentId"]}-slide-#{i + 1}" aria-label="#{translate(locale, "carousel_slide", {"current" => (i + 1).to_s, "total" => image_array.size.to_s})}" tabindex="0"> + <img loading="lazy" src="/ggpht#{URI.parse(image_array[i][1]["url"].as_s).request_target}" alt="" /> + </div> + END_HTML + end + + html << <<-END_HTML + </div> + <div class="carousel__nav"> + END_HTML + attachment["images"].as_a.each_index do |i| + html << <<-END_HTML + <a class="slider-nav" href="##{child["commentId"]}-slide-#{i + 1}" aria-label="#{translate(locale, "carousel_go_to", (i + 1).to_s)}" tabindex="-1" aria-hidden="true">#{i + 1}</a> + END_HTML + end + html << <<-END_HTML + </div> + <div id="skip-#{child["commentId"]}"></div> + </section> + END_HTML else nil # Ignore end end diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index d140a858..174f620d 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -142,63 +142,8 @@ class APIHandler < Kemal::Handler exclude ["/api/v1/auth/notifications"], "POST" def call(env) - return call_next env unless only_match? env - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - # Since /api/v1/notifications is an event-stream, we don't want - # to wrap the response - return call_next env if exclude_match? env - - # Here we swap out the socket IO so we can modify the response as needed - output = env.response.output - env.response.output = IO::Memory.new - - begin - call_next env - - env.response.output.rewind - - if env.response.output.as(IO::Memory).size != 0 && - env.response.headers.includes_word?("Content-Type", "application/json") - response = JSON.parse(env.response.output) - - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - - if env.params.query["pretty"]?.try &.== "1" - response = response.to_pretty_json - else - response = response.to_json - end - else - response = env.response.output.gets_to_end - end - rescue ex - env.response.content_type = "application/json" if env.response.headers.includes_word?("Content-Type", "text/html") - env.response.status_code = 500 - - if env.response.headers.includes_word?("Content-Type", "application/json") - response = {"error" => ex.message || "Unspecified error"} - - if env.params.query["pretty"]?.try &.== "1" - response = response.to_pretty_json - else - response = response.to_json - end - end - ensure - env.response.output = output - env.response.print response - - env.response.flush - end + env.response.headers["Access-Control-Allow-Origin"] = "*" if only_match?(env) + call_next env end end diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 6dc9860e..6add0237 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -78,15 +78,6 @@ def create_notification_stream(env, topics, connection_channel) video.published = published response = JSON.parse(video.to_json(locale, nil)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -113,15 +104,6 @@ def create_notification_stream(env, topics, connection_channel) Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video| response = JSON.parse(video.to_json(locale)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -155,15 +137,6 @@ def create_notification_stream(env, topics, connection_channel) video.published = Time.unix(published) response = JSON.parse(video.to_json(locale, nil)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index 76e477a4..23a1aafc 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -78,7 +78,7 @@ def load_all_locales return locales end -def translate(locale : String?, key : String, text : String | Nil = nil) : String +def translate(locale : String?, key : String, text : String | Hash(String, String) | Nil = nil) : String # Log a warning if "key" doesn't exist in en-US locale and return # that key as the text, so this is more or less transparent to the user. if !LOCALES["en-US"].has_key?(key) @@ -101,10 +101,12 @@ def translate(locale : String?, key : String, text : String | Nil = nil) : Strin match_length = 0 raw_data.as_h.each do |hash_key, value| - if md = text.try &.match(/#{hash_key}/) - if md[0].size >= match_length - translation = value.as_s - match_length = md[0].size + if text.is_a?(String) + if md = text.try &.match(/#{hash_key}/) + if md[0].size >= match_length + translation = value.as_s + match_length = md[0].size + end end end end @@ -114,8 +116,13 @@ def translate(locale : String?, key : String, text : String | Nil = nil) : Strin raise "Invalid translation \"#{raw_data}\"" end - if text + if text.is_a?(String) translation = translation.gsub("`x`", text) + elsif text.is_a?(Hash(String, String)) + # adds support for multi string interpolation. Based on i18next https://www.i18next.com/translation-function/interpolation#basic + text.each_key do |hash_key| + translation = translation.gsub("{{#{hash_key}}}", text[hash_key]) + end end return translation diff --git a/src/invidious/helpers/i18next.cr b/src/invidious/helpers/i18next.cr index 252af6b9..9f4077e1 100644 --- a/src/invidious/helpers/i18next.cr +++ b/src/invidious/helpers/i18next.cr @@ -47,19 +47,19 @@ module I18next::Plurals private PLURAL_SETS = { PluralForms::Single_gt_one => [ - "ach", "ak", "am", "arn", "br", "fil", "gun", "ln", "mfe", "mg", - "mi", "oc", "pt", "tg", "tl", "ti", "tr", "uz", "wa", + "ach", "ak", "am", "arn", "br", "fa", "fil", "gun", "ln", "mfe", "mg", + "mi", "oc", "pt-PT", "tg", "tl", "ti", "tr", "uz", "wa", ], PluralForms::Single_not_one => [ "af", "an", "ast", "az", "bg", "bn", "ca", "da", "de", "dev", "el", "en", - "eo", "es", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi", + "eo", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi", "hu", "hy", "ia", "kk", "kn", "ku", "lb", "mai", "ml", "mn", "mr", "nah", "nap", "nb", "ne", "nl", "nn", "no", "nso", "pa", "pap", "pms", "ps", "rm", "sco", "se", "si", "so", "son", "sq", "sv", "sw", "ta", "te", "tk", "ur", "yo", ], PluralForms::None => [ - "ay", "bo", "cgg", "fa", "ht", "id", "ja", "jbo", "ka", "km", "ko", "ky", + "ay", "bo", "cgg", "ht", "id", "ja", "jbo", "ka", "km", "ko", "ky", "lo", "ms", "sah", "su", "th", "tt", "ug", "vi", "wo", "zh", ], PluralForms::Dual_Slavic => [ @@ -90,11 +90,13 @@ module I18next::Plurals "sk" => PluralForms::Special_Czech_Slovak, "sl" => PluralForms::Special_Slovenian, # Mixed v3/v4 rules - "fr" => PluralForms::Special_French_Portuguese, - "hr" => PluralForms::Special_Hungarian_Serbian, - "it" => PluralForms::Special_Spanish_Italian, - "pt-BR" => PluralForms::Special_French_Portuguese, - "sr" => PluralForms::Special_Hungarian_Serbian, + "es" => PluralForms::Special_Spanish_Italian, + "fr" => PluralForms::Special_French_Portuguese, + "hr" => PluralForms::Special_Hungarian_Serbian, + "it" => PluralForms::Special_Spanish_Italian, + "pt" => PluralForms::Special_French_Portuguese, + "pt" => PluralForms::Special_French_Portuguese, + "sr" => PluralForms::Special_Hungarian_Serbian, } # These are the v1 and v2 compatible suffixes. @@ -165,7 +167,7 @@ module I18next::Plurals def get_plural_form(locale : String) : PluralForms # Extract the ISO 639-1 or 639-2 code from an RFC 5646 language code - if !locale.matches?(/^pt-BR$/) + if !locale.matches?(/^pt-PT$/) locale = locale.split('-')[0] end diff --git a/src/invidious/helpers/json_filter.cr b/src/invidious/helpers/json_filter.cr deleted file mode 100644 index 3f4080ba..00000000 --- a/src/invidious/helpers/json_filter.cr +++ /dev/null @@ -1,248 +0,0 @@ -module JSONFilter - alias BracketIndex = Hash(Int64, Int64) - - alias GroupedFieldsValue = String | Array(GroupedFieldsValue) - alias GroupedFieldsList = Array(GroupedFieldsValue) - - class FieldsParser - class ParseError < Exception - end - - # Returns the `Regex` pattern used to match nest groups - def self.nest_group_pattern : Regex - # uses a '.' character to match json keys as they are allowed - # to contain any unicode codepoint - /(?:|,)(?<groupname>[^,\n]*?)\(/ - end - - # Returns the `Regex` pattern used to check if there are any empty nest groups - def self.unnamed_nest_group_pattern : Regex - /^\(|\(\(|\/\(/ - end - - def self.parse_fields(fields_text : String, &) : Nil - if fields_text.empty? - raise FieldsParser::ParseError.new "Fields is empty" - end - - opening_bracket_count = fields_text.count('(') - closing_bracket_count = fields_text.count(')') - - if opening_bracket_count != closing_bracket_count - bracket_type = opening_bracket_count > closing_bracket_count ? "opening" : "closing" - raise FieldsParser::ParseError.new "There are too many #{bracket_type} brackets (#{opening_bracket_count}:#{closing_bracket_count})" - elsif match_result = unnamed_nest_group_pattern.match(fields_text) - raise FieldsParser::ParseError.new "Unnamed nest group at position #{match_result.begin}" - end - - # first, handle top-level single nested properties: items/id, playlistItems/snippet, etc - parse_single_nests(fields_text) { |nest_list| yield nest_list } - - # next, handle nest groups: items(id, etag, etc) - parse_nest_groups(fields_text) { |nest_list| yield nest_list } - end - - def self.parse_single_nests(fields_text : String, &) : Nil - single_nests = remove_nest_groups(fields_text) - - if !single_nests.empty? - property_nests = single_nests.split(',') - - property_nests.each do |nest| - nest_list = nest.split('/') - if nest_list.includes? "" - raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list}" - end - yield nest_list - end - # else - # raise FieldsParser::ParseError.new "Empty key in nest list 22: #{fields_text} | #{single_nests}" - end - end - - def self.parse_nest_groups(fields_text : String, &) : Nil - nest_stack = [] of NamedTuple(group_name: String, closing_bracket_index: Int64) - bracket_pairs = get_bracket_pairs(fields_text, true) - - text_index = 0 - regex_index = 0 - - while regex_result = self.nest_group_pattern.match(fields_text, regex_index) - raw_match = regex_result[0] - group_name = regex_result["groupname"] - - text_index = regex_result.begin - regex_index = regex_result.end - - if text_index.nil? || regex_index.nil? - raise FieldsParser::ParseError.new "Received invalid index while parsing nest groups: text_index: #{text_index} | regex_index: #{regex_index}" - end - - offset = raw_match.starts_with?(',') ? 1 : 0 - - opening_bracket_index = (text_index + group_name.size) + offset - closing_bracket_index = bracket_pairs[opening_bracket_index] - content_start = opening_bracket_index + 1 - - content = fields_text[content_start...closing_bracket_index] - - if content.empty? - raise FieldsParser::ParseError.new "Empty nest group at position #{content_start}" - else - content = remove_nest_groups(content) - end - - while nest_stack.size > 0 && closing_bracket_index > nest_stack[nest_stack.size - 1][:closing_bracket_index] - if nest_stack.size - nest_stack.pop - end - end - - group_name.split('/').each do |name| - nest_stack.push({ - group_name: name, - closing_bracket_index: closing_bracket_index, - }) - end - - if !content.empty? - properties = content.split(',') - - properties.each do |prop| - nest_list = nest_stack.map { |nest_prop| nest_prop[:group_name] } - - if !prop.empty? - if prop.includes?('/') - parse_single_nests(prop) { |list| nest_list += list } - else - nest_list.push prop - end - else - raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list << prop}" - end - - yield nest_list - end - end - end - end - - def self.remove_nest_groups(text : String) : String - content_bracket_pairs = get_bracket_pairs(text, false) - - content_bracket_pairs.each_key.to_a.reverse.each do |opening_bracket| - closing_bracket = content_bracket_pairs[opening_bracket] - last_comma = text.rindex(',', opening_bracket) || 0 - - text = text[0...last_comma] + text[closing_bracket + 1...text.size] - end - - return text.starts_with?(',') ? text[1...text.size] : text - end - - def self.get_bracket_pairs(text : String, recursive = true) : BracketIndex - istart = [] of Int64 - bracket_index = BracketIndex.new - - text.each_char_with_index do |char, index| - if char == '(' - istart.push(index.to_i64) - end - - if char == ')' - begin - opening = istart.pop - if recursive || (!recursive && istart.size == 0) - bracket_index[opening] = index.to_i64 - end - rescue - raise FieldsParser::ParseError.new "No matching opening parenthesis at: #{index}" - end - end - end - - if istart.size != 0 - idx = istart.pop - raise FieldsParser::ParseError.new "No matching closing parenthesis at: #{idx}" - end - - return bracket_index - end - end - - class FieldsGrouper - alias SkeletonValue = Hash(String, SkeletonValue) - - def self.create_json_skeleton(fields_text : String) : SkeletonValue - root_hash = {} of String => SkeletonValue - - FieldsParser.parse_fields(fields_text) do |nest_list| - current_item = root_hash - nest_list.each do |key| - if current_item[key]? - current_item = current_item[key] - else - current_item[key] = {} of String => SkeletonValue - current_item = current_item[key] - end - end - end - root_hash - end - - def self.create_grouped_fields_list(json_skeleton : SkeletonValue) : GroupedFieldsList - grouped_fields_list = GroupedFieldsList.new - json_skeleton.each do |key, value| - grouped_fields_list.push key - - nested_keys = create_grouped_fields_list(value) - grouped_fields_list.push nested_keys unless nested_keys.empty? - end - return grouped_fields_list - end - end - - class FilterError < Exception - end - - def self.filter(item : JSON::Any, fields_text : String, in_place : Bool = true) - skeleton = FieldsGrouper.create_json_skeleton(fields_text) - grouped_fields_list = FieldsGrouper.create_grouped_fields_list(skeleton) - filter(item, grouped_fields_list, in_place) - end - - def self.filter(item : JSON::Any, grouped_fields_list : GroupedFieldsList, in_place : Bool = true) : JSON::Any - item = item.clone unless in_place - - if !item.as_h? && !item.as_a? - raise FilterError.new "Can't filter '#{item}' by #{grouped_fields_list}" - end - - top_level_keys = Array(String).new - grouped_fields_list.each do |value| - if value.is_a? String - top_level_keys.push value - elsif value.is_a? Array - if !top_level_keys.empty? - key_to_filter = top_level_keys.last - - if item.as_h? - filter(item[key_to_filter], value, in_place: true) - elsif item.as_a? - item.as_a.each { |arr_item| filter(arr_item[key_to_filter], value, in_place: true) } - end - else - raise FilterError.new "Tried to filter while top level keys list is empty" - end - end - end - - if item.as_h? - item.as_h.select! top_level_keys - elsif item.as_a? - item.as_a.map { |value| filter(value, top_level_keys, in_place: true) } - end - - item - end -end diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index a006d602..e438e3b9 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -262,7 +262,7 @@ def get_referer(env, fallback = "/", unroll = true) end referer = referer.request_target - referer = "/" + referer.gsub(/[^\/?@&%=\-_.:,0-9a-zA-Z]/, "").lstrip("/\\") + referer = "/" + referer.gsub(/[^\/?@&%=\-_.:,*0-9a-zA-Z]/, "").lstrip("/\\") if referer == env.request.path referer = fallback diff --git a/src/invidious/helpers/webvtt.cr b/src/invidious/helpers/webvtt.cr index 56f761ed..260d250f 100644 --- a/src/invidious/helpers/webvtt.cr +++ b/src/invidious/helpers/webvtt.cr @@ -4,13 +4,23 @@ module WebVTT # A WebVTT builder generates WebVTT files private class Builder + # See https://developer.mozilla.org/en-US/docs/Web/API/WebVTT_API#cue_payload + private ESCAPE_SUBSTITUTIONS = { + '&' => "&", + '<' => "<", + '>' => ">", + '\u200E' => "‎", + '\u200F' => "‏", + '\u00A0' => " ", + } + def initialize(@io : IO) end # Writes an vtt cue with the specified time stamp and contents def cue(start_time : Time::Span, end_time : Time::Span, text : String) timestamp(start_time, end_time) - @io << text + @io << self.escape(text) @io << "\n\n" end @@ -29,6 +39,10 @@ module WebVTT @io << '.' << timestamp.milliseconds.to_s.rjust(3, '0') end + private def escape(text : String) : String + return text.gsub(ESCAPE_SUBSTITUTIONS) + end + def document(setting_fields : Hash(String, String)? = nil, &) @io << "WEBVTT\n" diff --git a/src/invidious/jobs/statistics_refresh_job.cr b/src/invidious/jobs/statistics_refresh_job.cr index 72d1ce88..66c91ad5 100644 --- a/src/invidious/jobs/statistics_refresh_job.cr +++ b/src/invidious/jobs/statistics_refresh_job.cr @@ -56,8 +56,8 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob users = STATISTICS.dig("usage", "users").as(Hash(String, Int64)) users["total"] = Invidious::Database::Statistics.count_users_total - users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_1m - users["activeMonth"] = Invidious::Database::Statistics.count_users_active_6m + users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_6m + users["activeMonth"] = Invidious::Database::Statistics.count_users_active_1m STATISTICS["metadata"] = { "updatedAt" => Time.utc.to_unix, diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 67018660..1d409c79 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -90,6 +90,7 @@ module Invidious::Routes::API::V1::Channels json.field "allowedRegions", channel.allowed_regions json.field "tabs", channel.tabs + json.field "tags", channel.tags json.field "authorVerified", channel.verified json.field "latestVideos" do diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index b42ecd1a..12942906 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -191,6 +191,8 @@ module Invidious::Routes::API::V1::Misc json.object do json.field "ucid", sub_endpoint["browseId"].as_s if sub_endpoint["browseId"]? json.field "videoId", sub_endpoint["videoId"].as_s if sub_endpoint["videoId"]? + json.field "playlistId", sub_endpoint["playlistId"].as_s if sub_endpoint["playlistId"]? + json.field "startTimeSeconds", sub_endpoint["startTimeSeconds"].as_i if sub_endpoint["startTimeSeconds"]? json.field "params", params.try &.as_s json.field "pageType", pageType end diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index 9fb283c2..2922b060 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -32,11 +32,14 @@ module Invidious::Routes::API::V1::Search begin client = HTTP::Client.new("suggestqueries-clients6.youtube.com") - url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&xssi=t&gs_ri=youtube&ds=yt" + client.before_request { |r| add_yt_headers(r) } + + url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt" response = client.get(url).body + client.close - body = JSON.parse(response[5..-1]).as_a + body = JSON.parse(response[19..-2]).as_a suggestions = body[1].as_a[0..-2] JSON.build do |json| diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 1017ac9d..9281f4dd 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -363,4 +363,47 @@ module Invidious::Routes::API::V1::Videos end end end + + def self.clips(env) + locale = env.get("preferences").as(Preferences).locale + + env.response.content_type = "application/json" + + clip_id = env.params.url["id"] + region = env.params.query["region"]? + proxy = {"1", "true"}.any? &.== env.params.query["local"]? + + response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}") + return error_json(400, "Invalid clip ID") if response["error"]? + + video_id = response.dig?("endpoint", "watchEndpoint", "videoId").try &.as_s + return error_json(400, "Invalid clip ID") if video_id.nil? + + start_time = nil + end_time = nil + clip_title = nil + + if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s + start_time, end_time, clip_title = parse_clip_parameters(params) + end + + begin + video = get_video(video_id, region: region) + rescue ex : NotFoundException + return error_json(404, ex) + rescue ex + return error_json(500, ex) + end + + return JSON.build do |json| + json.object do + json.field "startTime", start_time + json.field "endTime", end_time + json.field "clipTitle", clip_title + json.field "video" do + Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy) + end + end + end + end end diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 3d935f0a..aabe8dfc 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -275,6 +275,12 @@ module Invidious::Routes::Watch return error_template(400, "Invalid clip ID") if response["error"]? if video_id = response.dig?("endpoint", "watchEndpoint", "videoId") + if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s + start_time, end_time, _ = parse_clip_parameters(params) + env.params.query["start"] = start_time.to_s if start_time != nil + env.params.query["end"] = end_time.to_s if end_time != nil + end + return env.redirect "/watch?v=#{video_id}&#{env.params.query}" else return error_template(404, "The requested clip doesn't exist") diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index d6bd991c..ba05da19 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -235,6 +235,7 @@ module Invidious::Routing get "/api/v1/captions/:id", {{namespace}}::Videos, :captions get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations get "/api/v1/comments/:id", {{namespace}}::Videos, :comments + get "/api/v1/clips/:id", {{namespace}}::Videos, :clips # Feeds get "/api/v1/trending", {{namespace}}::Feeds, :trending diff --git a/src/invidious/videos/clip.cr b/src/invidious/videos/clip.cr new file mode 100644 index 00000000..29c57182 --- /dev/null +++ b/src/invidious/videos/clip.cr @@ -0,0 +1,22 @@ +require "json" + +# returns start_time, end_time and clip_title +def parse_clip_parameters(params) : {Float64?, Float64?, String?} + decoded_protobuf = params.try { |i| URI.decode_www_form(i) } + .try { |i| Base64.decode(i) } + .try { |i| IO::Memory.new(i) } + .try { |i| Protodec::Any.parse(i) } + + start_time = decoded_protobuf + .try(&.["50:0:embedded"]["2:1:varint"].as_i64) + .try { |i| i/1000 } + + end_time = decoded_protobuf + .try(&.["50:0:embedded"]["3:2:varint"].as_i64) + .try { |i| i/1000 } + + clip_title = decoded_protobuf + .try(&.["50:0:embedded"]["4:3:string"].as_s) + + return start_time, end_time, clip_title +end diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 77520dbe..75fe4a36 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -142,8 +142,9 @@ end def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)? LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.") - # 2AMBCgIQBg is a workaround for streaming URLs that returns a 403. - response = YoutubeAPI.player(video_id: id, params: "2AMBCgIQBg", client_config: client_config) + # CgIIAdgDAQ%3D%3D is a workaround for streaming URLs that returns a 403. + # https://github.com/LuanRT/YouTube.js/pull/624 + response = YoutubeAPI.player(video_id: id, params: "CgIIAdgDAQ%3D%3D", client_config: client_config) playability_status = response["playabilityStatus"]["status"] LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.") diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index 031b46da..6d227cfc 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -82,11 +82,19 @@ </div> <div class="video-card-row flexible"> - <div class="flex-left"><a href="/channel/<%= item.ucid %>"> - <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> - <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> - </p> - </a></div> + <div class="flex-left"> + <% if !item.ucid.to_s.empty? %> + <a href="/channel/<%= item.ucid %>"> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + </a> + <% else %> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + <% end %> + </div> </div> <% when Category %> <% else %> @@ -160,11 +168,19 @@ </div> <div class="video-card-row flexible"> - <div class="flex-left"><a href="/channel/<%= item.ucid %>"> - <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> - <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> - </p> - </a></div> + <div class="flex-left"> + <% if !item.ucid.to_s.empty? %> + <a href="/channel/<%= item.ucid %>"> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + </a> + <% else %> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + <% end %> + </div> <%= rendered "components/video-context-buttons" %> </div> diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index 77265679..9904b4fc 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -1,5 +1,9 @@ +<% + locale = env.get("preferences").as(Preferences).locale + dark_mode = env.get("preferences").as(Preferences).dark_mode +%> <!DOCTYPE html> -<html lang="<%= env.get("preferences").as(Preferences).locale %>"> +<html lang="<%= locale %>"> <head> <meta charset="utf-8"> @@ -17,19 +21,14 @@ <link rel="stylesheet" href="/css/grids-responsive-min.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/ionicons.min.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/default.css?v=<%= ASSET_COMMIT %>"> + <link rel="stylesheet" href="/css/carousel.css?v=<%= ASSET_COMMIT %>"> <script src="/js/_helpers.js?v=<%= ASSET_COMMIT %>"></script> </head> -<% - locale = env.get("preferences").as(Preferences).locale - dark_mode = env.get("preferences").as(Preferences).dark_mode -%> - <body class="<%= dark_mode.blank? ? "no" : dark_mode %>-theme"> - <span style="display:none" id="dark_mode_pref"><%= env.get("preferences").as(Preferences).dark_mode %></span> + <span style="display:none" id="dark_mode_pref"><%= dark_mode %></span> <div class="pure-g"> - <div class="pure-u-1 pure-u-md-2-24"></div> - <div class="pure-u-1 pure-u-md-20-24" id="contents"> + <div class="pure-u-1 pure-u-xl-20-24" id="contents"> <div class="pure-g navbar h-box"> <% if navbar_search %> <div class="pure-u-1 pure-u-md-4-24"> @@ -43,8 +42,8 @@ <div class="pure-u-1 pure-u-md-8-24 user-field"> <% if env.get? "user" %> <div class="pure-u-1-4"> - <a id="toggle_theme" href="/toggle_theme?referer=<%= env.get?("current_page") %>" class="pure-menu-heading"> - <% if env.get("preferences").as(Preferences).dark_mode == "dark" %> + <a id="toggle_theme" href="/toggle_theme?referer=<%= env.get?("current_page") %>" class="pure-menu-heading" title="<%= translate(locale, "toggle_theme") %>"> + <% if dark_mode == "dark" %> <i class="icon ion-ios-sunny"></i> <% else %> <i class="icon ion-ios-moon"></i> @@ -81,8 +80,8 @@ </div> <% else %> <div class="pure-u-1-3"> - <a id="toggle_theme" href="/toggle_theme?referer=<%= env.get?("current_page") %>" class="pure-menu-heading"> - <% if env.get("preferences").as(Preferences).dark_mode == "dark" %> + <a id="toggle_theme" href="/toggle_theme?referer=<%= env.get?("current_page") %>" class="pure-menu-heading" title="<%= translate(locale, "toggle_theme") %>"> + <% if dark_mode == "dark" %> <i class="icon ion-ios-sunny"></i> <% else %> <i class="icon ion-ios-moon"></i> @@ -156,7 +155,6 @@ </footer> </div> - <div class="pure-u-1 pure-u-md-2-24"></div> </div> <script src="/js/handlers.js?v=<%= ASSET_COMMIT %>"></script> <script src="/js/themes.js?v=<%= ASSET_COMMIT %>"></script> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 07474896..7a1cf2c3 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -118,7 +118,7 @@ we're going to need to do it here in order to allow for translations. link_yt_embed = URI.new(scheme: "https", host: "www.youtube.com", path: "/embed/#{video.id}") if !plid.nil? && !continuation.nil? - link_yt_param = URI::Params{"plid" => [plid], "index" => [continuation.to_s]} + link_yt_param = URI::Params{"list" => [plid], "index" => [continuation.to_s]} link_yt_watch = IV::HttpServer::Utils.add_params_to_url(link_yt_watch, link_yt_param) link_yt_embed = IV::HttpServer::Utils.add_params_to_url(link_yt_embed, link_yt_param) end @@ -346,7 +346,7 @@ we're going to need to do it here in order to allow for translations. <h5 class="pure-g"> <div class="pure-u-14-24"> - <% if rv["ucid"]? %> + <% if !rv["ucid"].empty? %> <b style="width:100%"><a href="/channel/<%= rv["ucid"] %>"><%= rv["author"]? %><% if rv["author_verified"]? == "true" %> <i class="icon ion ion-md-checkmark-circle"></i><% end %></a></b> <% else %> <b style="width:100%"><%= rv["author"]? %><% if rv["author_verified"]? == "true" %> <i class="icon ion ion-md-checkmark-circle"></i><% end %></b> diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 56325cf7..0e72957e 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -822,9 +822,9 @@ module HelperExtractors end # Retrieves the ID required for querying the InnerTube browse endpoint. - # Raises when it's unable to do so + # Returns an empty string when it's unable to do so def self.get_browse_id(container) - return container.dig("navigationEndpoint", "browseEndpoint", "browseId").as_s + return container.dig?("navigationEndpoint", "browseEndpoint", "browseId").try &.as_s || "" end end diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index a5e621f2..9e0631f6 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -7,17 +7,18 @@ module YoutubeAPI private DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" - private ANDROID_APP_VERSION = "18.20.38" - # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1308 - private ANDROID_USER_AGENT = "com.google.android.youtube/18.20.38 (Linux; U; Android 12; US) gzip" + # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history + private ANDROID_APP_VERSION = "19.09.36" + private ANDROID_USER_AGENT = "com.google.android.youtube/19.09.36 (Linux; U; Android 12; US) gzip" private ANDROID_SDK_VERSION = 31_i64 private ANDROID_VERSION = "12" - private IOS_APP_VERSION = "18.21.3" - # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1330 - private IOS_USER_AGENT = "com.google.ios.youtube/18.21.3 (iPhone14,5; U; CPU iOS 15_6 like Mac OS X;)" - # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1224 - private IOS_VERSION = "15.6.0.19G71" + # For Apple device names, see https://gist.github.com/adamawolf/3048717 + # For iOS versions, see https://en.wikipedia.org/wiki/IOS_version_history#Releases, + # then go to the dedicated article of the major version you want. + private IOS_APP_VERSION = "19.09.3" + private IOS_USER_AGENT = "com.google.ios.youtube/19.09.3 (iPhone14,5; U; CPU iOS 17_4 like Mac OS X;)" + private IOS_VERSION = "17.4.0.21E219" # Major.Minor.Patch.Build private WINDOWS_VERSION = "10.0" @@ -45,7 +46,7 @@ module YoutubeAPI ClientType::Web => { name: "WEB", name_proto: "1", - version: "2.20230602.01.00", + version: "2.20240304.00.00", api_key: DEFAULT_API_KEY, screen: "WATCH_FULL_SCREEN", os_name: "Windows", @@ -55,7 +56,7 @@ module YoutubeAPI ClientType::WebEmbeddedPlayer => { name: "WEB_EMBEDDED_PLAYER", name_proto: "56", - version: "1.20220803.01.00", + version: "1.20240303.00.00", api_key: DEFAULT_API_KEY, screen: "EMBED", os_name: "Windows", @@ -65,7 +66,7 @@ module YoutubeAPI ClientType::WebMobile => { name: "MWEB", name_proto: "2", - version: "2.20230531.05.00", + version: "2.20240304.08.00", api_key: DEFAULT_API_KEY, os_name: "Android", os_version: ANDROID_VERSION, @@ -74,7 +75,7 @@ module YoutubeAPI ClientType::WebScreenEmbed => { name: "WEB", name_proto: "1", - version: "2.20220804.00.00", + version: "2.20240304.00.00", api_key: DEFAULT_API_KEY, screen: "EMBED", os_name: "Windows", @@ -99,7 +100,7 @@ module YoutubeAPI name: "ANDROID_EMBEDDED_PLAYER", name_proto: "55", version: ANDROID_APP_VERSION, - api_key: DEFAULT_API_KEY, + api_key: "AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw", }, ClientType::AndroidScreenEmbed => { name: "ANDROID", @@ -143,9 +144,9 @@ module YoutubeAPI ClientType::IOSMusic => { name: "IOS_MUSIC", name_proto: "26", - version: "5.21", + version: "6.42", api_key: "AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s", - user_agent: "com.google.ios.youtubemusic/5.21 (iPhone14,5; U; CPU iOS 15_6 like Mac OS X;)", + user_agent: "com.google.ios.youtubemusic/6.42 (iPhone14,5; U; CPU iOS 17_4 like Mac OS X;)", device_make: "Apple", device_model: "iPhone14,5", os_name: "iPhone", @@ -158,7 +159,7 @@ module YoutubeAPI ClientType::TvHtml5 => { name: "TVHTML5", name_proto: "7", - version: "7.20220325", + version: "7.20240304.10.00", api_key: DEFAULT_API_KEY, }, ClientType::TvHtml5ScreenEmbed => { |