diff options
Diffstat (limited to 'src')
25 files changed, 204 insertions, 152 deletions
diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index b5a27667..edaf5c12 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -72,6 +72,7 @@ def get_about_info(ucid, locale) : AboutChannel # Raises a KeyError on failure. banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? + banners ||= initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "banner", "imageBannerViewModel", "image", "sources") banner = banners.try &.[-1]?.try &.["url"].as_s? # if banner.includes? "channels/c4/default_banner" @@ -147,9 +148,17 @@ def get_about_info(ucid, locale) : AboutChannel end end - sub_count = initdata - .dig?("header", "c4TabbedHeaderRenderer", "subscriberCountText", "simpleText").try &.as_s? - .try { |text| short_text_to_number(text.split(" ")[0]).to_i32 } || 0 + sub_count = 0 + + if (metadata_rows = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows").try &.as_a) + metadata_rows.each do |row| + metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") } + if !metadata_part.nil? + sub_count = short_text_to_number(metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32 + end + break if sub_count != 0 + end + end AboutChannel.new( ucid: ucid, diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 351790d7..6cc30142 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -1,4 +1,4 @@ -def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) +def produce_channel_content_continuation(ucid, content_type, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) object_inner_2 = { "2:0:embedded" => { "1:0:varint" => 0_i64, @@ -16,6 +16,13 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } + content_type_numerical = + case content_type + when "videos" then 15 + when "livestreams" then 14 + else 15 # Fallback to "videos" + end + sort_by_numerical = case sort_by when "newest" then 1_i64 @@ -27,7 +34,7 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so object_inner_1 = { "110:embedded" => { "3:embedded" => { - "15:embedded" => { + "#{content_type_numerical}:embedded" => { "1:embedded" => { "1:string" => object_inner_2_encoded, }, @@ -62,6 +69,10 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so return continuation end +def make_initial_content_ctoken(ucid, content_type, sort_by) : String + return produce_channel_content_continuation(ucid, content_type, sort_by: sort_by) +end + module Invidious::Channel::Tabs extend self @@ -69,10 +80,6 @@ module Invidious::Channel::Tabs # Regular videos # ------------------- - def make_initial_video_ctoken(ucid, sort_by) : String - return produce_channel_videos_continuation(ucid, sort_by: sort_by) - end - # Wrapper for AboutChannel, as we still need to call get_videos with # an author name and ucid directly (e.g in RSS feeds). # TODO: figure out how to get rid of that @@ -94,7 +101,7 @@ module Invidious::Channel::Tabs end def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest") - continuation ||= make_initial_video_ctoken(ucid, sort_by) + continuation ||= make_initial_content_ctoken(ucid, "videos", sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, author, ucid) @@ -138,21 +145,18 @@ module Invidious::Channel::Tabs # Livestreams # ------------------- - def get_livestreams(channel : AboutChannel, continuation : String? = nil) - if continuation.nil? - # EgdzdHJlYW1z8gYECgJ6AA%3D%3D is the protobuf object to load "streams" - initial_data = YoutubeAPI.browse(channel.ucid, params: "EgdzdHJlYW1z8gYECgJ6AA%3D%3D") - else - initial_data = YoutubeAPI.browse(continuation: continuation) - end + def get_livestreams(channel : AboutChannel, continuation : String? = nil, sort_by = "newest") + continuation ||= make_initial_content_ctoken(channel.ucid, "livestreams", sort_by) + + initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, channel.author, channel.ucid) end - def get_60_livestreams(channel : AboutChannel, continuation : String? = nil) + def get_60_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") if continuation.nil? - # Fetch the first "page" of streams - items, next_continuation = get_livestreams(channel) + # Fetch the first "page" of stream + items, next_continuation = get_livestreams(channel, sort_by: sort_by) else # Fetch a "page" of streams using the given continuation token items, next_continuation = get_livestreams(channel, continuation: continuation) diff --git a/src/invidious/helpers/i18next.cr b/src/invidious/helpers/i18next.cr index 9f4077e1..684e6d14 100644 --- a/src/invidious/helpers/i18next.cr +++ b/src/invidious/helpers/i18next.cr @@ -95,7 +95,6 @@ module I18next::Plurals "hr" => PluralForms::Special_Hungarian_Serbian, "it" => PluralForms::Special_Spanish_Italian, "pt" => PluralForms::Special_French_Portuguese, - "pt" => PluralForms::Special_French_Portuguese, "sr" => PluralForms::Special_Hungarian_Serbian, } @@ -189,7 +188,7 @@ module I18next::Plurals # Emulate the `rule.numbers.size == 2 && rule.numbers[0] == 1` check # from original i18next code - private def is_simple_plural(form : PluralForms) : Bool + private def simple_plural?(form : PluralForms) : Bool case form when .single_gt_one? then return true when .single_not_one? then return true @@ -211,7 +210,7 @@ module I18next::Plurals idx = SuffixIndex.get_index(plural_form, count) # Simple plurals are handled differently in all versions (but v4) - if @simplify_plural_suffix && is_simple_plural(plural_form) + if @simplify_plural_suffix && simple_plural?(plural_form) return (idx == 1) ? "_plural" : "" end @@ -262,9 +261,9 @@ module I18next::Plurals when .special_hebrew? then return special_hebrew(count) when .special_odia? then return special_odia(count) # Mixed v3/v4 forms - when .special_spanish_italian? then return special_cldr_Spanish_Italian(count) - when .special_french_portuguese? then return special_cldr_French_Portuguese(count) - when .special_hungarian_serbian? then return special_cldr_Hungarian_Serbian(count) + when .special_spanish_italian? then return special_cldr_spanish_italian(count) + when .special_french_portuguese? then return special_cldr_french_portuguese(count) + when .special_hungarian_serbian? then return special_cldr_hungarian_serbian(count) else # default, if nothing matched above return 0_u8 @@ -535,7 +534,7 @@ module I18next::Plurals # # This rule is mostly compliant to CLDR v42 # - def self.special_cldr_Spanish_Italian(count : Int) : UInt8 + def self.special_cldr_spanish_italian(count : Int) : UInt8 return 0_u8 if (count == 1) # one return 1_u8 if (count != 0 && count % 1_000_000 == 0) # many return 2_u8 # other @@ -545,7 +544,7 @@ module I18next::Plurals # # This rule is mostly compliant to CLDR v42 # - def self.special_cldr_French_Portuguese(count : Int) : UInt8 + def self.special_cldr_french_portuguese(count : Int) : UInt8 return 0_u8 if (count == 0 || count == 1) # one return 1_u8 if (count % 1_000_000 == 0) # many return 2_u8 # other @@ -555,7 +554,7 @@ module I18next::Plurals # # This rule is mostly compliant to CLDR v42 # - def self.special_cldr_Hungarian_Serbian(count : Int) : UInt8 + def self.special_cldr_hungarian_serbian(count : Int) : UInt8 n_mod_10 = count % 10 n_mod_100 = count % 100 diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr index e2e50905..b443073e 100644 --- a/src/invidious/helpers/logger.cr +++ b/src/invidious/helpers/logger.cr @@ -34,24 +34,11 @@ class Invidious::LogHandler < Kemal::BaseLogHandler context end - def puts(message : String) - @io << message << '\n' - @io.flush - end - def write(message : String) @io << message @io.flush end - def set_log_level(level : String) - @level = LogLevel.parse(level) - end - - def set_log_level(level : LogLevel) - @level = level - end - {% for level in %w(trace debug info warn error fatal) %} def {{level.id}}(message : String) if LogLevel::{{level.id.capitalize}} >= @level diff --git a/src/invidious/http_server/utils.cr b/src/invidious/http_server/utils.cr index 222dfc4a..623a9177 100644 --- a/src/invidious/http_server/utils.cr +++ b/src/invidious/http_server/utils.cr @@ -11,11 +11,12 @@ module Invidious::HttpServer params = url.query_params params["host"] = url.host.not_nil! # Should never be nil, in theory params["region"] = region if !region.nil? + url.query_params = params if absolute - return "#{HOST_URL}#{url.request_target}?#{params}" + return "#{HOST_URL}#{url.request_target}" else - return "#{url.request_target}?#{params}" + return url.request_target end end diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index 0dced80b..59714828 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -114,25 +114,31 @@ module Invidious::JSONify::APIv1 json.field "projectionType", fmt["projectionType"] - if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) - fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 + height = fmt["height"]?.try &.as_i + width = fmt["width"]?.try &.as_i + + fps = fmt["fps"]?.try &.as_i + + if fps json.field "fps", fps - json.field "container", fmt_info["ext"] - json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] + end - if fmt_info["height"]? - json.field "resolution", "#{fmt_info["height"]}p" + if height && width + json.field "size", "#{width}x#{height}" + json.field "resolution", "#{height}p" - quality_label = "#{fmt_info["height"]}p" - if fps > 30 - quality_label += "60" - end - json.field "qualityLabel", quality_label + quality_label = "#{width > height ? height : width}p" - if fmt_info["width"]? - json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" - end + if fps && fps > 30 + quality_label += fps.to_s end + + json.field "qualityLabel", quality_label + end + + if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) + json.field "container", fmt_info["ext"] + json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] end # Livestream chunk infos @@ -163,26 +169,31 @@ module Invidious::JSONify::APIv1 json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]? - fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) - if fmt_info - fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 + height = fmt["height"]?.try &.as_i + width = fmt["width"]?.try &.as_i + + fps = fmt["fps"]?.try &.as_i + + if fps json.field "fps", fps - json.field "container", fmt_info["ext"] - json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] + end - if fmt_info["height"]? - json.field "resolution", "#{fmt_info["height"]}p" + if height && width + json.field "size", "#{width}x#{height}" + json.field "resolution", "#{height}p" - quality_label = "#{fmt_info["height"]}p" - if fps > 30 - quality_label += "60" - end - json.field "qualityLabel", quality_label + quality_label = "#{width > height ? height : width}p" - if fmt_info["width"]? - json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" - end + if fps && fps > 30 + quality_label += fps.to_s end + + json.field "qualityLabel", quality_label + end + + if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) + json.field "container", fmt_info["ext"] + json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] end end end diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 955e0855..a227f794 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -366,6 +366,8 @@ def fetch_playlist(plid : String) if text.includes? "video" video_count = text.gsub(/\D/, "").to_i? || 0 + elsif text.includes? "episode" + video_count = text.gsub(/\D/, "").to_i? || 0 elsif text.includes? "view" views = text.gsub(/\D/, "").to_i64? || 0_i64 else diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr index 9d930841..dd65e7a6 100644 --- a/src/invidious/routes/account.cr +++ b/src/invidious/routes/account.cr @@ -53,7 +53,7 @@ module Invidious::Routes::Account return error_template(401, "Password is a required field") end - new_passwords = env.params.body.select { |k, v| k.match(/^new_password\[\d+\]$/) }.map { |k, v| v } + new_passwords = env.params.body.select { |k, _| k.match(/^new_password\[\d+\]$/) }.map { |_, v| v } if new_passwords.size <= 1 || new_passwords.uniq.size != 1 return error_template(400, "New passwords must match") @@ -240,7 +240,7 @@ module Invidious::Routes::Account return error_template(400, ex) end - scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v } + scopes = env.params.body.select { |k, _| k.match(/^scopes\[\d+\]$/) }.map { |_, v| v } callback_url = env.params.body["callbackUrl"]? expire = env.params.body["expire"]?.try &.to_i? diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 7faf200a..43a5c35b 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -208,11 +208,12 @@ module Invidious::Routes::API::V1::Channels get_channel() # Retrieve continuation from URL parameters + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" continuation = env.params.query["continuation"]? begin videos, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation + channel, continuation: continuation, sort_by: sort_by ) rescue ex return error_json(500, ex) diff --git a/src/invidious/routes/api/v1/feeds.cr b/src/invidious/routes/api/v1/feeds.cr index 41865f34..fea2993c 100644 --- a/src/invidious/routes/api/v1/feeds.cr +++ b/src/invidious/routes/api/v1/feeds.cr @@ -31,7 +31,7 @@ module Invidious::Routes::API::V1::Feeds if !CONFIG.popular_enabled error_message = {"error" => "Administrator has disabled this endpoint."}.to_json - haltf env, 400, error_message + haltf env, 403, error_message end JSON.build do |json| diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index 12942906..0c79692d 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -74,7 +74,9 @@ module Invidious::Routes::API::V1::Misc response = playlist.to_json(offset, video_id: video_id) json_response = JSON.parse(response) - if json_response["videos"].as_a[0]["index"] != offset + if json_response["videos"].as_a.empty? + json_response = JSON.parse(response) + elsif json_response["videos"].as_a[0]["index"] != offset offset = json_response["videos"].as_a[0]["index"].as_i lookback = offset < 50 ? offset : 50 response = playlist.to_json(offset - lookback) diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 9281f4dd..42282f44 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -89,9 +89,14 @@ module Invidious::Routes::API::V1::Videos if CONFIG.use_innertube_for_captions params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated) - initial_data = YoutubeAPI.get_transcript(params) - webvtt = Invidious::Videos::Transcript.convert_transcripts_to_vtt(initial_data, caption.language_code) + transcript = Invidious::Videos::Transcript.from_raw( + YoutubeAPI.get_transcript(params), + caption.language_code, + caption.auto_generated + ) + + webvtt = transcript.to_vtt else # Timedtext API handling url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target @@ -136,7 +141,11 @@ module Invidious::Routes::API::V1::Videos end end else - webvtt = YT_POOL.client &.get("#{url}&fmt=vtt").body + uri = URI.parse(url) + query_params = uri.query_params + query_params["fmt"] = "vtt" + uri.query_params = query_params + webvtt = YT_POOL.client &.get(uri.request_target).body if webvtt.starts_with?("<?xml") webvtt = caption.timedtext_to_vtt(webvtt) @@ -210,7 +219,7 @@ module Invidious::Routes::API::V1::Videos storyboard[:storyboard_count].times do |i| url = storyboard[:url] - authority = /(i\d?).ytimg.com/.match(url).not_nil![1]? + authority = /(i\d?).ytimg.com/.match!(url)[1]? url = url.gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "") url = "#{HOST_URL}/sb/#{authority}/#{url}" @@ -245,7 +254,7 @@ module Invidious::Routes::API::V1::Videos if CONFIG.cache_annotations && (cached_annotation = Invidious::Database::Annotations.select(id)) annotations = cached_annotation.annotations else - index = CHARS_SAFE.index(id[0]).not_nil!.to_s.rjust(2, '0') + index = CHARS_SAFE.index!(id[0]).to_s.rjust(2, '0') # IA doesn't handle leading hyphens, # so we use https://archive.org/details/youtubeannotations_64 diff --git a/src/invidious/routes/before_all.cr b/src/invidious/routes/before_all.cr index 396840a4..5695dee9 100644 --- a/src/invidious/routes/before_all.cr +++ b/src/invidious/routes/before_all.cr @@ -30,7 +30,7 @@ module Invidious::Routes::BeforeAll # Only allow the pages at /embed/* to be embedded if env.request.resource.starts_with?("/embed") - frame_ancestors = "'self' http: https:" + frame_ancestors = "'self' file: http: https:" else frame_ancestors = "'none'" end diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index fea49bbe..360af2cd 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -81,13 +81,12 @@ module Invidious::Routes::Channels return env.redirect "/channel/#{channel.ucid}" end - # TODO: support sort option for livestreams - sort_by = "" - sort_options = [] of String + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" + sort_options = {"newest", "oldest", "popular"} # Fetch items and continuation token items, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation + channel, continuation: continuation, sort_by: sort_by ) selected_tab = Frontend::ChannelPage::TabsAvailable::Streams diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index 112535bd..05bc2714 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -214,7 +214,7 @@ module Invidious::Routes::PreferencesRoute statistics_enabled ||= "off" CONFIG.statistics_enabled = statistics_enabled == "on" - CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.try &.as(String) + CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.presence File.write("config/config.yml", CONFIG.to_yaml) end diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 108f2ccc..29b59293 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -182,7 +182,7 @@ struct Invidious::User if is_opml?(type, extension) subscriptions = XML.parse(body) user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| - channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] + channel["xmlUrl"].match!(/UC[a-zA-Z0-9_-]{22}/)[0] end elsif extension == "json" || type == "application/json" subscriptions = JSON.parse(body) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index c218b4ef..cdfca02c 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -250,7 +250,7 @@ struct Video end def genre_url : String? - info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil + info["genreUcid"].try &.as_s? ? "/channel/#{info["genreUcid"]}" : nil end def is_vr : Bool? diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 0e1a947c..4bdb2512 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -424,7 +424,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil), # Video metadata "genre" => JSON::Any.new(genre.try &.as_s || ""), - "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""), + "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s?), "license" => JSON::Any.new(license.try &.as_s || ""), # Music section "music" => JSON.parse(music_list.to_json), diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr index dac00eea..9cd064c5 100644 --- a/src/invidious/videos/transcript.cr +++ b/src/invidious/videos/transcript.cr @@ -1,8 +1,26 @@ module Invidious::Videos - # Namespace for methods primarily relating to Transcripts - module Transcript - record TranscriptLine, start_ms : Time::Span, end_ms : Time::Span, line : String + # A `Transcripts` struct encapsulates a sequence of lines that together forms the whole transcript for a given YouTube video. + # These lines can be categorized into two types: section headings and regular lines representing content from the video. + struct Transcript + # Types + record HeadingLine, start_ms : Time::Span, end_ms : Time::Span, line : String + record RegularLine, start_ms : Time::Span, end_ms : Time::Span, line : String + alias TranscriptLine = HeadingLine | RegularLine + property lines : Array(TranscriptLine) + + property language_code : String + property auto_generated : Bool + + # User friendly label for the current transcript. + # Example: "English (auto-generated)" + property label : String + + # Initializes a new Transcript struct with the contents and associated metadata describing it + def initialize(@lines : Array(TranscriptLine), @language_code : String, @auto_generated : Bool, @label : String) + end + + # Generates a protobuf string to fetch the requested transcript from YouTube def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String kind = auto_generated ? "asr" : "" @@ -30,48 +48,79 @@ module Invidious::Videos return params end - def self.convert_transcripts_to_vtt(initial_data : Hash(String, JSON::Any), target_language : String) : String - # Convert into array of TranscriptLine - lines = self.parse(initial_data) + # Constructs a Transcripts struct from the initial YouTube response + def self.from_raw(initial_data : Hash(String, JSON::Any), language_code : String, auto_generated : Bool) + transcript_panel = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", + "content", "transcriptSearchPanelRenderer") - settings_field = { - "Kind" => "captions", - "Language" => target_language, - } + segment_list = transcript_panel.dig("body", "transcriptSegmentListRenderer") - # Taken from Invidious::Videos::Captions::Metadata.timedtext_to_vtt() - vtt = WebVTT.build(settings_field) do |vtt| - lines.each do |line| - vtt.cue(line.start_ms, line.end_ms, line.line) - end + if !segment_list["initialSegments"]? + raise NotFoundException.new("Requested transcript does not exist") end - return vtt - end + # Extract user-friendly label for the current transcript + + footer_language_menu = transcript_panel.dig?( + "footer", "transcriptFooterRenderer", "languageMenu", "sortFilterSubMenuRenderer", "subMenuItems" + ) - private def self.parse(initial_data : Hash(String, JSON::Any)) - body = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", - "content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer", - "initialSegments").as_a + if footer_language_menu + label = footer_language_menu.as_a.select(&.["selected"].as_bool)[0]["title"].as_s + else + label = language_code + end + + # Extract transcript lines + + initial_segments = segment_list["initialSegments"].as_a lines = [] of TranscriptLine - body.each do |line| - # Transcript section headers. They are not apart of the captions and as such we can safely skip them. - if line.as_h.has_key?("transcriptSectionHeaderRenderer") - next + + initial_segments.each do |line| + if unpacked_line = line["transcriptSectionHeaderRenderer"]? + line_type = HeadingLine + else + unpacked_line = line["transcriptSegmentRenderer"] + line_type = RegularLine end - line = line["transcriptSegmentRenderer"] + start_ms = unpacked_line["startMs"].as_s.to_i.millisecond + end_ms = unpacked_line["endMs"].as_s.to_i.millisecond + text = extract_text(unpacked_line["snippet"]) || "" + + lines << line_type.new(start_ms, end_ms, text) + end + + return Transcript.new( + lines: lines, + language_code: language_code, + auto_generated: auto_generated, + label: label + ) + end - start_ms = line["startMs"].as_s.to_i.millisecond - end_ms = line["endMs"].as_s.to_i.millisecond + # Converts transcript lines to a WebVTT file + # + # This is used within Invidious to replace subtitles + # as to workaround YouTube's rate-limited timedtext endpoint. + def to_vtt + settings_field = { + "Kind" => "captions", + "Language" => @language_code, + } - text = extract_text(line["snippet"]) || "" + vtt = WebVTT.build(settings_field) do |vtt| + @lines.each do |line| + # Section headers are excluded from the VTT conversion as to + # match the regular captions returned from YouTube as much as possible + next if line.is_a? HeadingLine - lines << TranscriptLine.new(start_ms, end_ms, text) + vtt.cue(line.start_ms, line.end_ms, line.line) + end end - return lines + return vtt end end end diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr index 09df106d..a84e44bc 100644 --- a/src/invidious/views/channel.ecr +++ b/src/invidious/views/channel.ecr @@ -30,13 +30,13 @@ <meta property="og:site_name" content="Invidious"> <meta property="og:url" content="<%= HOST_URL %>/channel/<%= ucid %>"> <meta property="og:title" content="<%= author %>"> -<meta property="og:image" content="/ggpht<%= channel_profile_pic %>"> +<meta property="og:image" content="<%= HOST_URL %>/ggpht<%= channel_profile_pic %>"> <meta property="og:description" content="<%= channel.description %>"> <meta name="twitter:card" content="summary"> <meta name="twitter:url" content="<%= HOST_URL %>/channel/<%= ucid %>"> <meta name="twitter:title" content="<%= author %>"> <meta name="twitter:description" content="<%= channel.description %>"> -<meta name="twitter:image" content="/ggpht<%= channel_profile_pic %>"> +<meta name="twitter:image" content="<%= HOST_URL %>/ggpht<%= channel_profile_pic %>"> <link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/channel/<%= ucid %>" /> <%- end -%> diff --git a/src/invidious/views/user/preferences.ecr b/src/invidious/views/user/preferences.ecr index 55349c5a..b89c73ca 100644 --- a/src/invidious/views/user/preferences.ecr +++ b/src/invidious/views/user/preferences.ecr @@ -310,7 +310,7 @@ <div class="pure-control-group"> <label for="modified_source_code_url"><%= translate(locale, "adminprefs_modified_source_code_url_label") %></label> - <input name="modified_source_code_url" id="modified_source_code_url" type="input" <% if CONFIG.modified_source_code_url %>checked<% end %>> + <input name="modified_source_code_url" id="modified_source_code_url" type="url" value="<%= CONFIG.modified_source_code_url %>"> </div> <% end %> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index fd9e1592..36679bce 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -10,7 +10,7 @@ <meta property="og:site_name" content="<%= author %> | Invidious"> <meta property="og:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>"> <meta property="og:title" content="<%= title %>"> -<meta property="og:image" content="/vi/<%= video.id %>/maxres.jpg"> +<meta property="og:image" content="<%= HOST_URL %>/vi/<%= video.id %>/maxres.jpg"> <meta property="og:description" content="<%= HTML.escape(video.short_description) %>"> <meta property="og:type" content="video.other"> <meta property="og:video:url" content="<%= HOST_URL %>/embed/<%= video.id %>"> diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index d3dbcc0e..0ac785e6 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -24,7 +24,7 @@ struct YoutubeConnectionPool @pool = build_pool() end - def client(&block) + def client(&) conn = pool.checkout begin response = yield conn @@ -69,7 +69,7 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false) return client end -def make_client(url : URI, region = nil, force_resolve : Bool = false, &block) +def make_client(url : URI, region = nil, force_resolve : Bool = false, &) client = make_client(url, region, force_resolve) begin yield client diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 0e72957e..57a5dc3d 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -856,7 +856,7 @@ end # # This function yields the container so that items can be parsed separately. # -def extract_items(initial_data : InitialData, &block) +def extract_items(initial_data : InitialData, &) if unpackaged_data = initial_data["contents"]?.try &.as_h elsif unpackaged_data = initial_data["response"]?.try &.as_h elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 1).try &.as_h diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 727ce9a3..c8b037c8 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -5,9 +5,6 @@ module YoutubeAPI extend self - private DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" - private ANDROID_API_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w" - # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history private ANDROID_APP_VERSION = "19.14.42" private ANDROID_USER_AGENT = "com.google.android.youtube/19.14.42 (Linux; U; Android 12; US) gzip" @@ -52,7 +49,6 @@ module YoutubeAPI name: "WEB", name_proto: "1", version: "2.20240304.00.00", - api_key: DEFAULT_API_KEY, screen: "WATCH_FULL_SCREEN", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -62,7 +58,6 @@ module YoutubeAPI name: "WEB_EMBEDDED_PLAYER", name_proto: "56", version: "1.20240303.00.00", - api_key: DEFAULT_API_KEY, screen: "EMBED", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -72,7 +67,6 @@ module YoutubeAPI name: "MWEB", name_proto: "2", version: "2.20240304.08.00", - api_key: DEFAULT_API_KEY, os_name: "Android", os_version: ANDROID_VERSION, platform: "MOBILE", @@ -81,7 +75,6 @@ module YoutubeAPI name: "WEB", name_proto: "1", version: "2.20240304.00.00", - api_key: DEFAULT_API_KEY, screen: "EMBED", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -94,7 +87,6 @@ module YoutubeAPI name: "ANDROID", name_proto: "3", version: ANDROID_APP_VERSION, - api_key: ANDROID_API_KEY, android_sdk_version: ANDROID_SDK_VERSION, user_agent: ANDROID_USER_AGENT, os_name: "Android", @@ -105,13 +97,11 @@ module YoutubeAPI name: "ANDROID_EMBEDDED_PLAYER", name_proto: "55", version: ANDROID_APP_VERSION, - api_key: "AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw", }, ClientType::AndroidScreenEmbed => { name: "ANDROID", name_proto: "3", version: ANDROID_APP_VERSION, - api_key: DEFAULT_API_KEY, screen: "EMBED", android_sdk_version: ANDROID_SDK_VERSION, user_agent: ANDROID_USER_AGENT, @@ -123,7 +113,6 @@ module YoutubeAPI name: "ANDROID_TESTSUITE", name_proto: "30", version: ANDROID_TS_APP_VERSION, - api_key: ANDROID_API_KEY, android_sdk_version: ANDROID_SDK_VERSION, user_agent: ANDROID_TS_USER_AGENT, os_name: "Android", @@ -137,7 +126,6 @@ module YoutubeAPI name: "IOS", name_proto: "5", version: IOS_APP_VERSION, - api_key: "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc", user_agent: IOS_USER_AGENT, device_make: "Apple", device_model: "iPhone14,5", @@ -149,7 +137,6 @@ module YoutubeAPI name: "IOS_MESSAGES_EXTENSION", name_proto: "66", version: IOS_APP_VERSION, - api_key: DEFAULT_API_KEY, user_agent: IOS_USER_AGENT, device_make: "Apple", device_model: "iPhone14,5", @@ -161,7 +148,6 @@ module YoutubeAPI name: "IOS_MUSIC", name_proto: "26", version: "6.42", - api_key: "AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s", user_agent: "com.google.ios.youtubemusic/6.42 (iPhone14,5; U; CPU iOS 17_4 like Mac OS X;)", device_make: "Apple", device_model: "iPhone14,5", @@ -176,13 +162,11 @@ module YoutubeAPI name: "TVHTML5", name_proto: "7", version: "7.20240304.10.00", - api_key: DEFAULT_API_KEY, }, ClientType::TvHtml5ScreenEmbed => { name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER", name_proto: "85", version: "2.0", - api_key: DEFAULT_API_KEY, screen: "EMBED", }, } @@ -238,11 +222,6 @@ module YoutubeAPI end # :ditto: - def api_key : String - HARDCODED_CLIENTS[@client_type][:api_key] - end - - # :ditto: def screen : String HARDCODED_CLIENTS[@client_type][:screen]? || "" end @@ -606,7 +585,7 @@ module YoutubeAPI client_config ||= DEFAULT_CLIENT_CONFIG # Query parameters - url = "#{endpoint}?key=#{client_config.api_key}&prettyPrint=false" + url = "#{endpoint}?prettyPrint=false" headers = HTTP::Headers{ "Content-Type" => "application/json; charset=UTF-8", |