diff options
Diffstat (limited to 'src')
36 files changed, 813 insertions, 213 deletions
diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index 0054f8f2..8b60a728 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -18,8 +18,8 @@ record AboutChannel, def get_about_info(ucid, locale) : AboutChannel begin - # "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"} - initdata = YoutubeAPI.browse(browse_id: ucid, params: "EgVhYm91dA==") + # Fetch channel information from channel home page + initdata = YoutubeAPI.browse(browse_id: ucid, params: "") rescue raise InfoException.new("Could not get channel info.") end diff --git a/src/invidious/channels/community.cr b/src/invidious/channels/community.cr index 791f1641..49ffd990 100644 --- a/src/invidious/channels/community.cr +++ b/src/invidious/channels/community.cr @@ -24,7 +24,33 @@ def fetch_channel_community(ucid, cursor, locale, format, thin_mode) return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode) end -def extract_channel_community(items, *, ucid, locale, format, thin_mode) +def fetch_channel_community_post(ucid, post_id, locale, format, thin_mode) + object = { + "2:string" => "community", + "25:embedded" => { + "22:string" => post_id.to_s, + }, + "45:embedded" => { + "2:varint" => 1_i64, + "3:varint" => 1_i64, + }, + } + params = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + initial_data = YoutubeAPI.browse(ucid, params: params) + + items = [] of JSON::Any + extract_items(initial_data) do |item| + items << item + end + + return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode, is_single_post: true) +end + +def extract_channel_community(items, *, ucid, locale, format, thin_mode, is_single_post : Bool = false) if message = items[0]["messageRenderer"]? error_message = (message["text"]["simpleText"]? || message["text"]["runs"]?.try &.[0]?.try &.["text"]?) @@ -39,6 +65,9 @@ def extract_channel_community(items, *, ucid, locale, format, thin_mode) response = JSON.build do |json| json.object do json.field "authorId", ucid + if is_single_post + json.field "singlePost", true + end json.field "comments" do json.array do items.each do |post| @@ -240,8 +269,10 @@ def extract_channel_community(items, *, ucid, locale, format, thin_mode) end end end - if cont = items.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token") - json.field "continuation", extract_channel_community_cursor(cont.as_s) + if !is_single_post + if cont = items.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token") + json.field "continuation", extract_channel_community_cursor(cont.as_s) + end end end end diff --git a/src/invidious/comments/youtube.cr b/src/invidious/comments/youtube.cr index 1ba1b534..185d8e43 100644 --- a/src/invidious/comments/youtube.cr +++ b/src/invidious/comments/youtube.cr @@ -13,6 +13,51 @@ module Invidious::Comments client_config = YoutubeAPI::ClientConfig.new(region: region) response = YoutubeAPI.next(continuation: ctoken, client_config: client_config) + return parse_youtube(id, response, format, locale, thin_mode, sort_by) + end + + def fetch_community_post_comments(ucid, post_id) + object = { + "2:string" => "community", + "25:embedded" => { + "22:string" => post_id, + }, + "45:embedded" => { + "2:varint" => 1_i64, + "3:varint" => 1_i64, + }, + "53:embedded" => { + "4:embedded" => { + "6:varint" => 0_i64, + "27:varint" => 1_i64, + "29:string" => post_id, + "30:string" => ucid, + }, + "8:string" => "comments-section", + }, + } + + object_parsed = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + + object2 = { + "80226972:embedded" => { + "2:string" => ucid, + "3:string" => object_parsed, + }, + } + + continuation = object2.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + initial_data = YoutubeAPI.browse(continuation: continuation) + return initial_data + end + + def parse_youtube(id, response, format, locale, thin_mode, sort_by = "top", isPost = false) contents = nil if on_response_received_endpoints = response["onResponseReceivedEndpoints"]? @@ -68,7 +113,11 @@ module Invidious::Comments json.field "commentCount", comment_count end - json.field "videoId", id + if isPost + json.field "postId", id + else + json.field "videoId", id + end json.field "comments" do json.array do diff --git a/src/invidious/config.cr b/src/invidious/config.cr index cee33ce1..429d9246 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -127,6 +127,9 @@ class Config # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) property pool_size : Int32 = 100 + # Use Innertube's transcripts API instead of timedtext for closed captions + property use_innertube_for_captions : Bool = false + # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] property cookies : HTTP::Cookies = HTTP::Cookies.new diff --git a/src/invidious/frontend/comments_youtube.cr b/src/invidious/frontend/comments_youtube.cr index 41f43f04..ecc0bc1b 100644 --- a/src/invidious/frontend/comments_youtube.cr +++ b/src/invidious/frontend/comments_youtube.cr @@ -23,6 +23,24 @@ module Invidious::Frontend::Comments </div> </div> END_HTML + elsif comments["authorId"]? && !comments["singlePost"]? + # for posts we should display a link to the post + replies_count_text = translate_count(locale, + "comments_view_x_replies", + child["replyCount"].as_i64 || 0, + NumberFormatting::Separator + ) + + replies_html = <<-END_HTML + <div class="pure-g"> + <div class="pure-u-1-24"></div> + <div class="pure-u-23-24"> + <p> + <a href="/post/#{child["commentId"]}?ucid=#{comments["authorId"]}">#{replies_count_text}</a> + </p> + </div> + </div> + END_HTML end if !thin_mode diff --git a/src/invidious/frontend/watch_page.cr b/src/invidious/frontend/watch_page.cr index e3214469..c8cb7110 100644 --- a/src/invidious/frontend/watch_page.cr +++ b/src/invidious/frontend/watch_page.cr @@ -7,7 +7,7 @@ module Invidious::Frontend::WatchPage getter full_videos : Array(Hash(String, JSON::Any)) getter video_streams : Array(Hash(String, JSON::Any)) getter audio_streams : Array(Hash(String, JSON::Any)) - getter captions : Array(Invidious::Videos::Caption) + getter captions : Array(Invidious::Videos::Captions::Metadata) def initialize( @full_videos, @@ -42,8 +42,7 @@ module Invidious::Frontend::WatchPage str << translate(locale, "Download as: ") str << "</label>\n" - # TODO: remove inline style - str << "\t\t<select style=\"width:100%\" name='download_widget' id='download_widget'>\n" + str << "\t\t<select name='download_widget' id='download_widget'>\n" # Non-DASH videos (audio+video) diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 23ff0da9..6dc9860e 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -208,3 +208,20 @@ def proxy_file(response, env) IO.copy response.body_io, env.response end end + +# Fetch the playback requests tracker from the statistics endpoint. +# +# Creates a new tracker when unavailable. +def get_playback_statistic + if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]) && tracker.as(Hash).empty? + tracker = { + "totalRequests" => 0_i64, + "successfulRequests" => 0_i64, + "ratio" => 0_f64, + } + + Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"] = tracker + end + + return tracker.as(Hash(String, Int64 | Float64)) +end diff --git a/src/invidious/helpers/i18next.cr b/src/invidious/helpers/i18next.cr index e84f88fb..252af6b9 100644 --- a/src/invidious/helpers/i18next.cr +++ b/src/invidious/helpers/i18next.cr @@ -35,19 +35,27 @@ module I18next::Plurals Special_Slovenian = 21 Special_Hebrew = 22 Special_Odia = 23 + + # Mixed v3/v4 rules in Weblate + # `es`, `pt` and `pt-PT` doesn't seem to have been refreshed + # by weblate yet, but I suspect it will happen one day. + # See: https://github.com/translate/translate/issues/4873 + Special_French_Portuguese + Special_Hungarian_Serbian + Special_Spanish_Italian end private PLURAL_SETS = { PluralForms::Single_gt_one => [ - "ach", "ak", "am", "arn", "br", "fil", "fr", "gun", "ln", "mfe", "mg", - "mi", "oc", "pt", "pt-BR", "tg", "tl", "ti", "tr", "uz", "wa", + "ach", "ak", "am", "arn", "br", "fil", "gun", "ln", "mfe", "mg", + "mi", "oc", "pt", "tg", "tl", "ti", "tr", "uz", "wa", ], PluralForms::Single_not_one => [ "af", "an", "ast", "az", "bg", "bn", "ca", "da", "de", "dev", "el", "en", "eo", "es", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi", - "hu", "hy", "ia", "it", "kk", "kn", "ku", "lb", "mai", "ml", "mn", "mr", + "hu", "hy", "ia", "kk", "kn", "ku", "lb", "mai", "ml", "mn", "mr", "nah", "nap", "nb", "ne", "nl", "nn", "no", "nso", "pa", "pap", "pms", - "ps", "pt-PT", "rm", "sco", "se", "si", "so", "son", "sq", "sv", "sw", + "ps", "rm", "sco", "se", "si", "so", "son", "sq", "sv", "sw", "ta", "te", "tk", "ur", "yo", ], PluralForms::None => [ @@ -55,7 +63,7 @@ module I18next::Plurals "lo", "ms", "sah", "su", "th", "tt", "ug", "vi", "wo", "zh", ], PluralForms::Dual_Slavic => [ - "be", "bs", "cnr", "dz", "hr", "ru", "sr", "uk", + "be", "bs", "cnr", "dz", "ru", "uk", ], } @@ -81,6 +89,12 @@ module I18next::Plurals "ro" => PluralForms::Special_Romanian, "sk" => PluralForms::Special_Czech_Slovak, "sl" => PluralForms::Special_Slovenian, + # Mixed v3/v4 rules + "fr" => PluralForms::Special_French_Portuguese, + "hr" => PluralForms::Special_Hungarian_Serbian, + "it" => PluralForms::Special_Spanish_Italian, + "pt-BR" => PluralForms::Special_French_Portuguese, + "sr" => PluralForms::Special_Hungarian_Serbian, } # These are the v1 and v2 compatible suffixes. @@ -150,9 +164,8 @@ module I18next::Plurals end def get_plural_form(locale : String) : PluralForms - # Extract the ISO 639-1 or 639-2 code from an RFC 5646 language code, - # except for pt-BR and pt-PT which needs to be kept as-is. - if !locale.matches?(/^pt-(BR|PT)$/) + # Extract the ISO 639-1 or 639-2 code from an RFC 5646 language code + if !locale.matches?(/^pt-BR$/) locale = locale.split('-')[0] end @@ -246,6 +259,10 @@ module I18next::Plurals when .special_slovenian? then return special_slovenian(count) when .special_hebrew? then return special_hebrew(count) when .special_odia? then return special_odia(count) + # Mixed v3/v4 forms + when .special_spanish_italian? then return special_cldr_Spanish_Italian(count) + when .special_french_portuguese? then return special_cldr_French_Portuguese(count) + when .special_hungarian_serbian? then return special_cldr_Hungarian_Serbian(count) else # default, if nothing matched above return 0_u8 @@ -507,5 +524,42 @@ module I18next::Plurals def self.special_odia(count : Int) : UInt8 return (count == 1) ? 0_u8 : 1_u8 end + + # ------------------- + # "v3.5" rules + # ------------------- + + # Plural form for Spanish & Italian languages + # + # This rule is mostly compliant to CLDR v42 + # + def self.special_cldr_Spanish_Italian(count : Int) : UInt8 + return 0_u8 if (count == 1) # one + return 1_u8 if (count != 0 && count % 1_000_000 == 0) # many + return 2_u8 # other + end + + # Plural form for French and Portuguese + # + # This rule is mostly compliant to CLDR v42 + # + def self.special_cldr_French_Portuguese(count : Int) : UInt8 + return 0_u8 if (count == 0 || count == 1) # one + return 1_u8 if (count % 1_000_000 == 0) # many + return 2_u8 # other + end + + # Plural form for Hungarian and Serbian + # + # This rule is mostly compliant to CLDR v42 + # + def self.special_cldr_Hungarian_Serbian(count : Int) : UInt8 + n_mod_10 = count % 10 + n_mod_100 = count % 100 + + return 0_u8 if (n_mod_10 == 1 && n_mod_100 != 11) # one + return 1_u8 if (2 <= n_mod_10 <= 4 && (n_mod_100 < 12 || 14 < n_mod_100)) # few + return 2_u8 # other + end end end diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index e0bd7279..31a3cf44 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -186,6 +186,7 @@ struct SearchChannel property author_thumbnail : String property subscriber_count : Int32 property video_count : Int32 + property channel_handle : String? property description_html : String property auto_generated : Bool property author_verified : Bool @@ -214,6 +215,7 @@ struct SearchChannel json.field "autoGenerated", self.auto_generated json.field "subCount", self.subscriber_count json.field "videoCount", self.video_count + json.field "channelHandle", self.channel_handle json.field "description", html_to_content(self.description_html) json.field "descriptionHtml", self.description_html diff --git a/src/invidious/helpers/webvtt.cr b/src/invidious/helpers/webvtt.cr new file mode 100644 index 00000000..56f761ed --- /dev/null +++ b/src/invidious/helpers/webvtt.cr @@ -0,0 +1,67 @@ +# Namespace for logic relating to generating WebVTT files +# +# Probably not compliant to WebVTT's specs but it is enough for Invidious. +module WebVTT + # A WebVTT builder generates WebVTT files + private class Builder + def initialize(@io : IO) + end + + # Writes an vtt cue with the specified time stamp and contents + def cue(start_time : Time::Span, end_time : Time::Span, text : String) + timestamp(start_time, end_time) + @io << text + @io << "\n\n" + end + + private def timestamp(start_time : Time::Span, end_time : Time::Span) + timestamp_component(start_time) + @io << " --> " + timestamp_component(end_time) + + @io << '\n' + end + + private def timestamp_component(timestamp : Time::Span) + @io << timestamp.hours.to_s.rjust(2, '0') + @io << ':' << timestamp.minutes.to_s.rjust(2, '0') + @io << ':' << timestamp.seconds.to_s.rjust(2, '0') + @io << '.' << timestamp.milliseconds.to_s.rjust(3, '0') + end + + def document(setting_fields : Hash(String, String)? = nil, &) + @io << "WEBVTT\n" + + if setting_fields + setting_fields.each do |name, value| + @io << name << ": " << value << '\n' + end + end + + @io << '\n' + + yield + end + end + + # Returns the resulting `String` of writing WebVTT to the yielded `WebVTT::Builder` + # + # ``` + # string = WebVTT.build do |vtt| + # vtt.cue(Time::Span.new(seconds: 1), Time::Span.new(seconds: 2), "Line 1") + # vtt.cue(Time::Span.new(seconds: 2), Time::Span.new(seconds: 3), "Line 2") + # end + # + # string # => "WEBVTT\n\n00:00:01.000 --> 00:00:02.000\nLine 1\n\n00:00:02.000 --> 00:00:03.000\nLine 2\n\n" + # ``` + # + # Accepts an optional settings fields hash to add settings attribute to the resulting vtt file. + def self.build(setting_fields : Hash(String, String)? = nil, &) + String.build do |str| + builder = Builder.new(str) + builder.document(setting_fields) do + yield builder + end + end + end +end diff --git a/src/invidious/jobs/statistics_refresh_job.cr b/src/invidious/jobs/statistics_refresh_job.cr index a113bd77..72d1ce88 100644 --- a/src/invidious/jobs/statistics_refresh_job.cr +++ b/src/invidious/jobs/statistics_refresh_job.cr @@ -18,6 +18,13 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob "updatedAt" => Time.utc.to_unix, "lastChannelRefreshedAt" => 0_i64, }, + + # + # "totalRequests" => 0_i64, + # "successfulRequests" => 0_i64 + # "ratio" => 0_i64 + # + "playback" => {} of String => Int64 | Float64, } private getter db : DB::Database @@ -30,7 +37,7 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob loop do refresh_stats - sleep 1.minute + sleep 10.minute Fiber.yield end end @@ -56,5 +63,8 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob "updatedAt" => Time.utc.to_unix, "lastChannelRefreshedAt" => Invidious::Database::Statistics.channel_last_update.try &.to_unix || 0_i64, } + + # Reset playback requests tracker + STATISTICS["playback"] = {} of String => Int64 | Float64 end end diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index fe4b5223..1651559a 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -39,6 +39,7 @@ module Invidious::JSONify::APIv1 json.field "author", video.author json.field "authorId", video.ucid json.field "authorUrl", "/channel/#{video.ucid}" + json.field "authorVerified", video.author_verified json.field "authorThumbnails" do json.array do diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 013be268..955e0855 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -89,6 +89,7 @@ struct Playlist property views : Int64 property updated : Time property thumbnail : String? + property subtitle : String? def to_json(offset, json : JSON::Builder, video_id : String? = nil) json.object do @@ -100,6 +101,7 @@ struct Playlist json.field "author", self.author json.field "authorId", self.ucid json.field "authorUrl", "/channel/#{self.ucid}" + json.field "subtitle", self.subtitle json.field "authorThumbnails" do json.array do @@ -356,6 +358,8 @@ def fetch_playlist(plid : String) updated = Time.utc video_count = 0 + subtitle = extract_text(initial_data.dig?("header", "playlistHeaderRenderer", "subtitle")) + playlist_info["stats"]?.try &.as_a.each do |stat| text = stat["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || stat["simpleText"]?.try &.as_s next if !text @@ -397,6 +401,7 @@ def fetch_playlist(plid : String) views: views, updated: updated, thumbnail: thumbnail, + subtitle: subtitle, }) end diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index adf05d30..67018660 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -343,6 +343,59 @@ module Invidious::Routes::API::V1::Channels end end + def self.post(env) + locale = env.get("preferences").as(Preferences).locale + + env.response.content_type = "application/json" + id = env.params.url["id"].to_s + ucid = env.params.query["ucid"]? + + thin_mode = env.params.query["thin_mode"]? + thin_mode = thin_mode == "true" + + format = env.params.query["format"]? + format ||= "json" + + if ucid.nil? + response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}") + return error_json(400, "Invalid post ID") if response["error"]? + ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s + else + ucid = ucid.to_s + end + + begin + fetch_channel_community_post(ucid, id, locale, format, thin_mode) + rescue ex + return error_json(500, ex) + end + end + + def self.post_comments(env) + locale = env.get("preferences").as(Preferences).locale + + env.response.content_type = "application/json" + + id = env.params.url["id"] + + thin_mode = env.params.query["thin_mode"]? + thin_mode = thin_mode == "true" + + format = env.params.query["format"]? + format ||= "json" + + continuation = env.params.query["continuation"]? + + case continuation + when nil, "" + ucid = env.params.query["ucid"] + comments = Comments.fetch_community_post_comments(ucid, id) + else + comments = YoutubeAPI.browse(continuation: continuation) + end + return Comments.parse_youtube(id, comments, format, locale, thin_mode, isPost: true) + end + def self.channels(env) locale = env.get("preferences").as(Preferences).locale ucid = env.params.url["ucid"] diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index e499f4d6..b42ecd1a 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -6,6 +6,22 @@ module Invidious::Routes::API::V1::Misc if !CONFIG.statistics_enabled return {"software" => SOFTWARE}.to_json else + # Calculate playback success rate + if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]?) + tracker = tracker.as(Hash(String, Int64 | Float64)) + + if !tracker.empty? + total_requests = tracker["totalRequests"] + success_count = tracker["successfulRequests"] + + if total_requests.zero? + tracker["ratio"] = 1_i64 + else + tracker["ratio"] = (success_count / (total_requests)).round(2) + end + end + end + return Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json end end @@ -162,17 +178,20 @@ module Invidious::Routes::API::V1::Misc resolved_url = YoutubeAPI.resolve_url(url.as(String)) endpoint = resolved_url["endpoint"] pageType = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || "" - if resolved_ucid = endpoint.dig?("watchEndpoint", "videoId") - elsif resolved_ucid = endpoint.dig?("browseEndpoint", "browseId") - elsif pageType == "WEB_PAGE_TYPE_UNKNOWN" + if pageType == "WEB_PAGE_TYPE_UNKNOWN" return error_json(400, "Unknown url") end + + sub_endpoint = endpoint["watchEndpoint"]? || endpoint["browseEndpoint"]? || endpoint + params = sub_endpoint.try &.dig?("params") rescue ex return error_json(500, ex) end JSON.build do |json| json.object do - json.field "ucid", resolved_ucid.try &.as_s || "" + json.field "ucid", sub_endpoint["browseId"].as_s if sub_endpoint["browseId"]? + json.field "videoId", sub_endpoint["videoId"].as_s if sub_endpoint["videoId"]? + json.field "params", params.try &.as_s json.field "pageType", pageType end end diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index af4fc806..1017ac9d 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -87,70 +87,67 @@ module Invidious::Routes::API::V1::Videos caption = caption[0] end - url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target - - # Auto-generated captions often have cues that aren't aligned properly with the video, - # as well as some other markup that makes it cumbersome, so we try to fix that here - if caption.name.includes? "auto-generated" - caption_xml = YT_POOL.client &.get(url).body - - if caption_xml.starts_with?("<?xml") - webvtt = caption.timedtext_to_vtt(caption_xml, tlang) - else - caption_xml = XML.parse(caption_xml) - - webvtt = String.build do |str| - str << <<-END_VTT - WEBVTT - Kind: captions - Language: #{tlang || caption.language_code} + if CONFIG.use_innertube_for_captions + params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated) + initial_data = YoutubeAPI.get_transcript(params) + webvtt = Invidious::Videos::Transcript.convert_transcripts_to_vtt(initial_data, caption.language_code) + else + # Timedtext API handling + url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target - END_VTT + # Auto-generated captions often have cues that aren't aligned properly with the video, + # as well as some other markup that makes it cumbersome, so we try to fix that here + if caption.name.includes? "auto-generated" + caption_xml = YT_POOL.client &.get(url).body - caption_nodes = caption_xml.xpath_nodes("//transcript/text") - caption_nodes.each_with_index do |node, i| - start_time = node["start"].to_f.seconds - duration = node["dur"]?.try &.to_f.seconds - duration ||= start_time + settings_field = { + "Kind" => "captions", + "Language" => "#{tlang || caption.language_code}", + } - if caption_nodes.size > i + 1 - end_time = caption_nodes[i + 1]["start"].to_f.seconds - else - end_time = start_time + duration - end + if caption_xml.starts_with?("<?xml") + webvtt = caption.timedtext_to_vtt(caption_xml, tlang) + else + caption_xml = XML.parse(caption_xml) + + webvtt = WebVTT.build(settings_field) do |webvtt| + caption_nodes = caption_xml.xpath_nodes("//transcript/text") + caption_nodes.each_with_index do |node, i| + start_time = node["start"].to_f.seconds + duration = node["dur"]?.try &.to_f.seconds + duration ||= start_time + + if caption_nodes.size > i + 1 + end_time = caption_nodes[i + 1]["start"].to_f.seconds + else + end_time = start_time + duration + end - start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}" - end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}" + text = HTML.unescape(node.content) + text = text.gsub(/<font color="#[a-fA-F0-9]{6}">/, "") + text = text.gsub(/<\/font>/, "") + if md = text.match(/(?<name>.*) : (?<text>.*)/) + text = "<v #{md["name"]}>#{md["text"]}</v>" + end - text = HTML.unescape(node.content) - text = text.gsub(/<font color="#[a-fA-F0-9]{6}">/, "") - text = text.gsub(/<\/font>/, "") - if md = text.match(/(?<name>.*) : (?<text>.*)/) - text = "<v #{md["name"]}>#{md["text"]}</v>" + webvtt.cue(start_time, end_time, text) end - - str << <<-END_CUE - #{start_time} --> #{end_time} - #{text} - - - END_CUE end end - end - else - # Some captions have "align:[start/end]" and "position:[num]%" - # attributes. Those are causing issues with VideoJS, which is unable - # to properly align the captions on the video, so we remove them. - # - # See: https://github.com/iv-org/invidious/issues/2391 - webvtt = YT_POOL.client &.get("#{url}&format=vtt").body - if webvtt.starts_with?("<?xml") - webvtt = caption.timedtext_to_vtt(webvtt) else - webvtt = YT_POOL.client &.get("#{url}&format=vtt").body - .gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1") + webvtt = YT_POOL.client &.get("#{url}&fmt=vtt").body + + if webvtt.starts_with?("<?xml") + webvtt = caption.timedtext_to_vtt(webvtt) + else + # Some captions have "align:[start/end]" and "position:[num]%" + # attributes. Those are causing issues with VideoJS, which is unable + # to properly align the captions on the video, so we remove them. + # + # See: https://github.com/iv-org/invidious/issues/2391 + webvtt = webvtt.gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1") + end end end @@ -207,11 +204,7 @@ module Invidious::Routes::API::V1::Videos storyboard = storyboard[0] end - String.build do |str| - str << <<-END_VTT - WEBVTT - END_VTT - + WebVTT.build do |vtt| start_time = 0.milliseconds end_time = storyboard[:interval].milliseconds @@ -223,12 +216,8 @@ module Invidious::Routes::API::V1::Videos storyboard[:storyboard_height].times do |j| storyboard[:storyboard_width].times do |k| - str << <<-END_CUE - #{start_time}.000 --> #{end_time}.000 - #{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]} - - - END_CUE + current_cue_url = "#{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]}" + vtt.cue(start_time, end_time, current_cue_url) start_time += storyboard[:interval].milliseconds end_time += storyboard[:interval].milliseconds diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 9892ae2a..d4d8b1c1 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -1,6 +1,12 @@ {% skip_file if flag?(:api_only) %} module Invidious::Routes::Channels + # Redirection for unsupported routes ("tabs") + def self.redirect_home(env) + ucid = env.params.url["ucid"] + return env.redirect "/channel/#{URI.encode_www_form(ucid)}" + end + def self.home(env) self.videos(env) end @@ -159,6 +165,11 @@ module Invidious::Routes::Channels end locale, user, subscriptions, continuation, ucid, channel = data + # redirect to post page + if lb = env.params.query["lb"]? + env.redirect "/post/#{URI.encode_www_form(lb)}?ucid=#{URI.encode_www_form(ucid)}" + end + thin_mode = env.params.query["thin_mode"]? || env.get("preferences").as(Preferences).thin_mode thin_mode = thin_mode == "true" @@ -187,6 +198,44 @@ module Invidious::Routes::Channels templated "community" end + def self.post(env) + # /post/{postId} + id = env.params.url["id"] + ucid = env.params.query["ucid"]? + + prefs = env.get("preferences").as(Preferences) + + locale = prefs.locale + + thin_mode = env.params.query["thin_mode"]? || prefs.thin_mode + thin_mode = thin_mode == "true" + + nojs = env.params.query["nojs"]? + + nojs ||= "0" + nojs = nojs == "1" + + if !ucid.nil? + ucid = ucid.to_s + post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode) + else + # resolve the url to get the author's UCID + response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}") + return error_template(400, "Invalid post ID") if response["error"]? + + ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s + post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode) + end + + post_response = JSON.parse(post_response) + + if nojs + comments = Comments.fetch_community_post_comments(ucid, id) + comment_html = JSON.parse(Comments.parse_youtube(id, comments, "html", locale, thin_mode, isPost: true))["contentHtml"] + end + templated "post" + end + def self.channels(env) data = self.fetch_basic_information(env) return data if !data.is_a?(Tuple) @@ -217,6 +266,11 @@ module Invidious::Routes::Channels env.redirect "/channel/#{ucid}" end + private KNOWN_TABS = { + "home", "videos", "shorts", "streams", "podcasts", + "releases", "playlists", "community", "channels", "about", + } + # Redirects brand url channels to a normal /channel/:ucid route def self.brand_redirect(env) locale = env.get("preferences").as(Preferences).locale @@ -227,7 +281,10 @@ module Invidious::Routes::Channels yt_url_params = URI::Params.encode(env.params.query.to_h.select(["a", "u", "user"])) # Retrieves URL params that only Invidious uses - invidious_url_params = URI::Params.encode(env.params.query.to_h.select!(["a", "u", "user"])) + invidious_url_params = env.params.query.dup + invidious_url_params.delete_all("a") + invidious_url_params.delete_all("u") + invidious_url_params.delete_all("user") begin resolved_url = YoutubeAPI.resolve_url("https://youtube.com#{env.request.path}#{yt_url_params.size > 0 ? "?#{yt_url_params}" : ""}") @@ -236,14 +293,17 @@ module Invidious::Routes::Channels return error_template(404, translate(locale, "This channel does not exist.")) end - selected_tab = env.request.path.split("/")[-1] - if {"home", "videos", "shorts", "streams", "playlists", "community", "channels", "about"}.includes? selected_tab + selected_tab = env.params.url["tab"]? + + if KNOWN_TABS.includes? selected_tab url = "/channel/#{ucid}/#{selected_tab}" else url = "/channel/#{ucid}" end - env.redirect url + url += "?#{invidious_url_params}" if !invidious_url_params.empty? + + return env.redirect url end # Handles redirects for the /profile endpoint diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index abe0f34e..112535bd 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -319,6 +319,15 @@ module Invidious::Routes::PreferencesRoute response: error_template(415, "Invalid playlist file uploaded") ) end + when "import_youtube_wh" + filename = part.filename || "" + success = Invidious::User::Import.from_youtube_wh(user, body, filename, type) + + if !success + haltf(env, status_code: 415, + response: error_template(415, "Invalid watch history file uploaded") + ) + end when "import_freetube" Invidious::User::Import.from_freetube(user, body) when "import_newpipe_subscriptions" diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 9641e01a..1d5aa914 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -80,9 +80,14 @@ module Invidious::Routes::VideoPlayback # Remove the Range header added previously. headers.delete("Range") if range_header.nil? + playback_statistics = get_playback_statistic() + playback_statistics["totalRequests"] += 1 + if response.status_code >= 400 env.response.content_type = "text/plain" haltf env, response.status_code + else + playback_statistics["successfulRequests"] += 1 end if url.includes? "&file=seg.ts" diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index e5cf3716..3d935f0a 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -30,14 +30,6 @@ module Invidious::Routes::Watch return env.redirect "/" end - embed_link = "/embed/#{id}" - if env.params.query.size > 1 - embed_params = HTTP::Params.parse(env.params.query.to_s) - embed_params.delete_all("v") - embed_link += "?" - embed_link += embed_params.to_s - end - plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") continuation = process_continuation(env.params.query, plid, id) diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index 9c43171c..d6bd991c 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -124,28 +124,42 @@ module Invidious::Routing get "/channel/:ucid/community", Routes::Channels, :community get "/channel/:ucid/channels", Routes::Channels, :channels get "/channel/:ucid/about", Routes::Channels, :about + get "/channel/:ucid/live", Routes::Channels, :live get "/user/:user/live", Routes::Channels, :live get "/c/:user/live", Routes::Channels, :live + get "/post/:id", Routes::Channels, :post - {"", "/videos", "/shorts", "/streams", "/playlists", "/community", "/about"}.each do |path| - # /c/LinusTechTips - get "/c/:user#{path}", Routes::Channels, :brand_redirect - # /user/linustechtips | Not always the same as /c/ - get "/user/:user#{path}", Routes::Channels, :brand_redirect - # /@LinusTechTips | Handle - get "/@:user#{path}", Routes::Channels, :brand_redirect - # /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow - get "/attribution_link#{path}", Routes::Channels, :brand_redirect - # /profile?user=linustechtips - get "/profile/#{path}", Routes::Channels, :profile - end + # Channel catch-all, to redirect future routes to the channel's home + # NOTE: defined last in order to be processed after the other routes + get "/channel/:ucid/*", Routes::Channels, :redirect_home + + # /c/LinusTechTips + get "/c/:user", Routes::Channels, :brand_redirect + get "/c/:user/:tab", Routes::Channels, :brand_redirect + + # /user/linustechtips (Not always the same as /c/) + get "/user/:user", Routes::Channels, :brand_redirect + get "/user/:user/:tab", Routes::Channels, :brand_redirect + + # /@LinusTechTips (Handle) + get "/@:user", Routes::Channels, :brand_redirect + get "/@:user/:tab", Routes::Channels, :brand_redirect + + # /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow + get "/attribution_link", Routes::Channels, :brand_redirect + get "/attribution_link/:tab", Routes::Channels, :brand_redirect + + # /profile?user=linustechtips + get "/profile", Routes::Channels, :profile + get "/profile/*", Routes::Channels, :profile end def register_watch_routes get "/watch", Routes::Watch, :handle post "/watch_ajax", Routes::Watch, :mark_watched get "/watch/:id", Routes::Watch, :redirect + get "/live/:id", Routes::Watch, :redirect get "/shorts/:id", Routes::Watch, :redirect get "/clip/:clip", Routes::Watch, :clip get "/w/:id", Routes::Watch, :redirect @@ -240,6 +254,10 @@ module Invidious::Routing get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}} {% end %} + # Posts + get "/api/v1/post/:id", {{namespace}}::Channels, :post + get "/api/v1/post/:id/comments", {{namespace}}::Channels, :post_comments + # 301 redirects to new /api/v1/channels/community/:ucid and /:ucid/community get "/api/v1/channels/comments/:ucid", {{namespace}}::Channels, :channel_comments_redirect get "/api/v1/channels/:ucid/comments", {{namespace}}::Channels, :channel_comments_redirect @@ -249,6 +267,7 @@ module Invidious::Routing get "/api/v1/search/suggestions", {{namespace}}::Search, :search_suggestions get "/api/v1/hashtag/:hashtag", {{namespace}}::Search, :hashtag + # Authenticated # The notification APIs cannot be extracted yet! They require the *local* notifications constant defined in invidious.cr diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 86d0ce6e..108f2ccc 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -218,6 +218,26 @@ struct Invidious::User end end + def from_youtube_wh(user : User, body : String, filename : String, type : String) : Bool + extension = filename.split(".").last + + if extension == "json" || type == "application/json" + data = JSON.parse(body) + watched = data.as_a.compact_map do |item| + next unless url = item["titleUrl"]? + next unless match = url.as_s.match(/\?v=(?<video_id>[a-zA-Z0-9_-]+)$/) + match["video_id"] + end + watched.reverse! # YouTube have newest first + user.watched += watched + user.watched.uniq! + Invidious::Database::Users.update_watch_history(user) + return true + else + return false + end + end + # ------------------- # Freetube # ------------------- @@ -228,8 +248,12 @@ struct Invidious::User subs = matches.map(&.["channel_id"]) if subs.empty? - data = JSON.parse(body)["subscriptions"] - subs = data.as_a.map(&.["id"].as_s) + profiles = body.split('\n', remove_empty: true) + profiles.each do |profile| + if data = JSON.parse(profile)["subscriptions"]? + subs += data.as_a.map(&.["id"].as_s) + end + end end user.subscriptions += subs diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index f38b33e5..9fbd1374 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -24,7 +24,7 @@ struct Video property updated : Time @[DB::Field(ignore: true)] - @captions = [] of Invidious::Videos::Caption + @captions = [] of Invidious::Videos::Captions::Metadata @[DB::Field(ignore: true)] property adaptive_fmts : Array(Hash(String, JSON::Any))? @@ -215,9 +215,9 @@ struct Video keywords.includes? "YouTube Red" end - def captions : Array(Invidious::Videos::Caption) + def captions : Array(Invidious::Videos::Captions::Metadata) if @captions.empty? && @info.has_key?("captions") - @captions = Invidious::Videos::Caption.from_yt_json(info["captions"]) + @captions = Invidious::Videos::Captions::Metadata.from_yt_json(info["captions"]) end return @captions diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr index 13f81a31..484e61d2 100644 --- a/src/invidious/videos/caption.cr +++ b/src/invidious/videos/caption.cr @@ -1,100 +1,89 @@ require "json" module Invidious::Videos - struct Caption - property name : String - property language_code : String - property base_url : String + module Captions + struct Metadata + property name : String + property language_code : String + property base_url : String - def initialize(@name, @language_code, @base_url) - end + property auto_generated : Bool - # Parse the JSON structure from Youtube - def self.from_yt_json(container : JSON::Any) : Array(Caption) - caption_tracks = container - .dig?("playerCaptionsTracklistRenderer", "captionTracks") - .try &.as_a + def initialize(@name, @language_code, @base_url, @auto_generated) + end - captions_list = [] of Caption - return captions_list if caption_tracks.nil? + # Parse the JSON structure from Youtube + def self.from_yt_json(container : JSON::Any) : Array(Captions::Metadata) + caption_tracks = container + .dig?("playerCaptionsTracklistRenderer", "captionTracks") + .try &.as_a - caption_tracks.each do |caption| - name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"] - name = name.to_s.split(" - ")[0] + captions_list = [] of Captions::Metadata + return captions_list if caption_tracks.nil? - language_code = caption["languageCode"].to_s - base_url = caption["baseUrl"].to_s + caption_tracks.each do |caption| + name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"] + name = name.to_s.split(" - ")[0] - captions_list << Caption.new(name, language_code, base_url) - end + language_code = caption["languageCode"].to_s + base_url = caption["baseUrl"].to_s - return captions_list - end + auto_generated = (caption["kind"]? == "asr") - def timedtext_to_vtt(timedtext : String, tlang = nil) : String - # In the future, we could just directly work with the url. This is more of a POC - cues = [] of XML::Node - tree = XML.parse(timedtext) - tree = tree.children.first - - tree.children.each do |item| - if item.name == "body" - item.children.each do |cue| - if cue.name == "p" && !(cue.children.size == 1 && cue.children[0].content == "\n") - cues << cue - end - end - break + captions_list << Captions::Metadata.new(name, language_code, base_url, auto_generated) end - end - result = String.build do |result| - result << <<-END_VTT - WEBVTT - Kind: captions - Language: #{tlang || @language_code} - - - END_VTT - result << "\n\n" + return captions_list + end - cues.each_with_index do |node, i| - start_time = node["t"].to_f.milliseconds + def timedtext_to_vtt(timedtext : String, tlang = nil) : String + # In the future, we could just directly work with the url. This is more of a POC + cues = [] of XML::Node + tree = XML.parse(timedtext) + tree = tree.children.first + + tree.children.each do |item| + if item.name == "body" + item.children.each do |cue| + if cue.name == "p" && !(cue.children.size == 1 && cue.children[0].content == "\n") + cues << cue + end + end + break + end + end - duration = node["d"]?.try &.to_f.milliseconds + settings_field = { + "Kind" => "captions", + "Language" => "#{tlang || @language_code}", + } - duration ||= start_time + result = WebVTT.build(settings_field) do |vtt| + cues.each_with_index do |node, i| + start_time = node["t"].to_f.milliseconds - if cues.size > i + 1 - end_time = cues[i + 1]["t"].to_f.milliseconds - else - end_time = start_time + duration - end + duration = node["d"]?.try &.to_f.milliseconds - # start_time - result << start_time.hours.to_s.rjust(2, '0') - result << ':' << start_time.minutes.to_s.rjust(2, '0') - result << ':' << start_time.seconds.to_s.rjust(2, '0') - result << '.' << start_time.milliseconds.to_s.rjust(3, '0') + duration ||= start_time - result << " --> " - - # end_time - result << end_time.hours.to_s.rjust(2, '0') - result << ':' << end_time.minutes.to_s.rjust(2, '0') - result << ':' << end_time.seconds.to_s.rjust(2, '0') - result << '.' << end_time.milliseconds.to_s.rjust(3, '0') + if cues.size > i + 1 + end_time = cues[i + 1]["t"].to_f.milliseconds + else + end_time = start_time + duration + end - result << "\n" + text = String.build do |io| + node.children.each do |s| + io << s.content + end + end - node.children.each do |s| - result << s.content + vtt.cue(start_time, end_time, text) end - result << "\n" - result << "\n" end + + return result end - return result end # List of all caption languages available on Youtube. diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 06ff96b1..77520dbe 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -78,6 +78,11 @@ def extract_video_info(video_id : String, proxy_region : String? = nil) # YouTube may return a different video player response than expected. # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 # Line to be reverted if one day we solve the video not available issue. + + # Although technically not a call to /videoplayback the fact that YouTube is returning the + # wrong video means that we should count it as a failure. + get_playback_statistic()["totalRequests"] += 1 + return { "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), "reason" => JSON::Any.new("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. <a href=\"https://github.com/iv-org/invidious/issues/3822\">Click here for more info about the issue.</a>"), @@ -137,9 +142,8 @@ end def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)? LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.") - # CgIQBg is a workaround for streaming URLs that returns a 403. - # See https://github.com/iv-org/invidious/issues/4027#issuecomment-1666944520 - response = YoutubeAPI.player(video_id: id, params: "CgIQBg", client_config: client_config) + # 2AMBCgIQBg is a workaround for streaming URLs that returns a 403. + response = YoutubeAPI.player(video_id: id, params: "2AMBCgIQBg", client_config: client_config) playability_status = response["playabilityStatus"]["status"] LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.") diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr new file mode 100644 index 00000000..dac00eea --- /dev/null +++ b/src/invidious/videos/transcript.cr @@ -0,0 +1,77 @@ +module Invidious::Videos + # Namespace for methods primarily relating to Transcripts + module Transcript + record TranscriptLine, start_ms : Time::Span, end_ms : Time::Span, line : String + + def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String + kind = auto_generated ? "asr" : "" + + object = { + "1:0:string" => video_id, + + "2:base64" => { + "1:string" => kind, + "2:string" => language_code, + "3:string" => "", + }, + + "3:varint" => 1_i64, + "5:string" => "engagement-panel-searchable-transcript-search-panel", + "6:varint" => 1_i64, + "7:varint" => 1_i64, + "8:varint" => 1_i64, + } + + params = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + return params + end + + def self.convert_transcripts_to_vtt(initial_data : Hash(String, JSON::Any), target_language : String) : String + # Convert into array of TranscriptLine + lines = self.parse(initial_data) + + settings_field = { + "Kind" => "captions", + "Language" => target_language, + } + + # Taken from Invidious::Videos::Captions::Metadata.timedtext_to_vtt() + vtt = WebVTT.build(settings_field) do |vtt| + lines.each do |line| + vtt.cue(line.start_ms, line.end_ms, line.line) + end + end + + return vtt + end + + private def self.parse(initial_data : Hash(String, JSON::Any)) + body = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", + "content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer", + "initialSegments").as_a + + lines = [] of TranscriptLine + body.each do |line| + # Transcript section headers. They are not apart of the captions and as such we can safely skip them. + if line.as_h.has_key?("transcriptSectionHeaderRenderer") + next + end + + line = line["transcriptSegmentRenderer"] + + start_ms = line["startMs"].as_s.to_i.millisecond + end_ms = line["endMs"].as_s.to_i.millisecond + + text = extract_text(line["snippet"]) || "" + + lines << TranscriptLine.new(start_ms, end_ms, text) + end + + return lines + end + end +end diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr index 24efc34e..d2a305d3 100644 --- a/src/invidious/views/community.ecr +++ b/src/invidious/views/community.ecr @@ -26,7 +26,7 @@ <p><%= error_message %></p> </div> <% else %> - <div class="h-box pure-g" id="comments"> + <div class="h-box pure-g comments" id="comments"> <%= IV::Frontend::Comments.template_youtube(items.not_nil!, locale, thin_mode) %> </div> <% end %> diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index c29ec47b..031b46da 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -26,8 +26,9 @@ </a></div> </div> + <% if !item.channel_handle.nil? %><p class="channel-name" dir="auto"><%= item.channel_handle %></p><% end %> <p><%= translate_count(locale, "generic_subscribers_count", item.subscriber_count, NumberFormatting::Separator) %></p> - <% if !item.auto_generated %><p><%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %></p><% end %> + <% if !item.auto_generated && item.channel_handle.nil? %><p><%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %></p><% end %> <h5><%= item.description_html %></h5> <% when SearchHashtag %> <% if !thin_mode %> diff --git a/src/invidious/views/playlist.ecr b/src/invidious/views/playlist.ecr index ee9ba87b..24ba437d 100644 --- a/src/invidious/views/playlist.ecr +++ b/src/invidious/views/playlist.ecr @@ -70,7 +70,12 @@ </b> <% else %> <b> - <a href="/channel/<%= playlist.ucid %>"><%= author %></a> | + <% if !author.empty? %> + <a href="/channel/<%= playlist.ucid %>"><%= author %></a> | + <% elsif !playlist.subtitle.nil? %> + <% subtitle = playlist.subtitle || "" %> + <span><%= HTML.escape(subtitle[0..subtitle.rindex(" • ") || subtitle.size]) %></span> | + <% end %> <%= translate_count(locale, "generic_videos_count", playlist.video_count) %> | <%= translate(locale, "Updated `x` ago", recode_date(playlist.updated, locale)) %> </b> diff --git a/src/invidious/views/post.ecr b/src/invidious/views/post.ecr new file mode 100644 index 00000000..fb03a44c --- /dev/null +++ b/src/invidious/views/post.ecr @@ -0,0 +1,48 @@ +<% content_for "header" do %> +<title>Invidious</title> +<% end %> + +<div> + <div id="post" class="comments post-comments"> + <%= IV::Frontend::Comments.template_youtube(post_response.not_nil!, locale, thin_mode) %> + </div> + + <% if nojs %> + <hr> + <% end %> + <br /> + + <div id="comments" class="comments post-comments"> + <% if nojs %> + <%= comment_html %> + <% else %> + <noscript> + <a href="/post/<%= id %>?ucid=<%= ucid %>&nojs=1"> + <%= translate(locale, "Hi! Looks like you have JavaScript turned off. Click here to view comments, keep in mind they may take a bit longer to load.") %> + </a> + </noscript> + <% end %> + </div> +</div> + +<script id="video_data" type="application/json"> +<%= +{ + "id" => id, + "youtube_comments_text" => HTML.escape(translate(locale, "View YouTube comments")), + "reddit_comments_text" => "", + "reddit_permalink_text" => "", + "comments_text" => HTML.escape(translate(locale, "View `x` comments", "{commentCount}")), + "hide_replies_text" => HTML.escape(translate(locale, "Hide replies")), + "show_replies_text" => HTML.escape(translate(locale, "Show replies")), + "params" => { + "comments": ["youtube"] + }, + "preferences" => prefs, + "base_url" => "/api/v1/post/#{URI.encode_www_form(id)}/comments", + "ucid" => ucid +}.to_pretty_json +%> +</script> +<script src="/js/comments.js?v=<%= ASSET_COMMIT %>"></script> +<script src="/js/post.js?v=<%= ASSET_COMMIT %>"></script>
\ No newline at end of file diff --git a/src/invidious/views/user/data_control.ecr b/src/invidious/views/user/data_control.ecr index 27654b40..9ce42c99 100644 --- a/src/invidious/views/user/data_control.ecr +++ b/src/invidious/views/user/data_control.ecr @@ -27,6 +27,11 @@ </div> <div class="pure-control-group"> + <label for="import_youtube_wh"><%= translate(locale, "Import YouTube watch history (.json)") %></label> + <input type="file" id="import_youtube_wh" name="import_youtube_wh"> + </div> + + <div class="pure-control-group"> <label for="import_freetube"><%= translate(locale, "Import FreeTube subscriptions (.db)") %></label> <input type="file" id="import_freetube" name="import_freetube"> </div> diff --git a/src/invidious/views/user/preferences.ecr b/src/invidious/views/user/preferences.ecr index dfda1434..55349c5a 100644 --- a/src/invidious/views/user/preferences.ecr +++ b/src/invidious/views/user/preferences.ecr @@ -89,7 +89,7 @@ <label for="captions[0]"><%= translate(locale, "preferences_captions_label") %></label> <% preferences.captions.each_with_index do |caption, index| %> <select class="pure-u-1-6" name="captions[<%= index %>]" id="captions[<%= index %>]"> - <% Invidious::Videos::Caption::LANGUAGES.each do |option| %> + <% Invidious::Videos::Captions::LANGUAGES.each do |option| %> <option value="<%= option %>" <% if preferences.captions[index] == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option> <% end %> </select> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 498d57a1..07474896 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -64,7 +64,8 @@ we're going to need to do it here in order to allow for translations. "premiere_timestamp" => video.premiere_timestamp.try &.to_unix, "vr" => video.is_vr, "projection_type" => video.projection_type, - "local_disabled" => CONFIG.disabled?("local") + "local_disabled" => CONFIG.disabled?("local"), + "support_reddit" => true }.to_pretty_json %> </script> @@ -112,19 +113,36 @@ we're going to need to do it here in order to allow for translations. <div class="pure-u-1 pure-u-lg-1-5"> <div class="h-box"> <span id="watch-on-youtube"> - <a href="https://www.youtube.com/watch?v=<%= video.id %>"><%= translate(locale, "videoinfo_watch_on_youTube") %></a> - (<a href="https://www.youtube.com/embed/<%= video.id %>"><%= translate(locale, "videoinfo_youTube_embed_link") %></a>) + <%- + link_yt_watch = URI.new(scheme: "https", host: "www.youtube.com", path: "/watch", query: "v=#{video.id}") + link_yt_embed = URI.new(scheme: "https", host: "www.youtube.com", path: "/embed/#{video.id}") + + if !plid.nil? && !continuation.nil? + link_yt_param = URI::Params{"plid" => [plid], "index" => [continuation.to_s]} + link_yt_watch = IV::HttpServer::Utils.add_params_to_url(link_yt_watch, link_yt_param) + link_yt_embed = IV::HttpServer::Utils.add_params_to_url(link_yt_embed, link_yt_param) + end + -%> + <a id="link-yt-watch" data-base-url="<%= link_yt_watch %>" href="<%= link_yt_watch %>"><%= translate(locale, "videoinfo_watch_on_youTube") %></a> + (<a id="link-yt-embed" data-base-url="<%= link_yt_embed %>" href="<%= link_yt_embed %>"><%= translate(locale, "videoinfo_youTube_embed_link") %></a>) </span> + <p id="watch-on-another-invidious-instance"> - <% if env.get("preferences").as(Preferences).automatic_instance_redirect%> - <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a> - <% else %> - <a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a> - <% end %> + <%- link_iv_other = IV::Frontend::Misc.redirect_url(env) -%> + <a id="link-iv-other" data-base-url="<%= link_iv_other %>" href="<%= link_iv_other %>"><%= translate(locale, "Switch Invidious Instance") %></a> </p> + <p id="embed-link"> - <a href="<%= embed_link %>"><%= translate(locale, "videoinfo_invidious_embed_link") %></a> + <%- + params_iv_embed = env.params.query.dup + params_iv_embed.delete_all("v") + + link_iv_embed = URI.new(path: "/embed/#{id}") + link_iv_embed = IV::HttpServer::Utils.add_params_to_url(link_iv_embed, params_iv_embed) + -%> + <a id="link-iv-embed" data-base-url="<%= link_iv_embed %>" href="<%= link_iv_embed %>"><%= translate(locale, "videoinfo_invidious_embed_link") %></a> </p> + <p id="annotations"> <% if params.annotations %> <a href="/watch?<%= env.params.query %>&iv_load_policy=3"> @@ -270,7 +288,7 @@ we're going to need to do it here in order to allow for translations. <hr> <% end %> - <div id="comments"> + <div id="comments" class="comments"> <% if nojs %> <%= comment_html %> <% else %> @@ -352,4 +370,5 @@ we're going to need to do it here in order to allow for translations. </div> <% end %> </div> +<script src="/js/comments.js?v=<%= ASSET_COMMIT %>"></script> <script src="/js/watch.js?v=<%= ASSET_COMMIT %>"></script> diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index e9eb726c..03ff0ee4 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -37,7 +37,7 @@ struct YoutubeConnectionPool conn.close conn = HTTP::Client.new(url) - conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET + conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" response = yield conn @@ -52,7 +52,7 @@ struct YoutubeConnectionPool private def build_pool DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do conn = HTTP::Client.new(url) - conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET + conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" conn @@ -62,7 +62,7 @@ end def make_client(url : URI, region = nil) client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure) - client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC + client.family = CONFIG.force_resolve client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" client.read_timeout = 10.seconds client.connect_timeout = 10.seconds diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index aaf7772e..56325cf7 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -175,17 +175,18 @@ private module Parsers # Always simpleText # TODO change default value to nil - subscriber_count = item_contents.dig?("subscriberCountText", "simpleText") + subscriber_count = item_contents.dig?("subscriberCountText", "simpleText").try &.as_s + channel_handle = subscriber_count if (subscriber_count.try &.starts_with? "@") # Since youtube added channel handles, `VideoCountText` holds the number of # subscribers and `subscriberCountText` holds the handle, except when the # channel doesn't have a handle (e.g: some topic music channels). # See https://github.com/iv-org/invidious/issues/3394#issuecomment-1321261688 - if !subscriber_count || !subscriber_count.as_s.includes? " subscriber" - subscriber_count = item_contents.dig?("videoCountText", "simpleText") + if !subscriber_count || !subscriber_count.includes? " subscriber" + subscriber_count = item_contents.dig?("videoCountText", "simpleText").try &.as_s end subscriber_count = subscriber_count - .try { |s| short_text_to_number(s.as_s.split(" ")[0]).to_i32 } || 0 + .try { |s| short_text_to_number(s.split(" ")[0]).to_i32 } || 0 # Auto-generated channels doesn't have videoCountText # Taken from: https://github.com/iv-org/invidious/pull/2228#discussion_r717620922 @@ -200,6 +201,7 @@ private module Parsers author_thumbnail: author_thumbnail, subscriber_count: subscriber_count, video_count: video_count, + channel_handle: channel_handle, description_html: description_html, auto_generated: auto_generated, author_verified: author_verified, diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index aef9ddd9..a5e621f2 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -558,6 +558,30 @@ module YoutubeAPI end #################################################################### + # get_transcript(params, client_config?) + # + # Requests the youtubei/v1/get_transcript endpoint with the required headers + # and POST data in order to get a JSON reply. + # + # The requested data is a specially encoded protobuf string that denotes the specific language requested. + # + # An optional ClientConfig parameter can be passed, too (see + # `struct ClientConfig` above for more details). + # + + def get_transcript( + params : String, + client_config : ClientConfig | Nil = nil + ) : Hash(String, JSON::Any) + data = { + "context" => self.make_context(client_config), + "params" => params, + } + + return self._post_json("/youtubei/v1/get_transcript", data, client_config) + end + + #################################################################### # _post_json(endpoint, data, client_config?) # # Internal function that does the actual request to youtube servers |