diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/invidious.cr | 2 | ||||
| -rw-r--r-- | src/invidious/config.cr | 2 | ||||
| -rw-r--r-- | src/invidious/helpers/serialized_yt_data.cr | 21 | ||||
| -rw-r--r-- | src/invidious/routes/errors.cr | 5 | ||||
| -rw-r--r-- | src/invidious/routes/images.cr | 142 | ||||
| -rw-r--r-- | src/invidious/user/imports.cr | 2 | ||||
| -rw-r--r-- | src/invidious/videos/parser.cr | 11 | ||||
| -rw-r--r-- | src/invidious/views/components/item.ecr | 26 | ||||
| -rw-r--r-- | src/invidious/yt_backend/connection_pool.cr | 37 | ||||
| -rw-r--r-- | src/invidious/yt_backend/extractors.cr | 53 | ||||
| -rw-r--r-- | src/invidious/yt_backend/youtube_api.cr | 14 |
11 files changed, 142 insertions, 173 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index 84e1895d..e0bd0101 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -90,7 +90,7 @@ SOFTWARE = { "branch" => "#{CURRENT_BRANCH}", } -YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size, use_quic: CONFIG.use_quic) +YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size) # CLI Kemal.config.extra_options do |parser| diff --git a/src/invidious/config.cr b/src/invidious/config.cr index c88a4837..429d9246 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -126,8 +126,6 @@ class Config property host_binding : String = "0.0.0.0" # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) property pool_size : Int32 = 100 - # Use quic transport for youtube api - property use_quic : Bool = false # Use Innertube's transcripts API instead of timedtext for closed captions property use_innertube_for_captions : Bool = false diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index 7c12ad0e..e0bd7279 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -232,6 +232,25 @@ struct SearchChannel end end +struct SearchHashtag + include DB::Serializable + + property title : String + property url : String + property video_count : Int64 + property channel_count : Int64 + + def to_json(locale : String?, json : JSON::Builder) + json.object do + json.field "type", "hashtag" + json.field "title", self.title + json.field "url", self.url + json.field "videoCount", self.video_count + json.field "channelCount", self.channel_count + end + end +end + class Category include DB::Serializable @@ -274,4 +293,4 @@ struct Continuation end end -alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category +alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | SearchHashtag | Category diff --git a/src/invidious/routes/errors.cr b/src/invidious/routes/errors.cr index b138b562..1e9ab44e 100644 --- a/src/invidious/routes/errors.cr +++ b/src/invidious/routes/errors.cr @@ -1,5 +1,10 @@ module Invidious::Routes::ErrorRoutes def self.error_404(env) + # Workaround for #3117 + if HOST_URL.empty? && env.request.path.starts_with?("/v1/storyboards/sb") + return env.redirect "#{env.request.path[15..]}?#{env.params.query}" + end + if md = env.request.path.match(/^\/(?<id>([a-zA-Z0-9_-]{11})|(\w+))$/) item = md["id"] diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index 594a7869..b6a2e110 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -3,17 +3,7 @@ module Invidious::Routes::Images def self.ggpht(env) url = env.request.path.lchop("/ggpht") - headers = ( - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - HTTP::Headers{":authority" => "yt3.ggpht.com"} - else - HTTP::Headers.new - end - {% else %} - HTTP::Headers.new - {% end %} - ) + headers = HTTP::Headers.new REQUEST_HEADERS_WHITELIST.each do |header| if env.request.headers[header]? @@ -42,22 +32,9 @@ module Invidious::Routes::Images } begin - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - YT_POOL.client &.get(url, headers) do |resp| - return request_proc.call(resp) - end - else - HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| - return request_proc.call(resp) - end - end - {% else %} - # This can likely be optimized into a (small) pool sometime in the future. - HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| - return request_proc.call(resp) - end - {% end %} + HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| + return request_proc.call(resp) + end rescue ex end end @@ -78,10 +55,6 @@ module Invidious::Routes::Images headers = HTTP::Headers.new - {% unless flag?(:disable_quic) %} - headers[":authority"] = "#{authority}.ytimg.com" - {% end %} - REQUEST_HEADERS_WHITELIST.each do |header| if env.request.headers[header]? headers[header] = env.request.headers[header] @@ -107,22 +80,9 @@ module Invidious::Routes::Images } begin - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - YT_POOL.client &.get(url, headers) do |resp| - return request_proc.call(resp) - end - else - HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end - end - {% else %} - # This can likely be optimized into a (small) pool sometime in the future. - HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end - {% end %} + HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| + return request_proc.call(resp) + end rescue ex end end @@ -133,17 +93,7 @@ module Invidious::Routes::Images name = env.params.url["name"] url = env.request.resource - headers = ( - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - HTTP::Headers{":authority" => "i9.ytimg.com"} - else - HTTP::Headers.new - end - {% else %} - HTTP::Headers.new - {% end %} - ) + headers = HTTP::Headers.new REQUEST_HEADERS_WHITELIST.each do |header| if env.request.headers[header]? @@ -169,22 +119,9 @@ module Invidious::Routes::Images } begin - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - YT_POOL.client &.get(url, headers) do |resp| - return request_proc.call(resp) - end - else - HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end - end - {% else %} - # This can likely be optimized into a (small) pool sometime in the future. - HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end - {% end %} + HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| + return request_proc.call(resp) + end rescue ex end end @@ -223,41 +160,16 @@ module Invidious::Routes::Images id = env.params.url["id"] name = env.params.url["name"] - headers = ( - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - HTTP::Headers{":authority" => "i.ytimg.com"} - else - HTTP::Headers.new - end - {% else %} - HTTP::Headers.new - {% end %} - ) + headers = HTTP::Headers.new if name == "maxres.jpg" build_thumbnails(id).each do |thumb| thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" - # Logic here is short enough that manually typing them out should be fine. - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - if YT_POOL.client &.head(thumbnail_resource_path, headers).status_code == 200 - name = thumb[:url] + ".jpg" - break - end - else - if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 - name = thumb[:url] + ".jpg" - break - end - end - {% else %} - # This can likely be optimized into a (small) pool sometime in the future. - if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 - name = thumb[:url] + ".jpg" - break - end - {% end %} + # This can likely be optimized into a (small) pool sometime in the future. + if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 + name = thumb[:url] + ".jpg" + break + end end end @@ -287,22 +199,10 @@ module Invidious::Routes::Images } begin - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - YT_POOL.client &.get(url, headers) do |resp| - return request_proc.call(resp) - end - else - HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end - end - {% else %} - # This can likely be optimized into a (small) pool sometime in the future. - HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end - {% end %} + # This can likely be optimized into a (small) pool sometime in the future. + HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| + return request_proc.call(resp) + end rescue ex end end diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 0a2fe1e2..86d0ce6e 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -133,7 +133,7 @@ struct Invidious::User next if !video_id begin - video = get_video(video_id) + video = get_video(video_id, false) rescue ex next end diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 9cc0ffdc..06ff96b1 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -55,8 +55,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil) client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region) # Fetch data from the player endpoint - # 8AEB param is used to fetch YouTube stories - player_response = YoutubeAPI.player(video_id: video_id, params: "8AEB", client_config: client_config) + player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s @@ -119,6 +118,9 @@ def extract_video_info(video_id : String, proxy_region : String? = nil) # Replace player response and reset reason if !new_player_response.nil? + # Preserve storyboard data before replacement + new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]? + player_response = new_player_response params.delete("reason") end @@ -135,8 +137,9 @@ end def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)? LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.") - # 8AEB param is used to fetch YouTube stories - response = YoutubeAPI.player(video_id: id, params: "8AEB", client_config: client_config) + # CgIQBg is a workaround for streaming URLs that returns a 403. + # See https://github.com/iv-org/invidious/issues/4027#issuecomment-1666944520 + response = YoutubeAPI.player(video_id: id, params: "CgIQBg", client_config: client_config) playability_status = response["playabilityStatus"]["status"] LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.") diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index 7ffd2d93..c29ec47b 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -1,6 +1,6 @@ <%- thin_mode = env.get("preferences").as(Preferences).thin_mode - item_watched = !item.is_a?(SearchChannel | SearchPlaylist | InvidiousPlaylist | Category) && env.get?("user").try &.as(User).watched.index(item.id) != nil + item_watched = !item.is_a?(SearchChannel | SearchHashtag | SearchPlaylist | InvidiousPlaylist | Category) && env.get?("user").try &.as(User).watched.index(item.id) != nil author_verified = item.responds_to?(:author_verified) && item.author_verified -%> @@ -29,6 +29,30 @@ <p><%= translate_count(locale, "generic_subscribers_count", item.subscriber_count, NumberFormatting::Separator) %></p> <% if !item.auto_generated %><p><%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %></p><% end %> <h5><%= item.description_html %></h5> + <% when SearchHashtag %> + <% if !thin_mode %> + <a tabindex="-1" href="<%= item.url %>"> + <center><img style="width:56.25%" src="/hashtag.svg" alt="" /></center> + </a> + <%- else -%> + <div class="thumbnail-placeholder" style="width:56.25%"></div> + <% end %> + + <div class="video-card-row"> + <div class="flex-left"><a href="<%= item.url %>"><%= HTML.escape(item.title) %></a></div> + </div> + + <div class="video-card-row"> + <%- if item.video_count != 0 -%> + <p><%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %></p> + <%- end -%> + </div> + + <div class="video-card-row"> + <%- if item.channel_count != 0 -%> + <p><%= translate_count(locale, "generic_channels_count", item.channel_count, NumberFormatting::Separator) %></p> + <%- end -%> + </div> <% when SearchPlaylist, InvidiousPlaylist %> <%- if item.id.starts_with? "RD" diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 658731cf..e9eb726c 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -1,11 +1,3 @@ -{% unless flag?(:disable_quic) %} - require "lsquic" - - alias HTTPClientType = QUIC::Client | HTTP::Client -{% else %} - alias HTTPClientType = HTTP::Client -{% end %} - def add_yt_headers(request) if request.headers["User-Agent"] == "Crystal" request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" @@ -26,11 +18,11 @@ struct YoutubeConnectionPool property! url : URI property! capacity : Int32 property! timeout : Float64 - property pool : DB::Pool(HTTPClientType) + property pool : DB::Pool(HTTP::Client) - def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true) + def initialize(url : URI, @capacity = 5, @timeout = 5.0) @url = url - @pool = build_pool(use_quic) + @pool = build_pool() end def client(region = nil, &block) @@ -43,11 +35,7 @@ struct YoutubeConnectionPool response = yield conn rescue ex conn.close - {% unless flag?(:disable_quic) %} - conn = CONFIG.use_quic ? QUIC::Client.new(url) : HTTP::Client.new(url) - {% else %} - conn = HTTP::Client.new(url) - {% end %} + conn = HTTP::Client.new(url) conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC @@ -61,19 +49,9 @@ struct YoutubeConnectionPool response end - private def build_pool(use_quic) - DB::Pool(HTTPClientType).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do - conn = nil # Declare - {% unless flag?(:disable_quic) %} - if use_quic - conn = QUIC::Client.new(url) - else - conn = HTTP::Client.new(url) - end - {% else %} - conn = HTTP::Client.new(url) - {% end %} - + private def build_pool + DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do + conn = HTTP::Client.new(url) conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -83,7 +61,6 @@ struct YoutubeConnectionPool end def make_client(url : URI, region = nil) - # TODO: Migrate any applicable endpoints to QUIC client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure) client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 8cf59d50..aaf7772e 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -11,15 +11,16 @@ private ITEM_CONTAINER_EXTRACTOR = { } private ITEM_PARSERS = { + Parsers::RichItemRendererParser, Parsers::VideoRendererParser, Parsers::ChannelRendererParser, Parsers::GridPlaylistRendererParser, Parsers::PlaylistRendererParser, Parsers::CategoryRendererParser, - Parsers::RichItemRendererParser, Parsers::ReelItemRendererParser, Parsers::ItemSectionRendererParser, Parsers::ContinuationItemRendererParser, + Parsers::HashtagRendererParser, } private alias InitialData = Hash(String, JSON::Any) @@ -210,6 +211,56 @@ private module Parsers end end + # Parses an Innertube `hashtagTileRenderer` into a `SearchHashtag`. + # Returns `nil` when the given object is not a `hashtagTileRenderer`. + # + # A `hashtagTileRenderer` is a kind of search result. + # It can be found when searching for any hashtag (e.g "#hi" or "#shorts") + module HashtagRendererParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["hashtagTileRenderer"]? + return self.parse(item_contents) + end + end + + private def self.parse(item_contents) + title = extract_text(item_contents["hashtag"]).not_nil! # E.g "#hi" + + # E.g "/hashtag/hi" + url = item_contents.dig?("onTapCommand", "commandMetadata", "webCommandMetadata", "url").try &.as_s + url ||= URI.encode_path("/hashtag/#{title.lchop('#')}") + + video_count_txt = extract_text(item_contents["hashtagVideoCount"]?) # E.g "203K videos" + channel_count_txt = extract_text(item_contents["hashtagChannelCount"]?) # E.g "81K channels" + + # Fallback for video/channel counts + if channel_count_txt.nil? || video_count_txt.nil? + # E.g: "203K videos • 81K channels" + info_text = extract_text(item_contents["hashtagInfoText"]?).try &.split(" • ") + + if info_text && info_text.size == 2 + video_count_txt ||= info_text[0] + channel_count_txt ||= info_text[1] + end + end + + return SearchHashtag.new({ + title: title, + url: url, + video_count: short_text_to_number(video_count_txt || ""), + channel_count: short_text_to_number(channel_count_txt || ""), + }) + rescue ex + LOGGER.debug("HashtagRendererParser: Failed to extract renderer.") + LOGGER.debug("HashtagRendererParser: Got exception: #{ex.message}") + return nil + end + + def self.parser_name + return {{@type.name}} + end + end + # Parses a InnerTube gridPlaylistRenderer into a SearchPlaylist. Returns nil when the given object isn't a gridPlaylistRenderer # # A gridPlaylistRenderer renders a playlist, that is located in a grid, to click on within the YouTube and Invidious UI. diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index a3335bbf..a5e621f2 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -619,17 +619,9 @@ module YoutubeAPI LOGGER.trace("YoutubeAPI: POST data: #{data}") # Send the POST request - if {{ !flag?(:disable_quic) }} && CONFIG.use_quic - # Using QUIC client - body = YT_POOL.client(client_config.proxy_region, - &.post(url, headers: headers, body: data.to_json) - ).body - else - # Using HTTP client - body = YT_POOL.client(client_config.proxy_region) do |client| - client.post(url, headers: headers, body: data.to_json) do |response| - self._decompress(response.body_io, response.headers["Content-Encoding"]?) - end + body = YT_POOL.client(client_config.proxy_region) do |client| + client.post(url, headers: headers, body: data.to_json) do |response| + self._decompress(response.body_io, response.headers["Content-Encoding"]?) end end |