diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/invidious.cr | 196 | ||||
| -rw-r--r-- | src/invidious/comments.cr | 6 | ||||
| -rw-r--r-- | src/invidious/helpers/helpers.cr | 37 | ||||
| -rw-r--r-- | src/invidious/helpers/serialized_yt_data.cr | 9 | ||||
| -rw-r--r-- | src/invidious/helpers/utils.cr | 96 | ||||
| -rw-r--r-- | src/invidious/mixes.cr | 2 | ||||
| -rw-r--r-- | src/invidious/playlists.cr | 51 | ||||
| -rw-r--r-- | src/invidious/routes/api/v1/misc.cr | 30 | ||||
| -rw-r--r-- | src/invidious/routes/images.cr | 191 | ||||
| -rw-r--r-- | src/invidious/views/components/item.ecr | 12 | ||||
| -rw-r--r-- | src/invidious/views/template.ecr | 4 | ||||
| -rw-r--r-- | src/invidious/views/watch.ecr | 2 | ||||
| -rw-r--r-- | src/invidious/yt_backend/connection_pool.cr | 96 | ||||
| -rw-r--r-- | src/invidious/yt_backend/extractors.cr (renamed from src/invidious/helpers/extractors.cr) | 47 | ||||
| -rw-r--r-- | src/invidious/yt_backend/extractors_utils.cr | 67 | ||||
| -rw-r--r-- | src/invidious/yt_backend/proxy.cr (renamed from src/invidious/helpers/proxy.cr) | 0 | ||||
| -rw-r--r-- | src/invidious/yt_backend/youtube_api.cr (renamed from src/invidious/helpers/youtube_api.cr) | 0 |
17 files changed, 446 insertions, 400 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index f8f0784a..570c33e6 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -27,6 +27,7 @@ require "yaml" require "compress/zip" require "protodec/utils" require "./invidious/helpers/*" +require "./invidious/yt_backend/*" require "./invidious/*" require "./invidious/channels/*" require "./invidious/user/*" @@ -390,6 +391,13 @@ end Invidious::Routing.post "/feed/webhook/:token", Invidious::Routes::Feeds, :push_notifications_post {% end %} +Invidious::Routing.get "/ggpht/*", Invidious::Routes::Images, :ggpht +Invidious::Routing.options "/sb/:authority/:id/:storyboard/:index", Invidious::Routes::Images, :options_storyboard +Invidious::Routing.get "/sb/:authority/:id/:storyboard/:index", Invidious::Routes::Images, :get_storyboard +Invidious::Routing.get "/s_p/:id/:name", Invidious::Routes::Images, :s_p_image +Invidious::Routing.get "/yts/img/:name", Invidious::Routes::Images, :yts_image +Invidious::Routing.get "/vi/:id/:name", Invidious::Routes::Images, :thumbnails + # API routes (macro) define_v1_api_routes() @@ -1274,194 +1282,6 @@ post "/api/v1/auth/notifications" do |env| create_notification_stream(env, topics, connection_channel) end -get "/ggpht/*" do |env| - url = env.request.path.lchop("/ggpht") - - headers = HTTP::Headers{":authority" => "yt3.ggpht.com"} - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end - - begin - YT_POOL.client &.get(url, headers) do |response| - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - env.response.headers.delete("Transfer-Encoding") - break - end - - proxy_file(response, env) - end - rescue ex - end -end - -options "/sb/:authority/:id/:storyboard/:index" do |env| - env.response.headers["Access-Control-Allow-Origin"] = "*" - env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" - env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range" -end - -get "/sb/:authority/:id/:storyboard/:index" do |env| - authority = env.params.url["authority"] - id = env.params.url["id"] - storyboard = env.params.url["storyboard"] - index = env.params.url["index"] - - url = "/sb/#{id}/#{storyboard}/#{index}?#{env.params.query}" - - headers = HTTP::Headers.new - - headers[":authority"] = "#{authority}.ytimg.com" - - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end - - begin - YT_POOL.client &.get(url, headers) do |response| - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Connection"] = "close" - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - env.response.headers.delete("Transfer-Encoding") - break - end - - proxy_file(response, env) - end - rescue ex - end -end - -get "/s_p/:id/:name" do |env| - id = env.params.url["id"] - name = env.params.url["name"] - - url = env.request.resource - - headers = HTTP::Headers{":authority" => "i9.ytimg.com"} - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end - - begin - YT_POOL.client &.get(url, headers) do |response| - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - env.response.headers.delete("Transfer-Encoding") - break - end - - proxy_file(response, env) - end - rescue ex - end -end - -get "/yts/img/:name" do |env| - headers = HTTP::Headers.new - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end - - begin - YT_POOL.client &.get(env.request.resource, headers) do |response| - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - env.response.headers.delete("Transfer-Encoding") - break - end - - proxy_file(response, env) - end - rescue ex - end -end - -get "/vi/:id/:name" do |env| - id = env.params.url["id"] - name = env.params.url["name"] - - headers = HTTP::Headers{":authority" => "i.ytimg.com"} - - if name == "maxres.jpg" - build_thumbnails(id).each do |thumb| - if YT_POOL.client &.head("/vi/#{id}/#{thumb[:url]}.jpg", headers).status_code == 200 - name = thumb[:url] + ".jpg" - break - end - end - end - url = "/vi/#{id}/#{name}" - - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end - - begin - YT_POOL.client &.get(url, headers) do |response| - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - env.response.headers.delete("Transfer-Encoding") - break - end - - proxy_file(response, env) - end - rescue ex - end -end - get "/Captcha" do |env| headers = HTTP::Headers{":authority" => "accounts.google.com"} response = YT_POOL.client &.get(env.request.resource, headers) diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index a5506b03..9c788253 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -329,7 +329,7 @@ def template_youtube_comments(comments, locale, thin_mode, is_replies = false) html << <<-END_HTML <div class="pure-g" style="width:100%"> <div class="channel-profile pure-u-4-24 pure-u-md-2-24"> - <img style="margin-right:1em;margin-top:1em;width:90%" src="#{author_thumbnail}"> + <img loading="lazy" style="margin-right:1em;margin-top:1em;width:90%" src="#{author_thumbnail}"> </div> <div class="pure-u-20-24 pure-u-md-22-24"> <p> @@ -349,7 +349,7 @@ def template_youtube_comments(comments, locale, thin_mode, is_replies = false) html << <<-END_HTML <div class="pure-g"> <div class="pure-u-1 pure-u-md-1-2"> - <img style="width:100%" src="/ggpht#{URI.parse(attachment["url"].as_s).request_target}"> + <img loading="lazy" style="width:100%" src="/ggpht#{URI.parse(attachment["url"].as_s).request_target}"> </div> </div> END_HTML @@ -410,7 +410,7 @@ def template_youtube_comments(comments, locale, thin_mode, is_replies = false) html << <<-END_HTML <span class="creator-heart-container" title="#{translate(locale, "`x` marked it with a ❤", child["creatorHeart"]["creatorName"].as_s)}"> <div class="creator-heart"> - <img class="creator-heart-background-hearted" src="#{creator_thumbnail}"></img> + <img loading="lazy" class="creator-heart-background-hearted" src="#{creator_thumbnail}"></img> <div class="creator-heart-small-hearted"> <div class="icon ion-ios-heart creator-heart-small-container"></div> </div> diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 91377b0d..2e61d21f 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -60,43 +60,6 @@ def html_to_content(description_html : String) return description end -def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) - extracted = extract_items(initial_data, author_fallback, author_id_fallback) - - target = [] of SearchItem - extracted.each do |i| - if i.is_a?(Category) - i.contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video } - else - target << i - end - end - return target.select(&.is_a?(SearchVideo)).map(&.as(SearchVideo)) -end - -def extract_selected_tab(tabs) - # Extract the selected tab from the array of tabs Youtube returns - return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"].as_bool)[0]["tabRenderer"] -end - -def fetch_continuation_token(items : Array(JSON::Any)) - # Fetches the continuation token from an array of items - return items.last["continuationItemRenderer"]? - .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s -end - -def fetch_continuation_token(initial_data : Hash(String, JSON::Any)) - # Fetches the continuation token from initial data - if initial_data["onResponseReceivedActions"]? - continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"] - else - tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]) - continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"] - end - - return fetch_continuation_token(continuation_items.as_a) -end - def check_enum(db, enum_name, struct_type = nil) return # TODO diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index 61356555..a9798f0c 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -237,8 +237,15 @@ class Category def to_json(locale, json : JSON::Builder) json.object do + json.field "type", "category" json.field "title", self.title - json.field "contents", self.contents + json.field "contents" do + json.array do + self.contents.each do |item| + item.to_json(locale, json) + end + end + end end end diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index af5f553b..603b4e1f 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -1,70 +1,5 @@ -require "lsquic" require "db" -def add_yt_headers(request) - request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" - request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" - request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" - request.headers["accept-language"] ||= "en-us,en;q=0.5" - return if request.resource.starts_with? "/sorry/index" - request.headers["x-youtube-client-name"] ||= "1" - request.headers["x-youtube-client-version"] ||= "2.20200609" - # Preserve original cookies and add new YT consent cookie for EU servers - request.headers["cookie"] = "#{request.headers["cookie"]?}; CONSENT=YES+" - if !CONFIG.cookies.empty? - request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" - end -end - -struct YoutubeConnectionPool - property! url : URI - property! capacity : Int32 - property! timeout : Float64 - property pool : DB::Pool(QUIC::Client | HTTP::Client) - - def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true) - @url = url - @pool = build_pool(use_quic) - end - - def client(region = nil, &block) - if region - conn = make_client(url, region) - response = yield conn - else - conn = pool.checkout - begin - response = yield conn - rescue ex - conn.close - conn = QUIC::Client.new(url) - conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET - conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC - conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" - response = yield conn - ensure - pool.release(conn) - end - end - - response - end - - private def build_pool(use_quic) - DB::Pool(QUIC::Client | HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do - if use_quic - conn = QUIC::Client.new(url) - else - conn = HTTP::Client.new(url) - end - conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET - conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC - conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" - conn - end - end -end - # See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html def ci_lower_bound(pos, n) if n == 0 @@ -85,37 +20,6 @@ def elapsed_text(elapsed) "#{(millis * 1000).round(2)}µs" end -def make_client(url : URI, region = nil) - # TODO: Migrate any applicable endpoints to QUIC - client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure) - client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC - client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" - client.read_timeout = 10.seconds - client.connect_timeout = 10.seconds - - if region - PROXY_LIST[region]?.try &.sample(40).each do |proxy| - begin - proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) - client.set_proxy(proxy) - break - rescue ex - end - end - end - - return client -end - -def make_client(url : URI, region = nil, &block) - client = make_client(url, region) - begin - yield client - ensure - client.close - end -end - def decode_length_seconds(string) length_seconds = string.gsub(/[^0-9:]/, "").split(":").map &.to_i length_seconds = [0] * (3 - length_seconds.size) + length_seconds diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr index 55b01174..63ea434f 100644 --- a/src/invidious/mixes.cr +++ b/src/invidious/mixes.cr @@ -97,7 +97,7 @@ def template_mix(mix) <li class="pure-menu-item"> <a href="/watch?v=#{video["videoId"]}&list=#{mix["mixId"]}"> <div class="thumbnail"> - <img class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg"> + <img loading="lazy" class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg"> <p class="length">#{recode_length_seconds(video["lengthSeconds"].as_i)}</p> </div> <p style="width:100%">#{video["title"]}</p> diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index f56cc2ea..7940dc1f 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -107,7 +107,7 @@ struct Playlist property updated : Time property thumbnail : String? - def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil) + def to_json(offset, locale, json : JSON::Builder, video_id : String? = nil) json.object do json.field "type", "playlist" json.field "title", self.title @@ -142,7 +142,7 @@ struct Playlist json.field "videos" do json.array do - videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation) + videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, video_id: video_id) videos.each_with_index do |video, index| video.to_json(locale, json) end @@ -151,12 +151,12 @@ struct Playlist end end - def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil) + def to_json(offset, locale, json : JSON::Builder? = nil, video_id : String? = nil) if json - to_json(offset, locale, json, continuation: continuation) + to_json(offset, locale, json, video_id: video_id) else JSON.build do |json| - to_json(offset, locale, json, continuation: continuation) + to_json(offset, locale, json, video_id: video_id) end end end @@ -196,7 +196,7 @@ struct InvidiousPlaylist end end - def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil) + def to_json(offset, locale, json : JSON::Builder, video_id : String? = nil) json.object do json.field "type", "invidiousPlaylist" json.field "title", self.title @@ -218,11 +218,11 @@ struct InvidiousPlaylist json.field "videos" do json.array do if !offset || offset == 0 - index = PG_DB.query_one?("SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1", self.id, continuation, as: Int64) + index = PG_DB.query_one?("SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1", self.id, video_id, as: Int64) offset = self.index.index(index) || 0 end - videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation) + videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, video_id: video_id) videos.each_with_index do |video, index| video.to_json(locale, json, offset + index) end @@ -231,12 +231,12 @@ struct InvidiousPlaylist end end - def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil) + def to_json(offset, locale, json : JSON::Builder? = nil, video_id : String? = nil) if json - to_json(offset, locale, json, continuation: continuation) + to_json(offset, locale, json, video_id: video_id) else JSON.build do |json| - to_json(offset, locale, json, continuation: continuation) + to_json(offset, locale, json, video_id: video_id) end end end @@ -426,7 +426,7 @@ def fetch_playlist(plid, locale) }) end -def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil) +def get_playlist_videos(db, playlist, offset, locale = nil, video_id = nil) # Show empy playlist if requested page is out of range # (e.g, when a new playlist has been created, offset will be negative) if offset >= playlist.video_count || offset < 0 @@ -437,17 +437,26 @@ def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil) db.query_all("SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3", playlist.id, playlist.index, offset, as: PlaylistVideo) else - if offset >= 100 - # Normalize offset to match youtube's behavior (100 videos chunck per request) - offset = (offset / 100).to_i64 * 100_i64 + if video_id + initial_data = YoutubeAPI.next({ + "videoId" => video_id, + "playlistId" => playlist.id, + }) + offset = initial_data.dig?("contents", "twoColumnWatchNextResults", "playlist", "playlist", "currentIndex").try &.as_i || offset + end + + videos = [] of PlaylistVideo + until videos.size >= 200 || videos.size == playlist.video_count || offset >= playlist.video_count + # 100 videos per request ctoken = produce_playlist_continuation(playlist.id, offset) initial_data = YoutubeAPI.browse(ctoken) - else - initial_data = YoutubeAPI.browse("VL" + playlist.id, params: "") + videos += extract_playlist_videos(initial_data) + + offset += 100 end - return extract_playlist_videos(initial_data) + return videos end end @@ -523,10 +532,10 @@ def template_playlist(playlist) playlist["videos"].as_a.each do |video| html += <<-END_HTML - <li class="pure-menu-item"> - <a href="/watch?v=#{video["videoId"]}&list=#{playlist["playlistId"]}"> + <li class="pure-menu-item" id="#{video["videoId"]}"> + <a href="/watch?v=#{video["videoId"]}&list=#{playlist["playlistId"]}&index=#{video["index"]}"> <div class="thumbnail"> - <img class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg"> + <img loading="lazy" class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg"> <p class="length">#{recode_length_seconds(video["lengthSeconds"].as_i)}</p> </div> <p style="width:100%">#{video["title"]}</p> diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index cf95bd9b..80b59fd5 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -24,7 +24,7 @@ module Invidious::Routes::API::V1::Misc offset ||= env.params.query["page"]?.try &.to_i?.try { |page| (page - 1) * 100 } offset ||= 0 - continuation = env.params.query["continuation"]? + video_id = env.params.query["continuation"]? format = env.params.query["format"]? format ||= "json" @@ -46,12 +46,32 @@ module Invidious::Routes::API::V1::Misc return error_json(404, "Playlist does not exist.") end - response = playlist.to_json(offset, locale, continuation: continuation) + # includes into the playlist a maximum of 20 videos, before the offset + if offset > 0 + lookback = offset < 50 ? offset : 50 + response = playlist.to_json(offset - lookback, locale) + json_response = JSON.parse(response) + else + # Unless the continuation is really the offset 0, it becomes expensive. + # It happens when the offset is not set. + # First we find the actual offset, and then we lookback + # it shouldn't happen often though + + lookback = 0 + response = playlist.to_json(offset, locale, video_id: video_id) + json_response = JSON.parse(response) + + if json_response["videos"].as_a[0]["index"] != offset + offset = json_response["videos"].as_a[0]["index"].as_i + lookback = offset < 50 ? offset : 50 + response = playlist.to_json(offset - lookback, locale) + json_response = JSON.parse(response) + end + end if format == "html" - response = JSON.parse(response) - playlist_html = template_playlist(response) - index, next_video = response["videos"].as_a.skip(1).select { |video| !video["author"].as_s.empty? }[0]?.try { |v| {v["index"], v["videoId"]} } || {nil, nil} + playlist_html = template_playlist(json_response) + index, next_video = json_response["videos"].as_a.skip(1 + lookback).select { |video| !video["author"].as_s.empty? }[0]?.try { |v| {v["index"], v["videoId"]} } || {nil, nil} response = { "playlistHtml" => playlist_html, diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr new file mode 100644 index 00000000..bb924cdf --- /dev/null +++ b/src/invidious/routes/images.cr @@ -0,0 +1,191 @@ +module Invidious::Routes::Images + # Avatars, banners and other large image assets. + def self.ggpht(env) + url = env.request.path.lchop("/ggpht") + + headers = HTTP::Headers{":authority" => "yt3.ggpht.com"} + REQUEST_HEADERS_WHITELIST.each do |header| + if env.request.headers[header]? + headers[header] = env.request.headers[header] + end + end + + begin + YT_POOL.client &.get(url, headers) do |response| + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value + end + end + + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if response.status_code >= 300 + env.response.headers.delete("Transfer-Encoding") + break + end + + proxy_file(response, env) + end + rescue ex + end + end + + def self.options_storyboard(env) + env.response.headers["Access-Control-Allow-Origin"] = "*" + env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" + env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range" + end + + def self.get_storyboard(env) + authority = env.params.url["authority"] + id = env.params.url["id"] + storyboard = env.params.url["storyboard"] + index = env.params.url["index"] + + url = "/sb/#{id}/#{storyboard}/#{index}?#{env.params.query}" + + headers = HTTP::Headers.new + + headers[":authority"] = "#{authority}.ytimg.com" + + REQUEST_HEADERS_WHITELIST.each do |header| + if env.request.headers[header]? + headers[header] = env.request.headers[header] + end + end + + begin + YT_POOL.client &.get(url, headers) do |response| + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value + end + end + + env.response.headers["Connection"] = "close" + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if response.status_code >= 300 + env.response.headers.delete("Transfer-Encoding") + break + end + + proxy_file(response, env) + end + rescue ex + end + end + + # ??? maybe also for storyboards? + def self.s_p_image(env) + id = env.params.url["id"] + name = env.params.url["name"] + + url = env.request.resource + + headers = HTTP::Headers{":authority" => "i9.ytimg.com"} + REQUEST_HEADERS_WHITELIST.each do |header| + if env.request.headers[header]? + headers[header] = env.request.headers[header] + end + end + + begin + YT_POOL.client &.get(url, headers) do |response| + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value + end + end + + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if response.status_code >= 300 && response.status_code != 404 + env.response.headers.delete("Transfer-Encoding") + break + end + + proxy_file(response, env) + end + rescue ex + end + end + + def self.yts_image(env) + headers = HTTP::Headers.new + REQUEST_HEADERS_WHITELIST.each do |header| + if env.request.headers[header]? + headers[header] = env.request.headers[header] + end + end + + begin + YT_POOL.client &.get(env.request.resource, headers) do |response| + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value + end + end + + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if response.status_code >= 300 && response.status_code != 404 + env.response.headers.delete("Transfer-Encoding") + break + end + + proxy_file(response, env) + end + rescue ex + end + end + + def self.thumbnails(env) + id = env.params.url["id"] + name = env.params.url["name"] + + headers = HTTP::Headers{":authority" => "i.ytimg.com"} + + if name == "maxres.jpg" + build_thumbnails(id).each do |thumb| + if YT_POOL.client &.head("/vi/#{id}/#{thumb[:url]}.jpg", headers).status_code == 200 + name = thumb[:url] + ".jpg" + break + end + end + end + url = "/vi/#{id}/#{name}" + + REQUEST_HEADERS_WHITELIST.each do |header| + if env.request.headers[header]? + headers[header] = env.request.headers[header] + end + end + + begin + YT_POOL.client &.get(url, headers) do |response| + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value + end + end + + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if response.status_code >= 300 && response.status_code != 404 + env.response.headers.delete("Transfer-Encoding") + break + end + + proxy_file(response, env) + end + rescue ex + end + end +end diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index b15ae255..5788bf51 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -5,7 +5,7 @@ <a href="/channel/<%= item.ucid %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <center> - <img style="width:56.25%" src="/ggpht<%= URI.parse(item.author_thumbnail).request_target.gsub(/=s\d+/, "=s176") %>"/> + <img loading="lazy" style="width:56.25%" src="/ggpht<%= URI.parse(item.author_thumbnail).request_target.gsub(/=s\d+/, "=s176") %>"/> </center> <% end %> <p dir="auto"><%= HTML.escape(item.author) %></p> @@ -23,7 +23,7 @@ <a style="width:100%" href="<%= url %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> - <img class="thumbnail" src="<%= URI.parse(item.thumbnail || "/").request_target %>"/> + <img loading="lazy" class="thumbnail" src="<%= URI.parse(item.thumbnail || "/").request_target %>"/> <p class="length"><%= number_with_separator(item.video_count) %> videos</p> </div> <% end %> @@ -36,7 +36,7 @@ <a href="/watch?v=<%= item.id %>&list=<%= item.rdid %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> - <img class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/> + <img loading="lazy" class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/> <% if item.length_seconds != 0 %> <p class="length"><%= recode_length_seconds(item.length_seconds) %></p> <% end %> @@ -48,10 +48,10 @@ <p dir="auto"><b><%= HTML.escape(item.author) %></b></p> </a> <% when PlaylistVideo %> - <a style="width:100%" href="/watch?v=<%= item.id %>&list=<%= item.plid %>"> + <a style="width:100%" href="/watch?v=<%= item.id %>&list=<%= item.plid %>&index=<%= item.index %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> - <img class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/> + <img loading="lazy" class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/> <% if plid = env.get?("remove_playlist_items") %> <form data-onsubmit="return_false" action="/playlist_ajax?action_remove_video=1&set_video_id=<%= item.index %>&playlist_id=<%= plid %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> @@ -114,7 +114,7 @@ <a style="width:100%" href="/watch?v=<%= item.id %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> - <img class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/> + <img loading="lazy" class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/> <% if env.get? "show_watched" %> <form data-onsubmit="return_false" action="/watch_ajax?action_mark_watched=1&id=<%= item.id %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index b7020598..3fb2fe18 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -149,9 +149,7 @@ <div class="pure-u-1 pure-u-md-1-3"> <span> <i class="icon ion-ios-wallet"></i> - <%= translate(locale, "footer_donate") %> - <a href="bitcoin:bc1qfhe7rq3lqzuayzjxzyt9waz9ytrs09kla3tsgr">BTC</a> / - <a href="monero:41nMCtek197boJtiUvGnTFYMatrLEpnpkQDmUECqx5Es2uX3sTKKWVhSL76suXsG3LXqkEJBrCZBgPTwJrDp1FrZJfycGPR">XMR</a> + <a href="https://invidious.io/donate/"><%= translate(locale, "footer_donate_page") %></a> </span> <span><%= translate(locale, "Current version: ") %> <%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %> @ <%= CURRENT_BRANCH %></span> </div> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 68e7eb80..398e25b6 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -303,7 +303,7 @@ we're going to need to do it here in order to allow for translations. <a href="/watch?v=<%= rv["id"] %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> - <img class="thumbnail" src="/vi/<%= rv["id"] %>/mqdefault.jpg"> + <img loading="lazy" class="thumbnail" src="/vi/<%= rv["id"] %>/mqdefault.jpg"> <p class="length"><%= recode_length_seconds(rv["length_seconds"]?.try &.to_i? || 0) %></p> </div> <% end %> diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr new file mode 100644 index 00000000..5ba2d73c --- /dev/null +++ b/src/invidious/yt_backend/connection_pool.cr @@ -0,0 +1,96 @@ +require "lsquic" + +def add_yt_headers(request) + request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" + request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" + request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + request.headers["accept-language"] ||= "en-us,en;q=0.5" + return if request.resource.starts_with? "/sorry/index" + request.headers["x-youtube-client-name"] ||= "1" + request.headers["x-youtube-client-version"] ||= "2.20200609" + # Preserve original cookies and add new YT consent cookie for EU servers + request.headers["cookie"] = "#{request.headers["cookie"]?}; CONSENT=YES+" + if !CONFIG.cookies.empty? + request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" + end +end + +struct YoutubeConnectionPool + property! url : URI + property! capacity : Int32 + property! timeout : Float64 + property pool : DB::Pool(QUIC::Client | HTTP::Client) + + def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true) + @url = url + @pool = build_pool(use_quic) + end + + def client(region = nil, &block) + if region + conn = make_client(url, region) + response = yield conn + else + conn = pool.checkout + begin + response = yield conn + rescue ex + conn.close + conn = QUIC::Client.new(url) + conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET + conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC + conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" + response = yield conn + ensure + pool.release(conn) + end + end + + response + end + + private def build_pool(use_quic) + DB::Pool(QUIC::Client | HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do + if use_quic + conn = QUIC::Client.new(url) + else + conn = HTTP::Client.new(url) + end + conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET + conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC + conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" + conn + end + end +end + +def make_client(url : URI, region = nil) + # TODO: Migrate any applicable endpoints to QUIC + client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure) + client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC + client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" + client.read_timeout = 10.seconds + client.connect_timeout = 10.seconds + + if region + PROXY_LIST[region]?.try &.sample(40).each do |proxy| + begin + proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) + client.set_proxy(proxy) + break + rescue ex + end + end + end + + return client +end + +def make_client(url : URI, region = nil, &block) + client = make_client(url, region) + begin + yield client + ensure + client.close + end +end diff --git a/src/invidious/helpers/extractors.cr b/src/invidious/yt_backend/extractors.cr index c8a6cd4a..8398ca8e 100644 --- a/src/invidious/helpers/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -43,7 +43,7 @@ private module Parsers private def self.parse(item_contents, author_fallback) video_id = item_contents["videoId"].as_s - title = extract_text(item_contents["title"]) || "" + title = extract_text(item_contents["title"]?) || "" # Extract author information if author_info = item_contents.dig?("ownerText", "runs", 0) @@ -321,11 +321,13 @@ private module Parsers content_container = item_contents["contents"] end - raw_contents = content_container["items"].as_a - raw_contents.each do |item| - result = extract_item(item) - if !result.nil? - contents << result + raw_contents = content_container["items"]?.try &.as_a + if !raw_contents.nil? + raw_contents.each do |item| + result = extract_item(item) + if !result.nil? + contents << result + end end end @@ -399,7 +401,7 @@ private module Extractors items_container = renderer_container_contents end - items_container["items"].as_a.each do |item| + items_container["items"]?.try &.as_a.each do |item| raw_items << item end end @@ -531,37 +533,6 @@ private module HelperExtractors end end -# Extracts text from InnerTube response -# -# InnerTube can package text in three different formats -# "runs": [ -# {"text": "something"}, -# {"text": "cont"}, -# ... -# ] -# -# "SimpleText": "something" -# -# Or sometimes just none at all as with the data returned from -# category continuations. -# -# In order to facilitate calling this function with `#[]?`: -# A nil will be accepted. Of course, since nil cannot be parsed, -# another nil will be returned. -def extract_text(item : JSON::Any?) : String? - if item.nil? - return nil - end - - if text_container = item["simpleText"]? - return text_container.as_s - elsif text_container = item["runs"]? - return text_container.as_a.map(&.["text"].as_s).join("") - else - nil - end -end - # Parses an item from Youtube's JSON response into a more usable structure. # The end result can either be a SearchVideo, SearchPlaylist or SearchChannel. def extract_item(item : JSON::Any, author_fallback : String? = "", diff --git a/src/invidious/yt_backend/extractors_utils.cr b/src/invidious/yt_backend/extractors_utils.cr new file mode 100644 index 00000000..97cc0997 --- /dev/null +++ b/src/invidious/yt_backend/extractors_utils.cr @@ -0,0 +1,67 @@ +# Extracts text from InnerTube response +# +# InnerTube can package text in three different formats +# "runs": [ +# {"text": "something"}, +# {"text": "cont"}, +# ... +# ] +# +# "SimpleText": "something" +# +# Or sometimes just none at all as with the data returned from +# category continuations. +# +# In order to facilitate calling this function with `#[]?`: +# A nil will be accepted. Of course, since nil cannot be parsed, +# another nil will be returned. +def extract_text(item : JSON::Any?) : String? + if item.nil? + return nil + end + + if text_container = item["simpleText"]? + return text_container.as_s + elsif text_container = item["runs"]? + return text_container.as_a.map(&.["text"].as_s).join("") + else + nil + end +end + +def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) + extracted = extract_items(initial_data, author_fallback, author_id_fallback) + + target = [] of SearchItem + extracted.each do |i| + if i.is_a?(Category) + i.contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video } + else + target << i + end + end + return target.select(&.is_a?(SearchVideo)).map(&.as(SearchVideo)) +end + +def extract_selected_tab(tabs) + # Extract the selected tab from the array of tabs Youtube returns + return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"].as_bool)[0]["tabRenderer"] +end + +def fetch_continuation_token(items : Array(JSON::Any)) + # Fetches the continuation token from an array of items + return items.last["continuationItemRenderer"]? + .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s +end + +def fetch_continuation_token(initial_data : Hash(String, JSON::Any)) + # Fetches the continuation token from initial data + if initial_data["onResponseReceivedActions"]? + continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"] + else + tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]) + continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"] + end + + return fetch_continuation_token(continuation_items.as_a) +end diff --git a/src/invidious/helpers/proxy.cr b/src/invidious/yt_backend/proxy.cr index 3418d887..3418d887 100644 --- a/src/invidious/helpers/proxy.cr +++ b/src/invidious/yt_backend/proxy.cr diff --git a/src/invidious/helpers/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index b3815f6a..b3815f6a 100644 --- a/src/invidious/helpers/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr |