diff options
| -rw-r--r-- | .github/workflows/ci.yml | 8 | ||||
| -rw-r--r-- | assets/css/player.css | 1 | ||||
| -rw-r--r-- | config/config.example.yml | 11 | ||||
| m--------- | mocks | 0 | ||||
| -rw-r--r-- | shard.lock | 8 | ||||
| -rw-r--r-- | shard.yml | 3 | ||||
| -rw-r--r-- | spec/invidious/videos/regular_videos_extract_spec.cr | 42 | ||||
| -rw-r--r-- | src/invidious.cr | 5 | ||||
| -rw-r--r-- | src/invidious/config.cr | 11 | ||||
| -rw-r--r-- | src/invidious/helpers/crystal_class_overrides.cr | 34 | ||||
| -rw-r--r-- | src/invidious/helpers/sig_helper.cr | 23 | ||||
| -rw-r--r-- | src/invidious/routes/images.cr | 106 | ||||
| -rw-r--r-- | src/invidious/yt_backend/connection_pool.cr | 66 |
13 files changed, 196 insertions, 122 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de538915..411ec769 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,10 +38,11 @@ jobs: matrix: stable: [true] crystal: - - 1.9.2 - 1.10.1 - 1.11.2 - 1.12.1 + - 1.13.2 + - 1.14.0 include: - crystal: nightly stable: false @@ -51,6 +52,11 @@ jobs: with: submodules: true + - name: Install required APT packages + run: | + sudo apt install -y libsqlite3-dev + shell: bash + - name: Install Crystal uses: crystal-lang/install-crystal@v1.8.0 with: diff --git a/assets/css/player.css b/assets/css/player.css index 50c7a748..9cb400ad 100644 --- a/assets/css/player.css +++ b/assets/css/player.css @@ -68,6 +68,7 @@ .video-js.player-style-youtube .vjs-menu-button-popup .vjs-menu { margin-bottom: 2em; + padding-top: 2em } .video-js.player-style-youtube .vjs-progress-control .vjs-progress-holder, .video-js.player-style-youtube .vjs-progress-control {height: 5px; diff --git a/config/config.example.yml b/config/config.example.yml index e9eebfde..759b81e0 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -173,6 +173,17 @@ https_only: false ## #force_resolve: +## +## Configuration for using a HTTP proxy +## +## If unset, then no HTTP proxy will be used. +## +http_proxy: + user: + password: + host: + port: + ## ## Use Innertube's transcripts API instead of timedtext for closed captions diff --git a/mocks b/mocks -Subproject 11ec372f72747c09d48ffef04843f72be67d5b5 +Subproject b55d58dea94f7144ff0205857dfa70ec14eaa87 @@ -10,7 +10,7 @@ shards: backtracer: git: https://github.com/sija/backtracer.cr.git - version: 1.2.1 + version: 1.2.2 db: git: https://github.com/crystal-lang/crystal-db.git @@ -20,6 +20,10 @@ shards: git: https://github.com/crystal-loot/exception_page.git version: 0.2.2 + http_proxy: + git: https://github.com/mamantoha/http_proxy.git + version: 0.10.3 + kemal: git: https://github.com/kemalcr/kemal.git version: 1.1.2 @@ -42,7 +46,7 @@ shards: spectator: git: https://github.com/icy-arctic-fox/spectator.git - version: 0.10.4 + version: 0.10.6 sqlite3: git: https://github.com/crystal-lang/crystal-sqlite3.git @@ -28,6 +28,9 @@ dependencies: athena-negotiation: github: athena-framework/negotiation version: ~> 0.1.1 + http_proxy: + github: mamantoha/http_proxy + version: ~> 0.10.3 development_dependencies: spectator: diff --git a/spec/invidious/videos/regular_videos_extract_spec.cr b/spec/invidious/videos/regular_videos_extract_spec.cr index c647c1d1..f96703f6 100644 --- a/spec/invidious/videos/regular_videos_extract_spec.cr +++ b/spec/invidious/videos/regular_videos_extract_spec.cr @@ -17,8 +17,8 @@ Spectator.describe "parse_video_info" do # Basic video infos expect(info["title"].as_s).to eq("I Gave My 100,000,000th Subscriber An Island") - expect(info["views"].as_i).to eq(126_573_823) - expect(info["likes"].as_i).to eq(5_157_654) + expect(info["views"].as_i).to eq(220_226_287) + expect(info["likes"].as_i).to eq(6_870_691) # For some reason the video length from VideoDetails and the # one from microformat differs by 1s... @@ -48,12 +48,12 @@ Spectator.describe "parse_video_info" do expect(info["relatedVideos"].as_a.size).to eq(20) - expect(info["relatedVideos"][0]["id"]).to eq("Hwybp38GnZw") - expect(info["relatedVideos"][0]["title"]).to eq("I Built Willy Wonka's Chocolate Factory!") + expect(info["relatedVideos"][0]["id"]).to eq("krsBRQbOPQ4") + expect(info["relatedVideos"][0]["title"]).to eq("$1 vs $250,000,000 Private Island!") expect(info["relatedVideos"][0]["author"]).to eq("MrBeast") expect(info["relatedVideos"][0]["ucid"]).to eq("UCX6OQ3DkcsbYNE6H8uQQuVA") - expect(info["relatedVideos"][0]["view_count"]).to eq("179877630") - expect(info["relatedVideos"][0]["short_view_count"]).to eq("179M") + expect(info["relatedVideos"][0]["view_count"]).to eq("230617484") + expect(info["relatedVideos"][0]["short_view_count"]).to eq("230M") expect(info["relatedVideos"][0]["author_verified"]).to eq("true") # Description @@ -76,11 +76,11 @@ Spectator.describe "parse_video_info" do expect(info["ucid"].as_s).to eq("UCX6OQ3DkcsbYNE6H8uQQuVA") expect(info["authorThumbnail"].as_s).to eq( - "https://yt3.ggpht.com/ytc/AL5GRJVuqw82ERvHzsmBxL7avr1dpBtsVIXcEzBPZaloFg=s48-c-k-c0x00ffffff-no-rj" + "https://yt3.ggpht.com/fxGKYucJAVme-Yz4fsdCroCFCrANWqw0ql4GYuvx8Uq4l_euNJHgE-w9MTkLQA805vWCi-kE0g=s48-c-k-c0x00ffffff-no-rj" ) expect(info["authorVerified"].as_bool).to be_true - expect(info["subCountText"].as_s).to eq("143M") + expect(info["subCountText"].as_s).to eq("320M") end it "parses a regular video with no descrition/comments" do @@ -99,8 +99,8 @@ Spectator.describe "parse_video_info" do # Basic video infos expect(info["title"].as_s).to eq("Chris Rea - Auberge") - expect(info["views"].as_i).to eq(10_943_126) - expect(info["likes"].as_i).to eq(0) + expect(info["views"].as_i).to eq(14_324_584) + expect(info["likes"].as_i).to eq(35_870) expect(info["lengthSeconds"].as_i).to eq(283_i64) expect(info["published"].as_s).to eq("2012-05-21T00:00:00Z") @@ -132,14 +132,14 @@ Spectator.describe "parse_video_info" do # Related videos - expect(info["relatedVideos"].as_a.size).to eq(19) + expect(info["relatedVideos"].as_a.size).to eq(20) - expect(info["relatedVideos"][0]["id"]).to eq("Ww3KeZ2_Yv4") - expect(info["relatedVideos"][0]["title"]).to eq("Chris Rea") - expect(info["relatedVideos"][0]["author"]).to eq("PanMusic") - expect(info["relatedVideos"][0]["ucid"]).to eq("UCsKAPSuh1iNbLWUga_igPyA") - expect(info["relatedVideos"][0]["view_count"]).to eq("31581") - expect(info["relatedVideos"][0]["short_view_count"]).to eq("31K") + expect(info["relatedVideos"][0]["id"]).to eq("gUUdQfnshJ4") + expect(info["relatedVideos"][0]["title"]).to eq("Chris Rea - The Road To Hell 1989 Full Version") + expect(info["relatedVideos"][0]["author"]).to eq("NEA ZIXNH") + expect(info["relatedVideos"][0]["ucid"]).to eq("UCYMEOGcvav3gCgImK2J07CQ") + expect(info["relatedVideos"][0]["view_count"]).to eq("53298661") + expect(info["relatedVideos"][0]["short_view_count"]).to eq("53M") expect(info["relatedVideos"][0]["author_verified"]).to eq("false") # Description @@ -156,11 +156,13 @@ Spectator.describe "parse_video_info" do # Author infos - expect(info["author"].as_s).to eq("ChrisReaOfficial") + expect(info["author"].as_s).to eq("ChrisReaVideos") expect(info["ucid"].as_s).to eq("UC_5q6nWPbD30-y6oiWF_oNA") - expect(info["authorThumbnail"].as_s).to be_empty + expect(info["authorThumbnail"].as_s).to eq( + "https://yt3.ggpht.com/ytc/AIdro_n71nsegpKfjeRKwn1JJmK5IVMh_7j5m_h3_1KnUUg=s48-c-k-c0x00ffffff-no-rj" + ) expect(info["authorVerified"].as_bool).to be_false - expect(info["subCountText"].as_s).to eq("-") + expect(info["subCountText"].as_s).to eq("3.11K") end end diff --git a/src/invidious.cr b/src/invidious.cr index 63f2a9cc..56aca802 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -23,6 +23,7 @@ require "kilt" require "./ext/kemal_content_for.cr" require "./ext/kemal_static_file_handler.cr" +require "http_proxy" require "athena-negotiation" require "openssl/hmac" require "option_parser" @@ -92,6 +93,10 @@ SOFTWARE = { YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size) +# Image request pool + +GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size) + # CLI Kemal.config.extra_options do |parser| parser.banner = "Usage: invidious [arguments]" diff --git a/src/invidious/config.cr b/src/invidious/config.cr index a097b7f1..c1766fbb 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -55,6 +55,15 @@ struct ConfigPreferences end end +struct HTTPProxyConfig + include YAML::Serializable + + property user : String + property password : String + property host : String + property port : Int32 +end + class Config include YAML::Serializable @@ -129,6 +138,8 @@ class Config property host_binding : String = "0.0.0.0" # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) property pool_size : Int32 = 100 + # HTTP Proxy configuration + property http_proxy : HTTPProxyConfig? = nil # Use Innertube's transcripts API instead of timedtext for closed captions property use_innertube_for_captions : Bool = false diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr index fec3f62c..3040d7a0 100644 --- a/src/invidious/helpers/crystal_class_overrides.cr +++ b/src/invidious/helpers/crystal_class_overrides.cr @@ -18,6 +18,40 @@ end class HTTP::Client property family : Socket::Family = Socket::Family::UNSPEC + # Override stdlib to automatically initialize proxy if configured + # + # Accurate as of crystal 1.12.1 + + def initialize(@host : String, port = nil, tls : TLSContext = nil) + check_host_only(@host) + + {% if flag?(:without_openssl) %} + if tls + raise "HTTP::Client TLS is disabled because `-D without_openssl` was passed at compile time" + end + @tls = nil + {% else %} + @tls = case tls + when true + OpenSSL::SSL::Context::Client.new + when OpenSSL::SSL::Context::Client + tls + when false, nil + nil + end + {% end %} + + @port = (port || (@tls ? 443 : 80)).to_i + + self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + end + + def initialize(@io : IO, @host = "", @port = 80) + @reconnect = false + + self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + end + private def io io = @io return io if io diff --git a/src/invidious/helpers/sig_helper.cr b/src/invidious/helpers/sig_helper.cr index 9e72c1c7..6d198a42 100644 --- a/src/invidious/helpers/sig_helper.cr +++ b/src/invidious/helpers/sig_helper.cr @@ -175,8 +175,9 @@ module Invidious::SigHelper @queue = {} of TransactionID => Transaction @conn : Connection + @uri_or_path : String - def initialize(uri_or_path) + def initialize(@uri_or_path) @conn = Connection.new(uri_or_path) listen end @@ -186,10 +187,26 @@ module Invidious::SigHelper LOGGER.debug("SigHelper: Multiplexor listening") - # TODO: reopen socket if unexpectedly closed spawn do loop do - receive_data + begin + receive_data + rescue ex + LOGGER.info("SigHelper: Connection to helper died with '#{ex.message}' trying to reconnect...") + # We close the socket because for some reason is not closed. + @conn.close + loop do + begin + @conn = Connection.new(@uri_or_path) + LOGGER.info("SigHelper: Reconnected to SigHelper!") + rescue ex + LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}'. Retrying") + sleep 500.milliseconds + next + end + break if !@conn.closed? + end + end Fiber.yield end end diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index b6a2e110..639697db 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -11,29 +11,9 @@ module Invidious::Routes::Images end end - # We're encapsulating this into a proc in order to easily reuse this - # portion of the code for each request block below. - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - env.response.headers.delete("Transfer-Encoding") - return - end - - proxy_file(response, env) - } - begin - HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| - return request_proc.call(resp) + GGPHT_POOL.client &.get(url, headers) do |resp| + return self.proxy_image(env, resp) end rescue ex end @@ -61,27 +41,10 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Connection"] = "close" - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - return env.response.headers.delete("Transfer-Encoding") - end - - proxy_file(response, env) - } - begin - HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| - return request_proc.call(resp) + get_ytimg_pool(authority).client &.get(url, headers) do |resp| + env.response.headers["Connection"] = "close" + return self.proxy_image(env, resp) end rescue ex end @@ -101,26 +64,9 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - return env.response.headers.delete("Transfer-Encoding") - end - - proxy_file(response, env) - } - begin - HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| - return request_proc.call(resp) + get_ytimg_pool("i9").client &.get(url, headers) do |resp| + return self.proxy_image(env, resp) end rescue ex end @@ -165,8 +111,7 @@ module Invidious::Routes::Images if name == "maxres.jpg" build_thumbnails(id).each do |thumb| thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" - # This can likely be optimized into a (small) pool sometime in the future. - if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 + if get_ytimg_pool("i9").client &.head(thumbnail_resource_path, headers).status_code == 200 name = thumb[:url] + ".jpg" break end @@ -181,29 +126,28 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end + begin + get_ytimg_pool("i").client &.get(url, headers) do |resp| + return self.proxy_image(env, resp) end + rescue ex + end + end - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - return env.response.headers.delete("Transfer-Encoding") + private def self.proxy_image(env, response) + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value end + end - proxy_file(response, env) - } + env.response.headers["Access-Control-Allow-Origin"] = "*" - begin - # This can likely be optimized into a (small) pool sometime in the future. - HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end - rescue ex + if response.status_code >= 300 + return env.response.headers.delete("Transfer-Encoding") end + + return proxy_file(response, env) end end diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index ca612083..c7c4c675 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -1,17 +1,6 @@ -def add_yt_headers(request) - request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" - request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" - - request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" - request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" - request.headers["Accept-Language"] ||= "en-us,en;q=0.5" - - # Preserve original cookies and add new YT consent cookie for EU servers - request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}" - if !CONFIG.cookies.empty? - request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" - end -end +# Mapping of subdomain => YoutubeConnectionPool +# This is needed as we may need to access arbitrary subdomains of ytimg +private YTIMG_POOLS = {} of String => YoutubeConnectionPool struct YoutubeConnectionPool property! url : URI @@ -26,12 +15,16 @@ struct YoutubeConnectionPool def client(&) conn = pool.checkout + # Proxy needs to be reinstated every time we get a client from the pool + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + begin response = yield conn rescue ex conn.close - conn = HTTP::Client.new(url) + conn = HTTP::Client.new(url) + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -54,6 +47,21 @@ struct YoutubeConnectionPool end end +def add_yt_headers(request) + request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" + request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" + + request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" + request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + request.headers["Accept-Language"] ||= "en-us,en;q=0.5" + + # Preserve original cookies and add new YT consent cookie for EU servers + request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}" + if !CONFIG.cookies.empty? + request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" + end +end + def make_client(url : URI, region = nil, force_resolve : Bool = false) client = HTTP::Client.new(url) @@ -77,3 +85,31 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, &) client.close end end + +def make_configured_http_proxy_client + # This method is only called when configuration for an HTTP proxy are set + config_proxy = CONFIG.http_proxy.not_nil! + + return HTTP::Proxy::Client.new( + config_proxy.host, + config_proxy.port, + + username: config_proxy.user, + password: config_proxy.password, + ) +end + +# Fetches a HTTP pool for the specified subdomain of ytimg.com +# +# Creates a new one when the specified pool for the subdomain does not exist +def get_ytimg_pool(subdomain) + if pool = YTIMG_POOLS[subdomain]? + return pool + else + LOGGER.info("ytimg_pool: Creating a new HTTP pool for \"https://#{subdomain}.ytimg.com\"") + pool = YoutubeConnectionPool.new(URI.parse("https://#{subdomain}.ytimg.com"), capacity: CONFIG.pool_size) + YTIMG_POOLS[subdomain] = pool + + return pool + end +end |