diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/invidious.cr | 10 | ||||
| -rw-r--r-- | src/invidious/config.cr | 13 | ||||
| -rw-r--r-- | src/invidious/helpers/crystal_class_overrides.cr | 34 | ||||
| -rw-r--r-- | src/invidious/helpers/i18n.cr | 15 | ||||
| -rw-r--r-- | src/invidious/helpers/logger.cr | 20 | ||||
| -rw-r--r-- | src/invidious/helpers/sig_helper.cr | 23 | ||||
| -rw-r--r-- | src/invidious/routes/api/v1/search.cr | 4 | ||||
| -rw-r--r-- | src/invidious/routes/images.cr | 106 | ||||
| -rw-r--r-- | src/invidious/routes/video_playback.cr | 8 | ||||
| -rw-r--r-- | src/invidious/videos/caption.cr | 1 | ||||
| -rw-r--r-- | src/invidious/yt_backend/connection_pool.cr | 91 | ||||
| -rw-r--r-- | src/invidious/yt_backend/youtube_api.cr | 5 |
12 files changed, 209 insertions, 121 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index 63f2a9cc..b422dcbb 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -23,6 +23,7 @@ require "kilt" require "./ext/kemal_content_for.cr" require "./ext/kemal_static_file_handler.cr" +require "http_proxy" require "athena-negotiation" require "openssl/hmac" require "option_parser" @@ -92,6 +93,10 @@ SOFTWARE = { YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size) +# Image request pool + +GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size) + # CLI Kemal.config.extra_options do |parser| parser.banner = "Usage: invidious [arguments]" @@ -117,6 +122,9 @@ Kemal.config.extra_options do |parser| parser.on("-l LEVEL", "--log-level=LEVEL", "Log level, one of #{LogLevel.values} (default: #{CONFIG.log_level})") do |log_level| CONFIG.log_level = LogLevel.parse(log_level) end + parser.on("-k", "--colorize", "Colorize logs") do + CONFIG.colorize_logs = true + end parser.on("-v", "--version", "Print version") do puts SOFTWARE.to_pretty_json exit @@ -133,7 +141,7 @@ if CONFIG.output.upcase != "STDOUT" FileUtils.mkdir_p(File.dirname(CONFIG.output)) end OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mode: "a") -LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) +LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level, CONFIG.colorize_logs) # Check table integrity Invidious::Database.check_integrity(CONFIG) diff --git a/src/invidious/config.cr b/src/invidious/config.cr index a097b7f1..c4ca622f 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -55,6 +55,15 @@ struct ConfigPreferences end end +struct HTTPProxyConfig + include YAML::Serializable + + property user : String + property password : String + property host : String + property port : Int32 +end + class Config include YAML::Serializable @@ -69,6 +78,8 @@ class Config property output : String = "STDOUT" # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr property log_level : LogLevel = LogLevel::Info + # Enables colors in logs. Useful for debugging purposes + property colorize_logs : Bool = false # Database configuration with separate parameters (username, hostname, etc) property db : DBConfig? = nil @@ -129,6 +140,8 @@ class Config property host_binding : String = "0.0.0.0" # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) property pool_size : Int32 = 100 + # HTTP Proxy configuration + property http_proxy : HTTPProxyConfig? = nil # Use Innertube's transcripts API instead of timedtext for closed captions property use_innertube_for_captions : Bool = false diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr index fec3f62c..3040d7a0 100644 --- a/src/invidious/helpers/crystal_class_overrides.cr +++ b/src/invidious/helpers/crystal_class_overrides.cr @@ -18,6 +18,40 @@ end class HTTP::Client property family : Socket::Family = Socket::Family::UNSPEC + # Override stdlib to automatically initialize proxy if configured + # + # Accurate as of crystal 1.12.1 + + def initialize(@host : String, port = nil, tls : TLSContext = nil) + check_host_only(@host) + + {% if flag?(:without_openssl) %} + if tls + raise "HTTP::Client TLS is disabled because `-D without_openssl` was passed at compile time" + end + @tls = nil + {% else %} + @tls = case tls + when true + OpenSSL::SSL::Context::Client.new + when OpenSSL::SSL::Context::Client + tls + when false, nil + nil + end + {% end %} + + @port = (port || (@tls ? 443 : 80)).to_i + + self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + end + + def initialize(@io : IO, @host = "", @port = 80) + @reconnect = false + + self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + end + private def io io = @io return io if io diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index 23a1aafc..1ba3ea61 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -1,8 +1,22 @@ +# Languages requiring a better level of translation (at least 20%) +# to be added to the list below: +# +# "af" => "", # Afrikaans +# "az" => "", # Azerbaijani +# "be" => "", # Belarusian +# "bn_BD" => "", # Bengali (Bangladesh) +# "ia" => "", # Interlingua +# "or" => "", # Odia +# "tk" => "", # Turkmen +# "tok => "", # Toki Pona +# LOCALES_LIST = { "ar" => "العربية", # Arabic + "bg" => "български", # Bulgarian "bn" => "বাংলা", # Bengali "ca" => "Català", # Catalan "cs" => "Čeština", # Czech + "cy" => "Cymraeg", # Welsh "da" => "Dansk", # Danish "de" => "Deutsch", # German "el" => "Ελληνικά", # Greek @@ -23,6 +37,7 @@ LOCALES_LIST = { "it" => "Italiano", # Italian "ja" => "日本語", # Japanese "ko" => "한국어", # Korean + "lmo" => "Lombard", # Lombard "lt" => "Lietuvių", # Lithuanian "nb-NO" => "Norsk bokmål", # Norwegian Bokmål "nl" => "Nederlands", # Dutch diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr index b443073e..03349595 100644 --- a/src/invidious/helpers/logger.cr +++ b/src/invidious/helpers/logger.cr @@ -1,3 +1,5 @@ +require "colorize" + enum LogLevel All = 0 Trace = 1 @@ -10,7 +12,9 @@ enum LogLevel end class Invidious::LogHandler < Kemal::BaseLogHandler - def initialize(@io : IO = STDOUT, @level = LogLevel::Debug) + def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, use_color : Bool = true) + Colorize.enabled = use_color + Colorize.on_tty_only! end def call(context : HTTP::Server::Context) @@ -39,10 +43,22 @@ class Invidious::LogHandler < Kemal::BaseLogHandler @io.flush end + def color(level) + case level + when LogLevel::Trace then :cyan + when LogLevel::Debug then :green + when LogLevel::Info then :white + when LogLevel::Warn then :yellow + when LogLevel::Error then :red + when LogLevel::Fatal then :magenta + else :default + end + end + {% for level in %w(trace debug info warn error fatal) %} def {{level.id}}(message : String) if LogLevel::{{level.id.capitalize}} >= @level - puts("#{Time.utc} [{{level.id}}] #{message}") + puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}}))) end end {% end %} diff --git a/src/invidious/helpers/sig_helper.cr b/src/invidious/helpers/sig_helper.cr index 9e72c1c7..6d198a42 100644 --- a/src/invidious/helpers/sig_helper.cr +++ b/src/invidious/helpers/sig_helper.cr @@ -175,8 +175,9 @@ module Invidious::SigHelper @queue = {} of TransactionID => Transaction @conn : Connection + @uri_or_path : String - def initialize(uri_or_path) + def initialize(@uri_or_path) @conn = Connection.new(uri_or_path) listen end @@ -186,10 +187,26 @@ module Invidious::SigHelper LOGGER.debug("SigHelper: Multiplexor listening") - # TODO: reopen socket if unexpectedly closed spawn do loop do - receive_data + begin + receive_data + rescue ex + LOGGER.info("SigHelper: Connection to helper died with '#{ex.message}' trying to reconnect...") + # We close the socket because for some reason is not closed. + @conn.close + loop do + begin + @conn = Connection.new(@uri_or_path) + LOGGER.info("SigHelper: Reconnected to SigHelper!") + rescue ex + LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}'. Retrying") + sleep 500.milliseconds + next + end + break if !@conn.closed? + end + end Fiber.yield end end diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index 2922b060..59a30745 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -31,9 +31,7 @@ module Invidious::Routes::API::V1::Search query = env.params.query["q"]? || "" begin - client = HTTP::Client.new("suggestqueries-clients6.youtube.com") - client.before_request { |r| add_yt_headers(r) } - + client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers: true) url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt" response = client.get(url).body diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index b6a2e110..639697db 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -11,29 +11,9 @@ module Invidious::Routes::Images end end - # We're encapsulating this into a proc in order to easily reuse this - # portion of the code for each request block below. - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - env.response.headers.delete("Transfer-Encoding") - return - end - - proxy_file(response, env) - } - begin - HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| - return request_proc.call(resp) + GGPHT_POOL.client &.get(url, headers) do |resp| + return self.proxy_image(env, resp) end rescue ex end @@ -61,27 +41,10 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Connection"] = "close" - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - return env.response.headers.delete("Transfer-Encoding") - end - - proxy_file(response, env) - } - begin - HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| - return request_proc.call(resp) + get_ytimg_pool(authority).client &.get(url, headers) do |resp| + env.response.headers["Connection"] = "close" + return self.proxy_image(env, resp) end rescue ex end @@ -101,26 +64,9 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - return env.response.headers.delete("Transfer-Encoding") - end - - proxy_file(response, env) - } - begin - HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| - return request_proc.call(resp) + get_ytimg_pool("i9").client &.get(url, headers) do |resp| + return self.proxy_image(env, resp) end rescue ex end @@ -165,8 +111,7 @@ module Invidious::Routes::Images if name == "maxres.jpg" build_thumbnails(id).each do |thumb| thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" - # This can likely be optimized into a (small) pool sometime in the future. - if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 + if get_ytimg_pool("i9").client &.head(thumbnail_resource_path, headers).status_code == 200 name = thumb[:url] + ".jpg" break end @@ -181,29 +126,28 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end + begin + get_ytimg_pool("i").client &.get(url, headers) do |resp| + return self.proxy_image(env, resp) end + rescue ex + end + end - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - return env.response.headers.delete("Transfer-Encoding") + private def self.proxy_image(env, response) + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value end + end - proxy_file(response, env) - } + env.response.headers["Access-Control-Allow-Origin"] = "*" - begin - # This can likely be optimized into a (small) pool sometime in the future. - HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end - rescue ex + if response.status_code >= 300 + return env.response.headers.delete("Transfer-Encoding") end + + return proxy_file(response, env) end end diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 24693662..26852d06 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -42,7 +42,7 @@ module Invidious::Routes::VideoPlayback headers["Range"] = "bytes=#{range_for_head}" end - client = make_client(URI.parse(host), region, force_resolve = true) + client = make_client(URI.parse(host), region, force_resolve: true) response = HTTP::Client::Response.new(500) error = "" 5.times do @@ -57,7 +57,7 @@ module Invidious::Routes::VideoPlayback if new_host != host host = new_host client.close - client = make_client(URI.parse(new_host), region, force_resolve = true) + client = make_client(URI.parse(new_host), region, force_resolve: true) end url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" @@ -71,7 +71,7 @@ module Invidious::Routes::VideoPlayback fvip = "3" host = "https://r#{fvip}---#{mn}.googlevideo.com" - client = make_client(URI.parse(host), region, force_resolve = true) + client = make_client(URI.parse(host), region, force_resolve: true) rescue ex error = ex.message end @@ -196,7 +196,7 @@ module Invidious::Routes::VideoPlayback break else client.close - client = make_client(URI.parse(host), region, force_resolve = true) + client = make_client(URI.parse(host), region, force_resolve: true) end end diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr index 484e61d2..c811cfe1 100644 --- a/src/invidious/videos/caption.cr +++ b/src/invidious/videos/caption.cr @@ -123,6 +123,7 @@ module Invidious::Videos "Esperanto", "Estonian", "Filipino", + "Filipino (auto-generated)", "Finnish", "French", "French (auto-generated)", diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index ca612083..c4a73aa7 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -1,17 +1,6 @@ -def add_yt_headers(request) - request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" - request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" - - request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" - request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" - request.headers["Accept-Language"] ||= "en-us,en;q=0.5" - - # Preserve original cookies and add new YT consent cookie for EU servers - request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}" - if !CONFIG.cookies.empty? - request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" - end -end +# Mapping of subdomain => YoutubeConnectionPool +# This is needed as we may need to access arbitrary subdomains of ytimg +private YTIMG_POOLS = {} of String => YoutubeConnectionPool struct YoutubeConnectionPool property! url : URI @@ -26,15 +15,15 @@ struct YoutubeConnectionPool def client(&) conn = pool.checkout + # Proxy needs to be reinstated every time we get a client from the pool + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + begin response = yield conn rescue ex conn.close - conn = HTTP::Client.new(url) + conn = make_client(url, force_resolve: true) - conn.family = CONFIG.force_resolve - conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC - conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" response = yield conn ensure pool.release(conn) @@ -44,25 +33,45 @@ struct YoutubeConnectionPool end private def build_pool - DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do - conn = HTTP::Client.new(url) - conn.family = CONFIG.force_resolve - conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC - conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" - conn + options = DB::Pool::Options.new( + initial_pool_size: 0, + max_pool_size: capacity, + max_idle_pool_size: capacity, + checkout_timeout: timeout + ) + + DB::Pool(HTTP::Client).new(options) do + next make_client(url, force_resolve: true) end end end -def make_client(url : URI, region = nil, force_resolve : Bool = false) +def add_yt_headers(request) + request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" + request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" + + request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" + request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + request.headers["Accept-Language"] ||= "en-us,en;q=0.5" + + # Preserve original cookies and add new YT consent cookie for EU servers + request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}" + if !CONFIG.cookies.empty? + request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" + end +end + +def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false) client = HTTP::Client.new(url) + client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy # Force the usage of a specific configured IP Family if force_resolve client.family = CONFIG.force_resolve + client.family = Socket::Family::INET if client.family == Socket::Family::UNSPEC end - client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" + client.before_request { |r| add_yt_headers(r) } if url.host.try &.ends_with?("youtube.com") || force_youtube_headers client.read_timeout = 10.seconds client.connect_timeout = 10.seconds @@ -70,10 +79,38 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false) end def make_client(url : URI, region = nil, force_resolve : Bool = false, &) - client = make_client(url, region, force_resolve) + client = make_client(url, region, force_resolve: force_resolve) begin yield client ensure client.close end end + +def make_configured_http_proxy_client + # This method is only called when configuration for an HTTP proxy are set + config_proxy = CONFIG.http_proxy.not_nil! + + return HTTP::Proxy::Client.new( + config_proxy.host, + config_proxy.port, + + username: config_proxy.user, + password: config_proxy.password, + ) +end + +# Fetches a HTTP pool for the specified subdomain of ytimg.com +# +# Creates a new one when the specified pool for the subdomain does not exist +def get_ytimg_pool(subdomain) + if pool = YTIMG_POOLS[subdomain]? + return pool + else + LOGGER.info("ytimg_pool: Creating a new HTTP pool for \"https://#{subdomain}.ytimg.com\"") + pool = YoutubeConnectionPool.new(URI.parse("https://#{subdomain}.ytimg.com"), capacity: CONFIG.pool_size) + YTIMG_POOLS[subdomain] = pool + + return pool + end +end diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index baa3cd92..e0a3181f 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -638,6 +638,11 @@ module YoutubeAPI # Send the POST request body = YT_POOL.client() do |client| client.post(url, headers: headers, body: data.to_json) do |response| + if response.status_code != 200 + raise InfoException.new("Error: non 200 status code. Youtube API returned \ + status code #{response.status_code}. See <a href=\"https://docs.invidious.io/youtube-errors-explained/\"> \ + https://docs.invidious.io/youtube-errors-explained/</a> for troubleshooting.") + end self._decompress(response.body_io, response.headers["Content-Encoding"]?) end end |