From eb8fcc9e88bf71694e7d9e40600ef3fd334bc46e Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 25 Aug 2023 16:08:02 -0700 Subject: Add support for using HTTP proxies --- src/invidious.cr | 1 + src/invidious/config.cr | 11 +++++++++++ src/invidious/yt_backend/connection_pool.cr | 18 ++++++++++++++++++ 3 files changed, 30 insertions(+) (limited to 'src') diff --git a/src/invidious.cr b/src/invidious.cr index e0bd0101..d4114386 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -23,6 +23,7 @@ require "kilt" require "./ext/kemal_content_for.cr" require "./ext/kemal_static_file_handler.cr" +require "http_proxy" require "athena-negotiation" require "openssl/hmac" require "option_parser" diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 09c2168b..e12054d0 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -54,6 +54,15 @@ struct ConfigPreferences end end +struct HTTPProxyConfig + include YAML::Serializable + + property user : String + property password : String + property host : String + property port : Int32 +end + class Config include YAML::Serializable @@ -126,6 +135,8 @@ class Config property host_binding : String = "0.0.0.0" # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) property pool_size : Int32 = 100 + # HTTP Proxy configuration + property http_proxy : HTTPProxyConfig? = nil # Use Innertube's transcripts API instead of timedtext for closed captions property use_innertube_for_captions : Bool = false diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index d3dbcc0e..0e4d8aff 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -26,12 +26,14 @@ struct YoutubeConnectionPool def client(&block) conn = pool.checkout + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy begin response = yield conn rescue ex conn.close conn = HTTP::Client.new(url) + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -46,6 +48,7 @@ struct YoutubeConnectionPool private def build_pool DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do conn = HTTP::Client.new(url) + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -66,6 +69,8 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false) client.read_timeout = 10.seconds client.connect_timeout = 10.seconds + client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + return client end @@ -77,3 +82,16 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, &block) client.close end end + +def make_configured_http_proxy_client + # This method is only called when configuration for an HTTP proxy are set + config_proxy = CONFIG.http_proxy.not_nil! + + return HTTP::Proxy::Client.new( + config_proxy.host, + config_proxy.port, + + username: config_proxy.user, + password: config_proxy.password, + ) +end -- cgit v1.2.3 From 3b471ae964ad0122c81205965c221a352e3a658f Mon Sep 17 00:00:00 2001 From: syeopite Date: Wed, 4 Oct 2023 14:36:04 -0400 Subject: Automatically initialize proxy via stdlib override --- src/invidious/helpers/crystal_class_overrides.cr | 34 ++++++++++++++++++++++++ src/invidious/yt_backend/connection_pool.cr | 7 +++-- 2 files changed, 37 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr index bf56d826..71038703 100644 --- a/src/invidious/helpers/crystal_class_overrides.cr +++ b/src/invidious/helpers/crystal_class_overrides.cr @@ -18,6 +18,40 @@ end class HTTP::Client property family : Socket::Family = Socket::Family::UNSPEC + # Override stdlib to automatically initialize proxy if configured + # + # Accurate as of crystal 1.10.1 + + def initialize(@host : String, port = nil, tls : TLSContext = nil) + check_host_only(@host) + + {% if flag?(:without_openssl) %} + if tls + raise "HTTP::Client TLS is disabled because `-D without_openssl` was passed at compile time" + end + @tls = nil + {% else %} + @tls = case tls + when true + OpenSSL::SSL::Context::Client.new + when OpenSSL::SSL::Context::Client + tls + when false, nil + nil + end + {% end %} + + @port = (port || (@tls ? 443 : 80)).to_i + + self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + end + + def initialize(@io : IO, @host = "", @port = 80) + @reconnect = false + + self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + end + private def io io = @io return io if io diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 0e4d8aff..f34f48c5 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -26,13 +26,15 @@ struct YoutubeConnectionPool def client(&block) conn = pool.checkout + # Proxy needs to be reinstated every time we get a client from the pool conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + begin response = yield conn rescue ex conn.close - conn = HTTP::Client.new(url) + conn = HTTP::Client.new(url) conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC @@ -48,7 +50,6 @@ struct YoutubeConnectionPool private def build_pool DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do conn = HTTP::Client.new(url) - conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -69,8 +70,6 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false) client.read_timeout = 10.seconds client.connect_timeout = 10.seconds - client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy - return client end -- cgit v1.2.3 From 6b7e7301009e1a9fc2b536bd8d8de04fb8e22ec0 Mon Sep 17 00:00:00 2001 From: syeopite Date: Wed, 22 May 2024 13:10:46 -0700 Subject: Validate override for crystal 1.12.1 --- src/invidious/helpers/crystal_class_overrides.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr index 71038703..a7d2a5e6 100644 --- a/src/invidious/helpers/crystal_class_overrides.cr +++ b/src/invidious/helpers/crystal_class_overrides.cr @@ -20,7 +20,7 @@ class HTTP::Client # Override stdlib to automatically initialize proxy if configured # - # Accurate as of crystal 1.10.1 + # Accurate as of crystal 1.12.1 def initialize(@host : String, port = nil, tls : TLSContext = nil) check_host_only(@host) -- cgit v1.2.3