diff options
30 files changed, 346 insertions, 188 deletions
diff --git a/config/config.example.yml b/config/config.example.yml index 4cc6e7f4..c3d52d32 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -157,10 +157,14 @@ https_only: false ## Note 2: Using QUIC prevents some captcha challenges from appearing. ## See: https://github.com/iv-org/invidious/issues/957#issuecomment-576424042 ## +## Note 3: As of 2021-11-12, Google seems to have disabled QUIC on +## their servers. The default has been changed to 'false'.Read more +## at: https://github.com/iv-org/invidious/issues/2577 +## ## Accepted values: true, false -## Default: true +## Default: false ## -#use_quic: true +#use_quic: false ## ## Additionnal cookies to be sent when requesting the youtube API. @@ -466,9 +470,9 @@ default_user_preferences: ## ## Default user interface language (locale). ## - ## Note: overridin the default (no preferred caption language) - ## is not recommended, in order to not penalize people using - ## other languages. + ## Note: When hosting a public instance, overriding the + ## default (english) is not recommended, as it may + ## people using other languages. ## ## Accepted values: ## ar (Arabic) diff --git a/docker-compose.yml b/docker-compose.yml index cc62d6e5..c76c314c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ services: POSTGRES_PASSWORD: kemal POSTGRES_USER: kemal healthcheck: - test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER $$POSTGRES_DB"] + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] invidious: build: context: . diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index 4fe6b1ea..4d5d46bf 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -1,5 +1,5 @@ FROM alpine:edge AS builder -RUN apk add --no-cache 'crystal=1.2.1-r0' shards sqlite-static yaml-static yaml-dev libxml2-dev zlib-static openssl-libs-static openssl-dev musl-dev +RUN apk add --no-cache 'crystal=1.2.2-r0' shards sqlite-static yaml-static yaml-dev libxml2-dev zlib-static openssl-libs-static openssl-dev musl-dev ARG release diff --git a/locales/ar.json b/locales/ar.json index df45c060..15cf89bf 100644 --- a/locales/ar.json +++ b/locales/ar.json @@ -431,5 +431,6 @@ "footer_modfied_source_code": "شفرة المصدر المعدلة", "adminprefs_modified_source_code_url_label": "URL إلى مستودع التعليمات البرمجية المصدرية المعدلة", "footer_documentation": "التوثيق", - "footer_donate_page": "تبرّع" + "footer_donate_page": "تبرّع", + "preferences_region_label": "بلد المحتوى:. " } diff --git a/locales/en-US.json b/locales/en-US.json index 480326ce..036c22f4 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -69,6 +69,7 @@ "preferences_local_label": "Proxy videos: ", "preferences_speed_label": "Default speed: ", "preferences_quality_label": "Preferred video quality: ", + "preferences_quality_dash_label": "Preferred dash video quality: ", "preferences_volume_label": "Player volume: ", "preferences_comments_label": "Default comments: ", "youtube": "YouTube", diff --git a/locales/eo.json b/locales/eo.json index c9c16a74..f8a07c9e 100644 --- a/locales/eo.json +++ b/locales/eo.json @@ -431,5 +431,7 @@ "adminprefs_modified_source_code_url_label": "URL al modifita deponejo de fontkodo", "footer_modfied_source_code": "Modifita Fontkodo", "footer_original_source_code": "Originala fontkodo", - "footer_donate_page": "Donaci" + "footer_donate_page": "Donaci", + "preferences_region_label": "Lando de la enhavo: ", + "preferences_quality_dash_label": "Preferata DASH-a videkvalito: " } diff --git a/locales/es.json b/locales/es.json index 49d1e09e..1403a731 100644 --- a/locales/es.json +++ b/locales/es.json @@ -386,7 +386,7 @@ "%A %B %-d, %Y": "%A %B %-d, %Y", "(edited)": "(editado)", "YouTube comment permalink": "Enlace permanente de YouTube del comentario", - "permalink": "permalink", + "permalink": "enlace permanente", "`x` marked it with a ❤": "`x` lo ha marcado con un ❤", "Audio mode": "Modo de audio", "Video mode": "Modo de vídeo", @@ -431,5 +431,7 @@ "adminprefs_modified_source_code_url_label": "URL al repositorio de código fuente modificado", "footer_source_code": "Código fuente", "footer_modfied_source_code": "Código fuente modificado", - "footer_donate_page": "Donar" + "footer_donate_page": "Donar", + "preferences_region_label": "País del contenido: ", + "preferences_quality_dash_label": "Calidad de vídeo DASH preferida: " } diff --git a/locales/eu.json b/locales/eu.json index 1458e9a9..a62bb60b 100644 --- a/locales/eu.json +++ b/locales/eu.json @@ -73,6 +73,5 @@ "Dark mode: ": "Gai iluna: ", "preferences_dark_mode_label": "Gaia: ", "dark": "iluna", - "light": "argia", - "Subscription preferences": "Harpidetzen hobespenak" + "light": "argia" } diff --git a/locales/hr.json b/locales/hr.json index 884bf021..6b8c718c 100644 --- a/locales/hr.json +++ b/locales/hr.json @@ -431,5 +431,6 @@ "footer_source_code": "Izvorni kod", "footer_modfied_source_code": "Izmijenjeni izvorni kod", "footer_documentation": "Dokumentacija", - "footer_original_source_code": "Izvoran izvorni kod" + "footer_original_source_code": "Izvoran izvorni kod", + "preferences_region_label": "Zemlja sadržaja: " } diff --git a/locales/id.json b/locales/id.json index 2623efe4..07910143 100644 --- a/locales/id.json +++ b/locales/id.json @@ -87,7 +87,7 @@ "light": "terang", "preferences_thin_mode_label": "Mode tipis: ", "preferences_category_misc": "Preferensi lainnya", - "preferences_automatic_instance_redirect_label": "Pengalihan instans otomatis (fallback ke redirect.invidious.io): ", + "preferences_automatic_instance_redirect_label": "Pengalihan peladen otomatis (balik kembali ke redirect.invidious.io): ", "preferences_category_subscription": "Preferensi langganan", "preferences_annotations_subscribed_label": "Tampilkan anotasi secara default untuk kanal langganan: ", "Redirect homepage to feed: ": "Arahkan kembali laman beranda ke umpan: ", @@ -164,8 +164,8 @@ "Show more": "Tampilkan lainnya", "Show less": "Tampilkan lebih sedikit", "Watch on YouTube": "Tonton di YouTube", - "Switch Invidious Instance": "Beralih Instance Invidious", - "Broken? Try another Invidious Instance": "Rusak? Coba Instance Invidious lain", + "Switch Invidious Instance": "Ganti peladen Invidious", + "Broken? Try another Invidious Instance": "Rusak? Coba peladen Invidious yang lain", "Hide annotations": "Sembunyikan anotasi", "Show annotations": "Tampilkan anotasi", "Genre: ": "Genre: ", @@ -386,7 +386,7 @@ "%A %B %-d, %Y": "%A %B %-d, %Y", "(edited)": "(disunting)", "YouTube comment permalink": "Komentar YouTube permalink", - "permalink": "permalink", + "permalink": "tautan permanen", "`x` marked it with a ❤": "`x` telah ditandai dengan ❤", "Audio mode": "Mode audio", "Video mode": "Mode video", @@ -431,5 +431,7 @@ "short": "Pendek (< 4 menit)", "long": "Panjang (> 20 menit)", "footer_modfied_source_code": "Kode sumber yang dimodifikasi", - "footer_documentation": "Dokumentasi" + "footer_documentation": "Dokumentasi", + "preferences_region_label": "Konten dari negara: ", + "preferences_quality_dash_label": "Kualitas video dash yang disukai: " } diff --git a/locales/ja.json b/locales/ja.json index f67150fe..ea334e98 100644 --- a/locales/ja.json +++ b/locales/ja.json @@ -72,7 +72,7 @@ "preferences_volume_label": "プレイヤーの音量: ", "preferences_comments_label": "デフォルトのコメント: ", "youtube": "YouTube", - "reddit": "reddit", + "reddit": "Reddit", "preferences_captions_label": "デフォルトの字幕: ", "Fallback captions: ": "フォールバック時の字幕: ", "preferences_related_videos_label": "関連動画を表示: ", @@ -145,7 +145,7 @@ }, "search": "検索", "Log out": "ログアウト", - "Released under the AGPLv3 on Github.": "Github 上で AGPLv3 の元で公開されています", + "Released under the AGPLv3 on Github.": "GitHub 上で AGPLv3 の元で公開されています。", "Source available here.": "ソースはここで閲覧可能です。", "View JavaScript license information.": "JavaScript ライセンス情報", "View privacy policy.": "プライバシーポリシー", @@ -430,5 +430,7 @@ "footer_original_source_code": "ソースコード（元）", "footer_modfied_source_code": "ソースコード（編集）", "adminprefs_modified_source_code_url_label": "編集したソースコードのレポジトリーURL", - "long": "20 分以上" + "long": "20 分以上", + "preferences_region_label": "地域: ", + "footer_donate_page": "寄付する" } diff --git a/locales/lt.json b/locales/lt.json index cc609126..1881ac0b 100644 --- a/locales/lt.json +++ b/locales/lt.json @@ -431,5 +431,7 @@ "footer_original_source_code": "Pradinis pirminis kodas", "adminprefs_modified_source_code_url_label": "URL į pakeisto pirminio kodo repozitoriją", "footer_modfied_source_code": "Pakeistas pirminis kodas", - "footer_donate_page": "Paaukoti" + "footer_donate_page": "Paaukoti", + "preferences_region_label": "Turinio šalis: ", + "preferences_quality_dash_label": "Pageidaujama DASH vaizdo kokybė: " } diff --git a/locales/pt.json b/locales/pt.json index 554569f4..3665d0b5 100644 --- a/locales/pt.json +++ b/locales/pt.json @@ -431,5 +431,7 @@ "adminprefs_modified_source_code_url_label": "URL do repositório do código-fonte alterado", "footer_documentation": "Documentação", "footer_modfied_source_code": "Código-fonte alterado", - "footer_donate_page": "Doar" + "footer_donate_page": "Doar", + "preferences_region_label": "País do conteúdo: ", + "preferences_quality_dash_label": "Qualidade de vídeo DASH preferida: " } diff --git a/locales/si.json b/locales/si.json deleted file mode 100644 index 09ed43a4..00000000 --- a/locales/si.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "LIVE": "සජීව", - "popular": "ජනප්රිය" -} diff --git a/locales/sv-SE.json b/locales/sv-SE.json index beacc278..d366aa91 100644 --- a/locales/sv-SE.json +++ b/locales/sv-SE.json @@ -420,5 +420,10 @@ "filter": "Filter", "Current version: ": "Nuvarande version: ", "next_steps_error_message_refresh": "Uppdatera", - "next_steps_error_message_go_to_youtube": "Gå till Youtube" + "next_steps_error_message_go_to_youtube": "Gå till Youtube", + "Released under the AGPLv3 on Github.": "Publicerad under AGPLv3 på Github.", + "footer_source_code": "Källkod", + "long": "Lång (> 20 minuter)", + "footer_documentation": "Dokumentation", + "short": "Kort (< 4 minuter)" } diff --git a/locales/tr.json b/locales/tr.json index 28f46963..3bb9405d 100644 --- a/locales/tr.json +++ b/locales/tr.json @@ -431,5 +431,7 @@ "footer_original_source_code": "Orijinal kaynak kodları", "footer_modfied_source_code": "Değiştirilmiş kaynak kodları", "adminprefs_modified_source_code_url_label": "Değiştirilmiş kaynak kodları deposunun URL'si", - "footer_donate_page": "Bağış yap" + "footer_donate_page": "Bağış yap", + "preferences_region_label": "İçerik ülkesi: ", + "preferences_quality_dash_label": "Tercih edilen dash video kalitesi: " } diff --git a/locales/zh-CN.json b/locales/zh-CN.json index 9bb3ae71..8821ccf9 100644 --- a/locales/zh-CN.json +++ b/locales/zh-CN.json @@ -431,5 +431,7 @@ "footer_modfied_source_code": "修改的源代码", "adminprefs_modified_source_code_url_label": "更改的源代码仓库网址", "footer_original_source_code": "原始源代码", - "footer_donate_page": "捐赠" + "footer_donate_page": "捐赠", + "preferences_region_label": "内容国家： ", + "preferences_quality_dash_label": "首选 dash 视频分辨率： " } diff --git a/locales/zh-TW.json b/locales/zh-TW.json index d27491da..b23e439a 100644 --- a/locales/zh-TW.json +++ b/locales/zh-TW.json @@ -431,5 +431,7 @@ "footer_original_source_code": "原本的原始碼", "footer_modfied_source_code": "修改後的原始碼", "adminprefs_modified_source_code_url_label": "修改後的原始碼倉庫 URL", - "footer_donate_page": "捐款" + "footer_donate_page": "捐款", + "preferences_region_label": "內容國家： ", + "preferences_quality_dash_label": "偏好的 dash 影片品質： " } diff --git a/src/invidious/config.cr b/src/invidious/config.cr index bacdb4ac..578e31fd 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -93,7 +93,7 @@ class Config property port : Int32 = 3000 # Port to listen for connections (overrided by command line argument) property host_binding : String = "0.0.0.0" # Host to bind (overrided by command line argument) property pool_size : Int32 = 100 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) - property use_quic : Bool = true # Use quic transport for youtube api + property use_quic : Bool = false # Use quic transport for youtube api @[YAML::Field(converter: Preferences::StringToCookies)] property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 9c053d74..c3b356a9 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -206,7 +206,7 @@ def create_notification_stream(env, topics, connection_channel) video = get_video(video_id, PG_DB) video.published = published - response = JSON.parse(video.to_json(locale)) + response = JSON.parse(video.to_json(locale, nil)) if fields_text = env.params.query["fields"]? begin @@ -282,7 +282,7 @@ def create_notification_stream(env, topics, connection_channel) video = get_video(video_id, PG_DB) video.published = Time.unix(published) - response = JSON.parse(video.to_json(locale)) + response = JSON.parse(video.to_json(locale, nil)) if fields_text = env.params.query["fields"]? begin diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index 1ba3f20e..f92b7b89 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -58,17 +58,13 @@ struct SearchVideo end end - def to_xml(auto_generated, query_params, xml : XML::Builder | Nil = nil) - if xml - to_xml(HOST_URL, auto_generated, query_params, xml) - else - XML.build do |xml| - to_xml(HOST_URL, auto_generated, query_params, xml) - end + def to_xml(auto_generated, query_params, _xml : Nil) + XML.build do |xml| + to_xml(auto_generated, query_params, xml) end end - def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder) + def to_json(locale : Hash(String, JSON::Any) | Nil, json : JSON::Builder) json.object do json.field "type", "video" json.field "title", self.title @@ -99,16 +95,17 @@ struct SearchVideo end end - def to_json(locale, json : JSON::Builder | Nil = nil) - if json + # TODO: remove the locale and follow the crystal convention + def to_json(locale : Hash(String, JSON::Any) | Nil, _json : Nil) + JSON.build do |json| to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end end end + def to_json(json : JSON::Builder) + to_json(nil, json) + end + def is_upcoming premiere_timestamp ? true : false end @@ -133,7 +130,7 @@ struct SearchPlaylist property videos : Array(SearchPlaylistVideo) property thumbnail : String? - def to_json(locale, json : JSON::Builder) + def to_json(locale : Hash(String, JSON::Any) | Nil, json : JSON::Builder) json.object do json.field "type", "playlist" json.field "title", self.title @@ -163,15 +160,16 @@ struct SearchPlaylist end end - def to_json(locale, json : JSON::Builder | Nil = nil) - if json + # TODO: remove the locale and follow the crystal convention + def to_json(locale : Hash(String, JSON::Any) | Nil, _json : Nil) + JSON.build do |json| to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end end end + + def to_json(json : JSON::Builder) + to_json(nil, json) + end end struct SearchChannel @@ -185,7 +183,7 @@ struct SearchChannel property description_html : String property auto_generated : Bool - def to_json(locale, json : JSON::Builder) + def to_json(locale : Hash(String, JSON::Any) | Nil, json : JSON::Builder) json.object do json.field "type", "channel" json.field "author", self.author @@ -215,15 +213,16 @@ struct SearchChannel end end - def to_json(locale, json : JSON::Builder | Nil = nil) - if json + # TODO: remove the locale and follow the crystal convention + def to_json(locale : Hash(String, JSON::Any) | Nil, _json : Nil) + JSON.build do |json| to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end end end + + def to_json(json : JSON::Builder) + to_json(nil, json) + end end class Category @@ -235,7 +234,7 @@ class Category property description_html : String property badges : Array(Tuple(String, String))? - def to_json(locale, json : JSON::Builder) + def to_json(locale : Hash(String, JSON::Any) | Nil, json : JSON::Builder) json.object do json.field "type", "category" json.field "title", self.title @@ -249,15 +248,16 @@ class Category end end - def to_json(locale, json : JSON::Builder | Nil = nil) - if json + # TODO: remove the locale and follow the crystal convention + def to_json(locale : Hash(String, JSON::Any) | Nil, _json : Nil) + JSON.build do |json| to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end end end + + def to_json(json : JSON::Builder) + to_json(nil, json) + end end alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 443d19d7..f37667b5 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -11,7 +11,7 @@ struct PlaylistVideo property index : Int64 property live_now : Bool - def to_xml(auto_generated, xml : XML::Builder) + def to_xml(xml : XML::Builder) xml.element("entry") do xml.element("id") { xml.text "yt:video:#{self.id}" } xml.element("yt:videoId") { xml.text self.id } @@ -20,13 +20,8 @@ struct PlaylistVideo xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?v=#{self.id}") xml.element("author") do - if auto_generated - xml.element("name") { xml.text self.author } - xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } - else - xml.element("name") { xml.text author } - xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" } - end + xml.element("name") { xml.text self.author } + xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } end xml.element("content", type: "xhtml") do @@ -47,17 +42,11 @@ struct PlaylistVideo end end - def to_xml(auto_generated, xml : XML::Builder? = nil) - if xml - to_xml(auto_generated, xml) - else - XML.build do |xml| - to_xml(auto_generated, xml) - end - end + def to_xml(_xml : Nil = nil) + XML.build { |xml| to_xml(xml) } end - def to_json(locale, json : JSON::Builder, index : Int32?) + def to_json(json : JSON::Builder, index : Int32? = nil) json.object do json.field "title", self.title json.field "videoId", self.id @@ -81,14 +70,8 @@ struct PlaylistVideo end end - def to_json(locale, json : JSON::Builder? = nil, index : Int32? = nil) - if json - to_json(locale, json, index: index) - else - JSON.build do |json| - to_json(locale, json, index: index) - end - end + def to_json(_json : Nil, index : Int32? = nil) + JSON.build { |json| to_json(json, index: index) } end end @@ -144,7 +127,7 @@ struct Playlist json.array do videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, video_id: video_id) videos.each do |video| - video.to_json(locale, json) + video.to_json(json) end end end @@ -224,7 +207,7 @@ struct InvidiousPlaylist videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, video_id: video_id) videos.each_with_index do |video, index| - video.to_json(locale, json, offset + index) + video.to_json(json, offset + index) end end end diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index 7950b302..cdd9e2f6 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -274,7 +274,10 @@ module Invidious::Routes::API::V1::Authenticated env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{plid}/videos/#{playlist_video.index.to_u64.to_s(16).upcase}" env.response.status_code = 201 - playlist_video.to_json(locale, index: playlist.index.size) + + JSON.build do |json| + playlist_video.to_json(json, index: playlist.index.size) + end end def self.delete_video_in_playlist(env) diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index d483bca6..1edee29c 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -16,7 +16,7 @@ module Invidious::Routes::API::V1::Videos return error_json(500, ex) end - video.to_json(locale) + video.to_json(locale, nil) end def self.captions(env) diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 40c41dc1..f4a8467b 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -281,9 +281,7 @@ module Invidious::Routes::Feeds xml.element("name") { xml.text playlist.author } end - videos.each do |video| - video.to_xml(false, xml) - end + videos.each &.to_xml(xml) end end else diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index bb924cdf..594a7869 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -3,31 +3,61 @@ module Invidious::Routes::Images def self.ggpht(env) url = env.request.path.lchop("/ggpht") - headers = HTTP::Headers{":authority" => "yt3.ggpht.com"} + headers = ( + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + HTTP::Headers{":authority" => "yt3.ggpht.com"} + else + HTTP::Headers.new + end + {% else %} + HTTP::Headers.new + {% end %} + ) + REQUEST_HEADERS_WHITELIST.each do |header| if env.request.headers[header]? headers[header] = env.request.headers[header] end end - begin - YT_POOL.client &.get(url, headers) do |response| - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end + # We're encapsulating this into a proc in order to easily reuse this + # portion of the code for each request block below. + request_proc = ->(response : HTTP::Client::Response) { + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value end + end - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - env.response.headers.delete("Transfer-Encoding") - break - end + env.response.headers["Access-Control-Allow-Origin"] = "*" - proxy_file(response, env) + if response.status_code >= 300 + env.response.headers.delete("Transfer-Encoding") + return end + + proxy_file(response, env) + } + + begin + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + YT_POOL.client &.get(url, headers) do |resp| + return request_proc.call(resp) + end + else + HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| + return request_proc.call(resp) + end + end + {% else %} + # This can likely be optimized into a (small) pool sometime in the future. + HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| + return request_proc.call(resp) + end + {% end %} rescue ex end end @@ -48,7 +78,9 @@ module Invidious::Routes::Images headers = HTTP::Headers.new - headers[":authority"] = "#{authority}.ytimg.com" + {% unless flag?(:disable_quic) %} + headers[":authority"] = "#{authority}.ytimg.com" + {% end %} REQUEST_HEADERS_WHITELIST.each do |header| if env.request.headers[header]? @@ -56,25 +88,41 @@ module Invidious::Routes::Images end end - begin - YT_POOL.client &.get(url, headers) do |response| - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end + request_proc = ->(response : HTTP::Client::Response) { + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value end + end - env.response.headers["Connection"] = "close" - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - env.response.headers.delete("Transfer-Encoding") - break - end + env.response.headers["Connection"] = "close" + env.response.headers["Access-Control-Allow-Origin"] = "*" - proxy_file(response, env) + if response.status_code >= 300 + return env.response.headers.delete("Transfer-Encoding") end + + proxy_file(response, env) + } + + begin + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + YT_POOL.client &.get(url, headers) do |resp| + return request_proc.call(resp) + end + else + HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| + return request_proc.call(resp) + end + end + {% else %} + # This can likely be optimized into a (small) pool sometime in the future. + HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| + return request_proc.call(resp) + end + {% end %} rescue ex end end @@ -83,34 +131,60 @@ module Invidious::Routes::Images def self.s_p_image(env) id = env.params.url["id"] name = env.params.url["name"] - url = env.request.resource - headers = HTTP::Headers{":authority" => "i9.ytimg.com"} + headers = ( + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + HTTP::Headers{":authority" => "i9.ytimg.com"} + else + HTTP::Headers.new + end + {% else %} + HTTP::Headers.new + {% end %} + ) + REQUEST_HEADERS_WHITELIST.each do |header| if env.request.headers[header]? headers[header] = env.request.headers[header] end end - begin - YT_POOL.client &.get(url, headers) do |response| - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end + request_proc = ->(response : HTTP::Client::Response) { + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value end + end - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - env.response.headers.delete("Transfer-Encoding") - break - end + env.response.headers["Access-Control-Allow-Origin"] = "*" - proxy_file(response, env) + if response.status_code >= 300 && response.status_code != 404 + return env.response.headers.delete("Transfer-Encoding") end + + proxy_file(response, env) + } + + begin + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + YT_POOL.client &.get(url, headers) do |resp| + return request_proc.call(resp) + end + else + HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| + return request_proc.call(resp) + end + end + {% else %} + # This can likely be optimized into a (small) pool sometime in the future. + HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| + return request_proc.call(resp) + end + {% end %} rescue ex end end @@ -149,16 +223,44 @@ module Invidious::Routes::Images id = env.params.url["id"] name = env.params.url["name"] - headers = HTTP::Headers{":authority" => "i.ytimg.com"} + headers = ( + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + HTTP::Headers{":authority" => "i.ytimg.com"} + else + HTTP::Headers.new + end + {% else %} + HTTP::Headers.new + {% end %} + ) if name == "maxres.jpg" build_thumbnails(id).each do |thumb| - if YT_POOL.client &.head("/vi/#{id}/#{thumb[:url]}.jpg", headers).status_code == 200 - name = thumb[:url] + ".jpg" - break - end + thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" + # Logic here is short enough that manually typing them out should be fine. + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + if YT_POOL.client &.head(thumbnail_resource_path, headers).status_code == 200 + name = thumb[:url] + ".jpg" + break + end + else + if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 + name = thumb[:url] + ".jpg" + break + end + end + {% else %} + # This can likely be optimized into a (small) pool sometime in the future. + if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 + name = thumb[:url] + ".jpg" + break + end + {% end %} end end + url = "/vi/#{id}/#{name}" REQUEST_HEADERS_WHITELIST.each do |header| @@ -167,24 +269,40 @@ module Invidious::Routes::Images end end - begin - YT_POOL.client &.get(url, headers) do |response| - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end + request_proc = ->(response : HTTP::Client::Response) { + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value end + end - env.response.headers["Access-Control-Allow-Origin"] = "*" + env.response.headers["Access-Control-Allow-Origin"] = "*" - if response.status_code >= 300 && response.status_code != 404 - env.response.headers.delete("Transfer-Encoding") - break - end - - proxy_file(response, env) + if response.status_code >= 300 && response.status_code != 404 + return env.response.headers.delete("Transfer-Encoding") end + + proxy_file(response, env) + } + + begin + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + YT_POOL.client &.get(url, headers) do |resp| + return request_proc.call(resp) + end + else + HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| + return request_proc.call(resp) + end + end + {% else %} + # This can likely be optimized into a (small) pool sometime in the future. + HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| + return request_proc.call(resp) + end + {% end %} rescue ex end end diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index b719b571..562d88e5 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -53,7 +53,13 @@ module Invidious::Routes::Login # See https://github.com/ytdl-org/youtube-dl/blob/2019.04.07/youtube_dl/extractor/youtube.py#L82 begin - client = QUIC::Client.new(LOGIN_URL) + client = nil # Declare variable + {% unless flag?(:disable_quic) %} + client = CONFIG.use_quic ? QUIC::Client.new(LOGIN_URL) : HTTP::Client.new(LOGIN_URL) + {% else %} + client = HTTP::Client.new(LOGIN_URL) + {% end %} + headers = HTTP::Headers.new login_page = client.get("/ServiceLogin") diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index d38a66d8..d3e5800c 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -275,7 +275,7 @@ struct Video end end - def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder) + def to_json(locale : Hash(String, JSON::Any) | Nil, json : JSON::Builder) json.object do json.field "type", "video" @@ -474,14 +474,13 @@ struct Video end end - def to_json(locale, json : JSON::Builder | Nil = nil) - if json - to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end - end + # TODO: remove the locale and follow the crystal convention + def to_json(locale : Hash(String, JSON::Any) | Nil, _json : Nil) + JSON.build { |json| to_json(locale, json) } + end + + def to_json(json : JSON::Builder | Nil = nil) + to_json(nil, json) end def title diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 5ba2d73c..3feb9233 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -1,4 +1,10 @@ -require "lsquic" +{% unless flag?(:disable_quic) %} + require "lsquic" + + alias HTTPClientType = QUIC::Client | HTTP::Client +{% else %} + alias HTTPClientType = HTTP::Client +{% end %} def add_yt_headers(request) request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" @@ -19,7 +25,7 @@ struct YoutubeConnectionPool property! url : URI property! capacity : Int32 property! timeout : Float64 - property pool : DB::Pool(QUIC::Client | HTTP::Client) + property pool : DB::Pool(HTTPClientType) def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true) @url = url @@ -36,7 +42,12 @@ struct YoutubeConnectionPool response = yield conn rescue ex conn.close - conn = QUIC::Client.new(url) + {% unless flag?(:disable_quic) %} + conn = CONFIG.use_quic ? QUIC::Client.new(url) : HTTP::Client.new(url) + {% else %} + conn = HTTP::Client.new(url) + {% end %} + conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -50,12 +61,18 @@ struct YoutubeConnectionPool end private def build_pool(use_quic) - DB::Pool(QUIC::Client | HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do - if use_quic - conn = QUIC::Client.new(url) - else + DB::Pool(HTTPClientType).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do + conn = nil # Declare + {% unless flag?(:disable_quic) %} + if use_quic + conn = QUIC::Client.new(url) + else + conn = HTTP::Client.new(url) + end + {% else %} conn = HTTP::Client.new(url) - end + {% end %} + conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 8ab2fe46..27f25036 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -404,10 +404,19 @@ module YoutubeAPI url = "#{endpoint}?key=#{client_config.api_key}" headers = HTTP::Headers{ - "Content-Type" => "application/json; charset=UTF-8", - "Accept-Encoding" => "gzip", + "Content-Type" => "application/json; charset=UTF-8", } + # The normal HTTP client automatically applies accept-encoding: gzip, + # and decompresses. However, explicitly applying it will remove this functionality. + # + # https://github.com/crystal-lang/crystal/issues/11252#issuecomment-929594741 + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + headers["Accept-Encoding"] = "gzip" + end + {% end %} + # Logging LOGGER.debug("YoutubeAPI: Using endpoint: \"#{endpoint}\"") LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}") |