diff options
42 files changed, 714 insertions, 349 deletions
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 4c1a6330..02bc3795 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -10,8 +10,10 @@ assignees: '' <!-- BEFORE TRYING TO REPORT A BUG: - * Read the FAQ! - * Use the search function to check if there is already an issue open for your problem! + * Read the FAQ: https://docs.invidious.io/faq/! + * Use the search function to check if there is already an issue open for your problem: https://github.com/search?q=repo%3Aiv-org%2Finvidious+replace+me+with+your+bug&type=issues! + + MAKE SURE TO FOLLOW THE TWO STEPS ABOVE BEFORE REPORTING A BUG. A BUG THAT ALREADY EXIST WILL IMMEDIATELY CLOSED. If you want to suggest a new feature please use "Feature request" instead If you want to suggest an enhancement to an existing feature please use "Enhancement" instead diff --git a/.github/workflows/build-nightly-container.yml b/.github/workflows/build-nightly-container.yml index bee27600..5ff3322f 100644 --- a/.github/workflows/build-nightly-container.yml +++ b/.github/workflows/build-nightly-container.yml @@ -23,19 +23,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Install Crystal - uses: crystal-lang/install-crystal@v1.8.2 - with: - crystal: 1.12.2 - - - name: Run lint - run: | - if ! crystal tool format --check; then - crystal tool format - git diff - exit 1 - fi - - name: Set up QEMU uses: docker/setup-qemu-action@v3 with: diff --git a/.github/workflows/build-stable-container.yml b/.github/workflows/build-stable-container.yml index d2d106b6..25571ed6 100644 --- a/.github/workflows/build-stable-container.yml +++ b/.github/workflows/build-stable-container.yml @@ -14,19 +14,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Install Crystal - uses: crystal-lang/install-crystal@v1.8.2 - with: - crystal: 1.12.2 - - - name: Run lint - run: | - if ! crystal tool format --check; then - crystal tool format - git diff - exit 1 - fi - - name: Set up QEMU uses: docker/setup-qemu-action@v3 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 411ec769..5f859613 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,11 +38,10 @@ jobs: matrix: stable: [true] crystal: - - 1.10.1 - - 1.11.2 - 1.12.1 - 1.13.2 - 1.14.0 + - 1.15.0 include: - crystal: nightly stable: false @@ -54,7 +53,7 @@ jobs: - name: Install required APT packages run: | - sudo apt install -y libsqlite3-dev + sudo apt install -y libsqlite3-dev shell: bash - name: Install Crystal @@ -65,7 +64,9 @@ jobs: - name: Cache Shards uses: actions/cache@v3 with: - path: ./lib + path: | + ./lib + ./bin key: shards-${{ hashFiles('shard.lock') }} - name: Install Shards @@ -77,14 +78,6 @@ jobs: - name: Run tests run: crystal spec - - name: Run lint - run: | - if ! crystal tool format --check; then - crystal tool format - git diff - exit 1 - fi - - name: Build run: crystal build --warnings all --error-on-warnings --error-trace src/invidious.cr @@ -130,14 +123,19 @@ jobs: - name: Test Docker run: while curl -Isf http://localhost:3000; do sleep 1; done - ameba_lint: + lint: + runs-on: ubuntu-latest + + continue-on-error: true + steps: - uses: actions/checkout@v4 with: submodules: true - name: Install Crystal + id: lint_step_install_crystal uses: crystal-lang/install-crystal@v1.8.0 with: crystal: latest @@ -148,10 +146,21 @@ jobs: path: | ./lib ./bin - key: shards-${{ hashFiles('shard.lock') }} + key: shards-${{ hashFiles('shard.lock') }}-${{ steps.lint_step_install_crystal.outputs.crystal }} - name: Install Shards - run: shards install + run: | + if ! shards check; then + shards install + fi + + - name: Check Crystal formatter compliance + run: | + if ! crystal tool format --check; then + crystal tool format + git diff + exit 1 + fi - name: Run Ameba linter run: bin/ameba diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 16d3269b..498a2c1b 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -13,14 +13,11 @@ jobs: - uses: actions/stale@v8 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - days-before-stale: 365 - days-before-pr-stale: 90 - days-before-close: 30 - exempt-pr-labels: blocked,exempt-stale + days-before-stale: 730 + days-before-pr-stale: -1 + days-before-close: 60 stale-issue-message: 'This issue has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely outdated. If you think this issue is still relevant and applicable, you just have to post a comment and it will be unmarked.' - stale-pr-message: 'This pull request has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely abandoned or outdated. If you think this pull request is still relevant and applicable, you just have to post a comment and it will be unmarked.' stale-issue-label: "stale" - stale-pr-label: "stale" ascending: true - # Never mark feature requests/enhancements as stale - exempt-issue-labels: "feature-request,enhancement,exempt-stale" + # Exempt the following types of issues from being staled + exempt-issue-labels: "feature-request,enhancement,discussion,exempt-stale" diff --git a/CHANGELOG.md b/CHANGELOG.md index 15991668..5af38003 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,98 @@ ## vX.Y.0 (future) +## v2.20241110.0 + +### Wrap-up + +This release is most importantly here to fix to the annoying "Youtube API returned error 400" +error that prevented all channel pages from loading. + +If you're updating from the previous release, it provides no improvements on the ability to play +videos. If updating from a commit in-between release, it removes the "Please sign in" error caused +by a previous attempt at restoring video playback on large instances. + +In the preferences, a new option allows for control of video preload. When enabled, this option +tells the browser to load the video as soon as the page is loaded (this used to be the default). +When disabled, the video starts loading only when the "play" button is pressed. + +New interface languages available: Bulgarian, Welsh and Lombard + +New dependency required: `tzdata`. + +An HTTP proxy can be configured directly in Invidious, if needed. \ +**NOTE:** In that case, it is recommended to comment out `force_resolve`. + + +### New features & important changes + +#### For users + +* Channels: Fix "Youtube API returned error 400" error preventing channel pages from loading +* Channels: Shorts can now be sorted by "newest", "oldest" and "popular" +* Preferences: Addition of the new "preload" option +* New interface languages available: Bulgarian, Welsh and Lombard +* Added "Filipino (auto-generated)" to the list of caption languages available +* Lots of new translations from Weblate + +#### For instance owners + +* Allow the configuration of an HTTP proxy to talk to Youtube +* Invidious tries to reconnect to `inv_sig_helper` if the socket is closed +* The instance list is downloaded in the background to improve redirection speed +* New `colorize_logs` option makes each log level a different color + +#### For developpers + +* `/api/v1/channels/{id}/shorts` now supports the `sort-by` parameter with the following values: + `newest`, `oldest` and `popular` +* Older `/api/v1/channels/xyz/{id}` (tab name before UCID) were removed +* API/Search: New video metadata available: `isNew`, `is4k`, `is8k`, `isVr180`, `isVr360`, + `is3d` and `hasCaptions` + +### Bugs fixed + +#### User-side + +* Channels: The second page of shorts now loads as expected +* Channels: Fixed intermittent empty "playlists" tab +* Search: Fixed `youtu.be` URLs not being properly redirected to the watch page +* Fixed `DB::MappingException` error on the subscriptions feed (due to missing `tzdata` in docker) +* Switching to another instance is much faster +* Fixed an "invalid byte sequence" error when subscribing to a playlist +* Videos: Playback URLs were sometimes broken when cached and `inv_sig_helper` was used + +#### For instance owners + +* Fix `force_resolve` being ignored in some cases + +#### API + +* API/Videos: Fixed `live_now` and `premiere_timestamp` sometimes not having the right values + + ### Full list of pull requests merged since the last release (newest first) +* API: Add "sort_by" parameter to channels/shorts endpoint ([#5071], thanks @iBicha) +* Docker: Install tzdata in Dockerfile ([#5070], by @SamantazFox) +* Videos: Stop using TVHTML5_SIMPLY_EMBEDDED_PLAYER ([#5063], thanks @unixfox) +* Routing: Deprecate old channel API routes ([#5045], by @SamantazFox) +* Videos: use WEB client instead of WEB CREATOR ([#4984], thanks @unixfox) +* Parsers: Fix parsing live_now and premiere_timestamp ([#4934], thanks @absidue) +* Stale bot updates ([#5060], thanks @syeopite) +* Channels: Fix "Youtube API returned error 400" ([#5059], by @SamantazFox) +* Channels: Fix for live videos ([#5027], thanks @iBicha) +* Locales: Add Bulgarian, Welsh and Lombard to the list ([#5046], by @SamantazFox) +* Shards: Update database dependencies ([#5034], by @SamantazFox) +* Logger: Add color support for different log levels ([#4931], thanks @Fijxu) +* Fix named arg syntax when passing force_resolve ([#4754], thanks @syeopite) +* Use make_client instead of calling HTTP::Client ([#4709], thanks @syeopite) +* Add "Filipino (auto-generated)" to the list of caption languages ([#4995], by @SamantazFox) +* Makefile: Add MT option to enable the 'preview_mt' flag ([#4993], by @SamantazFox) +* SigHelper: Reconnect to signature helper ([#4991], thanks @Fijxu) +* Fix player menus hiding onHover ready ([#4750], thanks @giacomocerquone) +* Use connection pools when requesting images from YouTube ([#4326], thanks @syeopite) +* Add support for using Invidious through a HTTP Proxy ([#4270], thanks @syeopite) * Search: Fix 'youtu.be' URLs in sanitizer ([#4894], by @SamantazFox) * Ameba: Disable Style/RedundantNext rule ([#4888], thanks @syeopite) * Playlists: Fix 'invalid byte sequence' error when subscribing ([#4887], thanks @DmitrySandalov) @@ -22,7 +112,12 @@ [#4122]: https://github.com/iv-org/invidious/pull/4122 [#4193]: https://github.com/iv-org/invidious/pull/4193 +[#4270]: https://github.com/iv-org/invidious/pull/4270 +[#4326]: https://github.com/iv-org/invidious/pull/4326 [#4652]: https://github.com/iv-org/invidious/pull/4652 +[#4709]: https://github.com/iv-org/invidious/pull/4709 +[#4750]: https://github.com/iv-org/invidious/pull/4750 +[#4754]: https://github.com/iv-org/invidious/pull/4754 [#4850]: https://github.com/iv-org/invidious/pull/4850 [#4862]: https://github.com/iv-org/invidious/pull/4862 [#4863]: https://github.com/iv-org/invidious/pull/4863 @@ -32,7 +127,22 @@ [#4923]: https://github.com/iv-org/invidious/pull/4923 [#4928]: https://github.com/iv-org/invidious/pull/4928 [#4930]: https://github.com/iv-org/invidious/pull/4930 +[#4931]: https://github.com/iv-org/invidious/pull/4931 +[#4934]: https://github.com/iv-org/invidious/pull/4934 [#4942]: https://github.com/iv-org/invidious/pull/4942 +[#4984]: https://github.com/iv-org/invidious/pull/4984 +[#4991]: https://github.com/iv-org/invidious/pull/4991 +[#4993]: https://github.com/iv-org/invidious/pull/4993 +[#4995]: https://github.com/iv-org/invidious/pull/4995 +[#5027]: https://github.com/iv-org/invidious/pull/5027 +[#5034]: https://github.com/iv-org/invidious/pull/5034 +[#5045]: https://github.com/iv-org/invidious/pull/5045 +[#5046]: https://github.com/iv-org/invidious/pull/5046 +[#5059]: https://github.com/iv-org/invidious/pull/5059 +[#5060]: https://github.com/iv-org/invidious/pull/5060 +[#5063]: https://github.com/iv-org/invidious/pull/5063 +[#5070]: https://github.com/iv-org/invidious/pull/5070 +[#5071]: https://github.com/iv-org/invidious/pull/5071 ## v2.20240825.2 (2024-08-26) @@ -7,6 +7,11 @@ STATIC := 0 NO_DBG_SYMBOLS := 0 +# Enable multi-threading. +# Warning: Experimental feature!! +# invidious is not stable when MT is enabled. +MT := 0 + FLAGS ?= @@ -19,6 +24,10 @@ ifeq ($(STATIC), 1) FLAGS += --static endif +ifeq ($(MT), 1) + FLAGS += -Dpreview_mt +endif + ifeq ($(NO_DBG_SYMBOLS), 1) FLAGS += --no-debug diff --git a/assets/css/player.css b/assets/css/player.css index 50c7a748..9cb400ad 100644 --- a/assets/css/player.css +++ b/assets/css/player.css @@ -68,6 +68,7 @@ .video-js.player-style-youtube .vjs-menu-button-popup .vjs-menu { margin-bottom: 2em; + padding-top: 2em } .video-js.player-style-youtube .vjs-progress-control .vjs-progress-holder, .video-js.player-style-youtube .vjs-progress-control {height: 5px; diff --git a/config/config.example.yml b/config/config.example.yml index e9eebfde..bc2deda5 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -173,6 +173,17 @@ https_only: false ## #force_resolve: +## +## Configuration for using a HTTP proxy +## +## If unset, then no HTTP proxy will be used. +## +#http_proxy: +# user: +# password: +# host: +# port: + ## ## Use Innertube's transcripts API instead of timedtext for closed captions @@ -222,6 +233,17 @@ https_only: false ## #log_level: Info +## +## Enables colors in logs. Useful for debugging purposes +## This is overridden if "-k" or "--colorize" +## are passed on the command line. +## Colors are also disabled if the environment variable +## NO_COLOR is present and has any value +## +## Accepted values: true, false +## Default: true +## +#colorize_logs: false # ----------------------------- # Features diff --git a/docker/Dockerfile b/docker/Dockerfile index 3d9323fd..900c9e74 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM crystallang/crystal:1.12.1-alpine AS builder +FROM crystallang/crystal:1.12.2-alpine AS builder RUN apk add --no-cache sqlite-static yaml-static @@ -32,8 +32,8 @@ RUN if [[ "${release}" == 1 ]] ; then \ --link-flags "-lxml2 -llzma"; \ fi -FROM alpine:3.18 -RUN apk add --no-cache rsvg-convert ttf-opensans tini +FROM alpine:3.20 +RUN apk add --no-cache rsvg-convert ttf-opensans tini tzdata WORKDIR /invidious RUN addgroup -g 1000 -S invidious && \ adduser -u 1000 -S invidious -G invidious diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index f054b326..ce9bab08 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -1,5 +1,6 @@ -FROM alpine:3.19 AS builder -RUN apk add --no-cache 'crystal=1.10.1-r0' shards sqlite-static yaml-static yaml-dev libxml2-static zlib-static openssl-libs-static openssl-dev musl-dev xz-static +FROM alpine:3.20 AS builder +RUN apk add --no-cache 'crystal=1.12.2-r0' shards sqlite-static yaml-static yaml-dev libxml2-static \ + zlib-static openssl-libs-static openssl-dev musl-dev xz-static ARG release @@ -32,8 +33,8 @@ RUN if [[ "${release}" == 1 ]] ; then \ --link-flags "-lxml2 -llzma"; \ fi -FROM alpine:3.18 -RUN apk add --no-cache rsvg-convert ttf-opensans tini +FROM alpine:3.20 +RUN apk add --no-cache rsvg-convert ttf-opensans tini tzdata WORKDIR /invidious RUN addgroup -g 1000 -S invidious && \ adduser -u 1000 -S invidious -G invidious diff --git a/locales/en-US.json b/locales/en-US.json index 7827d9c6..c23f6bc3 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -286,6 +286,7 @@ "Esperanto": "Esperanto", "Estonian": "Estonian", "Filipino": "Filipino", + "Filipino (auto-generated)": "Filipino (auto-generated)", "Finnish": "Finnish", "French": "French", "French (auto-generated)": "French (auto-generated)", @@ -10,16 +10,20 @@ shards: backtracer: git: https://github.com/sija/backtracer.cr.git - version: 1.2.1 + version: 1.2.2 db: git: https://github.com/crystal-lang/crystal-db.git - version: 0.10.1 + version: 0.13.1 exception_page: git: https://github.com/crystal-loot/exception_page.git version: 0.2.2 + http_proxy: + git: https://github.com/mamantoha/http_proxy.git + version: 0.10.3 + kemal: git: https://github.com/kemalcr/kemal.git version: 1.1.2 @@ -30,7 +34,7 @@ shards: pg: git: https://github.com/will/crystal-pg.git - version: 0.24.0 + version: 0.28.0 protodec: git: https://github.com/iv-org/protodec.git @@ -42,9 +46,9 @@ shards: spectator: git: https://github.com/icy-arctic-fox/spectator.git - version: 0.10.4 + version: 0.10.6 sqlite3: git: https://github.com/crystal-lang/crystal-sqlite3.git - version: 0.18.0 + version: 0.21.0 @@ -1,21 +1,20 @@ name: invidious -version: 0.20.1 +version: 2.20241110.0-dev authors: - - Omar Roth <omarroth@protonmail.com> - - Invidious team + - Invidious team <contact@invidious.io> + - Contributors! -targets: - invidious: - main: src/invidious.cr +description: | + Invidious is an alternative front-end to YouTube dependencies: pg: github: will/crystal-pg - version: ~> 0.24.0 + version: ~> 0.28.0 sqlite3: github: crystal-lang/crystal-sqlite3 - version: ~> 0.18.0 + version: ~> 0.21.0 kemal: github: kemalcr/kemal version: ~> 1.1.2 @@ -28,6 +27,9 @@ dependencies: athena-negotiation: github: athena-framework/negotiation version: ~> 0.1.1 + http_proxy: + github: mamantoha/http_proxy + version: ~> 0.10.3 development_dependencies: spectator: @@ -37,6 +39,10 @@ development_dependencies: github: crystal-ameba/ameba version: ~> 1.6.1 -crystal: ">= 1.0.0, < 2.0.0" +crystal: ">= 1.10.0, < 2.0.0" license: AGPLv3 + +repository: https://github.com/iv-org/invidious +homepage: https://invidious.io +documentation: https://docs.invidious.io diff --git a/src/invidious.cr b/src/invidious.cr index 63f2a9cc..b422dcbb 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -23,6 +23,7 @@ require "kilt" require "./ext/kemal_content_for.cr" require "./ext/kemal_static_file_handler.cr" +require "http_proxy" require "athena-negotiation" require "openssl/hmac" require "option_parser" @@ -92,6 +93,10 @@ SOFTWARE = { YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size) +# Image request pool + +GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size) + # CLI Kemal.config.extra_options do |parser| parser.banner = "Usage: invidious [arguments]" @@ -117,6 +122,9 @@ Kemal.config.extra_options do |parser| parser.on("-l LEVEL", "--log-level=LEVEL", "Log level, one of #{LogLevel.values} (default: #{CONFIG.log_level})") do |log_level| CONFIG.log_level = LogLevel.parse(log_level) end + parser.on("-k", "--colorize", "Colorize logs") do + CONFIG.colorize_logs = true + end parser.on("-v", "--version", "Print version") do puts SOFTWARE.to_pretty_json exit @@ -133,7 +141,7 @@ if CONFIG.output.upcase != "STDOUT" FileUtils.mkdir_p(File.dirname(CONFIG.output)) end OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mode: "a") -LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) +LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level, CONFIG.colorize_logs) # Check table integrity Invidious::Database.check_integrity(CONFIG) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 6cc30142..96400f47 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -1,78 +1,3 @@ -def produce_channel_content_continuation(ucid, content_type, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - object_inner_2 = { - "2:0:embedded" => { - "1:0:varint" => 0_i64, - }, - "5:varint" => 50_i64, - "6:varint" => 1_i64, - "7:varint" => (page * 30).to_i64, - "9:varint" => 1_i64, - "10:varint" => 0_i64, - } - - object_inner_2_encoded = object_inner_2 - .try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - content_type_numerical = - case content_type - when "videos" then 15 - when "livestreams" then 14 - else 15 # Fallback to "videos" - end - - sort_by_numerical = - case sort_by - when "newest" then 1_i64 - when "popular" then 2_i64 - when "oldest" then 4_i64 - else 1_i64 # Fallback to "newest" - end - - object_inner_1 = { - "110:embedded" => { - "3:embedded" => { - "#{content_type_numerical}:embedded" => { - "1:embedded" => { - "1:string" => object_inner_2_encoded, - }, - "2:embedded" => { - "1:string" => "00000000-0000-0000-0000-000000000000", - }, - "3:varint" => sort_by_numerical, - }, - }, - }, - } - - object_inner_1_encoded = object_inner_1 - .try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:string" => object_inner_1_encoded, - "35:string" => "browse-feed#{ucid}videos102", - }, - } - - continuation = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return continuation -end - -def make_initial_content_ctoken(ucid, content_type, sort_by) : String - return produce_channel_content_continuation(ucid, content_type, sort_by: sort_by) -end - module Invidious::Channel::Tabs extend self @@ -101,7 +26,7 @@ module Invidious::Channel::Tabs end def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest") - continuation ||= make_initial_content_ctoken(ucid, "videos", sort_by) + continuation ||= make_initial_videos_ctoken(ucid, sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, author, ucid) @@ -130,14 +55,10 @@ module Invidious::Channel::Tabs # Shorts # ------------------- - def get_shorts(channel : AboutChannel, continuation : String? = nil) - if continuation.nil? - # EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts" - # TODO: try to extract the continuation tokens that allows other sorting options - initial_data = YoutubeAPI.browse(channel.ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D") - else - initial_data = YoutubeAPI.browse(continuation: continuation) - end + def get_shorts(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") + continuation ||= make_initial_shorts_ctoken(channel.ucid, sort_by) + initial_data = YoutubeAPI.browse(continuation: continuation) + return extract_items(initial_data, channel.author, channel.ucid) end @@ -145,9 +66,8 @@ module Invidious::Channel::Tabs # Livestreams # ------------------- - def get_livestreams(channel : AboutChannel, continuation : String? = nil, sort_by = "newest") - continuation ||= make_initial_content_ctoken(channel.ucid, "livestreams", sort_by) - + def get_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") + continuation ||= make_initial_livestreams_ctoken(channel.ucid, sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, channel.author, channel.ucid) @@ -171,4 +91,102 @@ module Invidious::Channel::Tabs return items, next_continuation end + + # ------------------- + # C-tokens + # ------------------- + + private def sort_options_videos_short(sort_by : String) + case sort_by + when "newest" then return 4_i64 + when "popular" then return 2_i64 + when "oldest" then return 5_i64 + else return 4_i64 # Fallback to "newest" + end + end + + # Generate the initial "continuation token" to get the first page of the + # "videos" tab. The following page requires the ctoken provided in that + # first page, and so on. + private def make_initial_videos_ctoken(ucid : String, sort_by = "newest") + object = { + "15:embedded" => { + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "4:varint" => sort_options_videos_short(sort_by), + }, + } + + return channel_ctoken_wrap(ucid, object) + end + + # Generate the initial "continuation token" to get the first page of the + # "shorts" tab. The following page requires the ctoken provided in that + # first page, and so on. + private def make_initial_shorts_ctoken(ucid : String, sort_by = "newest") + object = { + "10:embedded" => { + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "4:varint" => sort_options_videos_short(sort_by), + }, + } + + return channel_ctoken_wrap(ucid, object) + end + + # Generate the initial "continuation token" to get the first page of the + # "livestreams" tab. The following page requires the ctoken provided in that + # first page, and so on. + private def make_initial_livestreams_ctoken(ucid : String, sort_by = "newest") + sort_by_numerical = + case sort_by + when "newest" then 12_i64 + when "popular" then 14_i64 + when "oldest" then 13_i64 + else 12_i64 # Fallback to "newest" + end + + object = { + "14:embedded" => { + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "5:varint" => sort_by_numerical, + }, + } + + return channel_ctoken_wrap(ucid, object) + end + + # The protobuf structure common between videos/shorts/livestreams + private def channel_ctoken_wrap(ucid : String, object) + object_inner = { + "110:embedded" => { + "3:embedded" => object, + }, + } + + object_inner_encoded = object_inner + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + object = { + "80226972:embedded" => { + "2:string" => ucid, + "3:string" => object_inner_encoded, + }, + } + + continuation = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + return continuation + end end diff --git a/src/invidious/config.cr b/src/invidious/config.cr index a097b7f1..4b3bdafc 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -55,6 +55,15 @@ struct ConfigPreferences end end +struct HTTPProxyConfig + include YAML::Serializable + + property user : String + property password : String + property host : String + property port : Int32 +end + class Config include YAML::Serializable @@ -69,6 +78,8 @@ class Config property output : String = "STDOUT" # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr property log_level : LogLevel = LogLevel::Info + # Enables colors in logs. Useful for debugging purposes + property colorize_logs : Bool = false # Database configuration with separate parameters (username, hostname, etc) property db : DBConfig? = nil @@ -129,6 +140,8 @@ class Config property host_binding : String = "0.0.0.0" # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) property pool_size : Int32 = 100 + # HTTP Proxy configuration + property http_proxy : HTTPProxyConfig? = nil # Use Innertube's transcripts API instead of timedtext for closed captions property use_innertube_for_captions : Bool = false @@ -171,6 +184,9 @@ class Config config = Config.from_yaml(config_yaml) # Update config from env vars (upcased and prefixed with "INVIDIOUS_") + # + # Also checks if any top-level config options are set to "CHANGE_ME!!" + # TODO: Support non-top-level config options such as the ones in DBConfig {% for ivar in Config.instance_vars %} {% env_id = "INVIDIOUS_#{ivar.id.upcase}" %} @@ -207,6 +223,12 @@ class Config exit(1) end end + + # Warn when any config attribute is set to "CHANGE_ME!!" + if config.{{ivar.id}} == "CHANGE_ME!!" + puts "Config: The value of '#{ {{ivar.stringify}} }' needs to be changed!!" + exit(1) + end {% end %} # HMAC_key is mandatory @@ -214,9 +236,6 @@ class Config if config.hmac_key.empty? puts "Config: 'hmac_key' is required/can't be empty" exit(1) - elsif config.hmac_key == "CHANGE_ME!!" - puts "Config: The value of 'hmac_key' needs to be changed!!" - exit(1) end # Build database_url from db.* if it's not set directly diff --git a/src/invidious/frontend/watch_page.cr b/src/invidious/frontend/watch_page.cr index c8cb7110..2e2f6ad0 100644 --- a/src/invidious/frontend/watch_page.cr +++ b/src/invidious/frontend/watch_page.cr @@ -13,7 +13,7 @@ module Invidious::Frontend::WatchPage @full_videos, @video_streams, @audio_streams, - @captions + @captions, ) end end diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr index b7643194..900cb0c6 100644 --- a/src/invidious/helpers/errors.cr +++ b/src/invidious/helpers/errors.cr @@ -130,7 +130,7 @@ def error_json_helper( env : HTTP::Server::Context, status_code : Int32, exception : Exception, - additional_fields : Hash(String, Object) | Nil = nil + additional_fields : Hash(String, Object) | Nil = nil, ) if exception.is_a?(InfoException) return error_json_helper(env, status_code, exception.message || "", additional_fields) @@ -152,7 +152,7 @@ def error_json_helper( env : HTTP::Server::Context, status_code : Int32, message : String, - additional_fields : Hash(String, Object) | Nil = nil + additional_fields : Hash(String, Object) | Nil = nil, ) env.response.content_type = "application/json" env.response.status_code = status_code diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index f3e3b951..13ea9fe9 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -27,6 +27,7 @@ class Kemal::RouteHandler # Processes the route if it's a match. Otherwise renders 404. private def process_request(context) raise Kemal::Exceptions::RouteNotFound.new(context) unless context.route_found? + return if context.response.closed? content = context.route.handler.call(context) if !Kemal.config.error_handlers.empty? && Kemal.config.error_handlers.has_key?(context.response.status_code) && exclude_match?(context) diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index 23a1aafc..1ba3ea61 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -1,8 +1,22 @@ +# Languages requiring a better level of translation (at least 20%) +# to be added to the list below: +# +# "af" => "", # Afrikaans +# "az" => "", # Azerbaijani +# "be" => "", # Belarusian +# "bn_BD" => "", # Bengali (Bangladesh) +# "ia" => "", # Interlingua +# "or" => "", # Odia +# "tk" => "", # Turkmen +# "tok => "", # Toki Pona +# LOCALES_LIST = { "ar" => "العربية", # Arabic + "bg" => "български", # Bulgarian "bn" => "বাংলা", # Bengali "ca" => "Català", # Catalan "cs" => "Čeština", # Czech + "cy" => "Cymraeg", # Welsh "da" => "Dansk", # Danish "de" => "Deutsch", # German "el" => "Ελληνικά", # Greek @@ -23,6 +37,7 @@ LOCALES_LIST = { "it" => "Italiano", # Italian "ja" => "日本語", # Japanese "ko" => "한국어", # Korean + "lmo" => "Lombard", # Lombard "lt" => "Lietuvių", # Lithuanian "nb-NO" => "Norsk bokmål", # Norwegian Bokmål "nl" => "Nederlands", # Dutch diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr index b443073e..03349595 100644 --- a/src/invidious/helpers/logger.cr +++ b/src/invidious/helpers/logger.cr @@ -1,3 +1,5 @@ +require "colorize" + enum LogLevel All = 0 Trace = 1 @@ -10,7 +12,9 @@ enum LogLevel end class Invidious::LogHandler < Kemal::BaseLogHandler - def initialize(@io : IO = STDOUT, @level = LogLevel::Debug) + def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, use_color : Bool = true) + Colorize.enabled = use_color + Colorize.on_tty_only! end def call(context : HTTP::Server::Context) @@ -39,10 +43,22 @@ class Invidious::LogHandler < Kemal::BaseLogHandler @io.flush end + def color(level) + case level + when LogLevel::Trace then :cyan + when LogLevel::Debug then :green + when LogLevel::Info then :white + when LogLevel::Warn then :yellow + when LogLevel::Error then :red + when LogLevel::Fatal then :magenta + else :default + end + end + {% for level in %w(trace debug info warn error fatal) %} def {{level.id}}(message : String) if LogLevel::{{level.id.capitalize}} >= @level - puts("#{Time.utc} [{{level.id}}] #{message}") + puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}}))) end end {% end %} diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index 1fef5f93..f8e8f187 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -24,6 +24,7 @@ struct SearchVideo property length_seconds : Int32 property premiere_timestamp : Time? property author_verified : Bool + property author_thumbnail : String? property badges : VideoBadges def to_xml(auto_generated, query_params, xml : XML::Builder) @@ -88,6 +89,24 @@ struct SearchVideo json.field "authorUrl", "/channel/#{self.ucid}" json.field "authorVerified", self.author_verified + author_thumbnail = self.author_thumbnail + + if author_thumbnail + json.field "authorThumbnails" do + json.array do + qualities = {32, 48, 76, 100, 176, 512} + + qualities.each do |quality| + json.object do + json.field "url", author_thumbnail.gsub(/=s\d+/, "=s#{quality}") + json.field "width", quality + json.field "height", quality + end + end + end + end + end + json.field "videoThumbnails" do Invidious::JSONify::APIv1.thumbnails(json, self.id) end @@ -223,7 +242,7 @@ struct SearchChannel qualities.each do |quality| json.object do - json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}") + json.field "url", self.author_thumbnail.gsub(/=s\d+/, "=s#{quality}") json.field "width", quality json.field "height", quality end diff --git a/src/invidious/helpers/sig_helper.cr b/src/invidious/helpers/sig_helper.cr index 9e72c1c7..6d198a42 100644 --- a/src/invidious/helpers/sig_helper.cr +++ b/src/invidious/helpers/sig_helper.cr @@ -175,8 +175,9 @@ module Invidious::SigHelper @queue = {} of TransactionID => Transaction @conn : Connection + @uri_or_path : String - def initialize(uri_or_path) + def initialize(@uri_or_path) @conn = Connection.new(uri_or_path) listen end @@ -186,10 +187,26 @@ module Invidious::SigHelper LOGGER.debug("SigHelper: Multiplexor listening") - # TODO: reopen socket if unexpectedly closed spawn do loop do - receive_data + begin + receive_data + rescue ex + LOGGER.info("SigHelper: Connection to helper died with '#{ex.message}' trying to reconnect...") + # We close the socket because for some reason is not closed. + @conn.close + loop do + begin + @conn = Connection.new(@uri_or_path) + LOGGER.info("SigHelper: Reconnected to SigHelper!") + rescue ex + LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}'. Retrying") + sleep 500.milliseconds + next + end + break if !@conn.closed? + end + end Fiber.yield end end diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index d89e752c..78b4906d 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -70,17 +70,23 @@ module Invidious::Routes::API::Manifest # OTF streams aren't supported yet (See https://github.com/TeamNewPipe/NewPipe/issues/2415) next if !(fmt.has_key?("indexRange") && fmt.has_key?("initRange")) + audio_track = fmt["audioTrack"]?.try &.as_h? || {} of String => JSON::Any + lang = audio_track["id"]?.try &.as_s.split('.')[0] || "und" + is_default = audio_track.has_key?("audioIsDefault") ? audio_track["audioIsDefault"].as_bool : i == 0 + displayname = audio_track["displayName"]?.try &.as_s || "Unknown" + bitrate = fmt["bitrate"] + # Different representations of the same audio should be groupped into one AdaptationSet. # However, most players don't support auto quality switching, so we have to trick them # into providing a quality selector. # See https://github.com/iv-org/invidious/issues/3074 for more details. - xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, label: fmt["bitrate"].to_s + "k") do + xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, label: "#{displayname} [#{bitrate}k]", lang: lang) do codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"') bandwidth = fmt["bitrate"].as_i itag = fmt["itag"].as_i url = fmt["url"].as_s - xml.element("Role", schemeIdUri: "urn:mpeg:dash:role:2011", value: i == 0 ? "main" : "alternate") + xml.element("Role", schemeIdUri: "urn:mpeg:dash:role:2011", value: is_default ? "main" : "alternate") xml.element("Representation", id: fmt["itag"], codecs: codecs, bandwidth: bandwidth) do xml.element("AudioChannelConfiguration", schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011", diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 2da76134..588bbc2a 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -197,6 +197,7 @@ module Invidious::Routes::API::V1::Channels get_channel() # Retrieve continuation from URL parameters + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" continuation = env.params.query["continuation"]? if channel.is_age_gated @@ -211,7 +212,7 @@ module Invidious::Routes::API::V1::Channels else begin videos, next_continuation = Channel::Tabs.get_shorts( - channel, continuation: continuation + channel, continuation: continuation, sort_by: sort_by ) rescue ex return error_json(500, ex) diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index 093669fe..94a7e9b6 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -141,9 +141,7 @@ module Invidious::Routes::API::V1::Misc json.field "authorUrl", "/channel/#{video.ucid}" json.field "videoThumbnails" do - json.array do - Invidious::JSONify::APIv1.thumbnails(json, video.id) - end + Invidious::JSONify::APIv1.thumbnails(json, video.id) end json.field "index", video.index diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index 2922b060..59a30745 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -31,9 +31,7 @@ module Invidious::Routes::API::V1::Search query = env.params.query["q"]? || "" begin - client = HTTP::Client.new("suggestqueries-clients6.youtube.com") - client.before_request { |r| add_yt_headers(r) } - + client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers: true) url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt" response = client.get(url).body diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 952098e0..7d634cbb 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -20,10 +20,11 @@ module Invidious::Routes::Channels sort_by = env.params.query["sort_by"]?.try &.downcase if channel.auto_generated + sort_by ||= "last" sort_options = {"last", "oldest", "newest"} items, next_continuation = fetch_channel_playlists( - channel.ucid, channel.author, continuation, (sort_by || "last") + channel.ucid, channel.author, continuation, sort_by ) items.uniq! do |item| @@ -49,9 +50,11 @@ module Invidious::Routes::Channels end next_continuation = nil else + sort_by ||= "newest" sort_options = {"newest", "oldest", "popular"} - items, next_continuation = Channel::Tabs.get_videos( - channel, continuation: continuation, sort_by: (sort_by || "newest") + + items, next_continuation = Channel::Tabs.get_60_videos( + channel, continuation: continuation, sort_by: sort_by ) end end @@ -82,13 +85,12 @@ module Invidious::Routes::Channels end next_continuation = nil else - # TODO: support sort option for shorts - sort_by = "" - sort_options = [] of String + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" + sort_options = {"newest", "oldest", "popular"} # Fetch items and continuation token items, next_continuation = Channel::Tabs.get_shorts( - channel, continuation: continuation + channel, continuation: continuation, sort_by: sort_by ) end diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index ea7fb396..82c04994 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -194,6 +194,7 @@ module Invidious::Routes::Feeds length_seconds: 0, premiere_timestamp: nil, author_verified: false, + author_thumbnail: nil, badges: VideoBadges::None, }) end diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index b6a2e110..639697db 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -11,29 +11,9 @@ module Invidious::Routes::Images end end - # We're encapsulating this into a proc in order to easily reuse this - # portion of the code for each request block below. - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - env.response.headers.delete("Transfer-Encoding") - return - end - - proxy_file(response, env) - } - begin - HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| - return request_proc.call(resp) + GGPHT_POOL.client &.get(url, headers) do |resp| + return self.proxy_image(env, resp) end rescue ex end @@ -61,27 +41,10 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Connection"] = "close" - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - return env.response.headers.delete("Transfer-Encoding") - end - - proxy_file(response, env) - } - begin - HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| - return request_proc.call(resp) + get_ytimg_pool(authority).client &.get(url, headers) do |resp| + env.response.headers["Connection"] = "close" + return self.proxy_image(env, resp) end rescue ex end @@ -101,26 +64,9 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - return env.response.headers.delete("Transfer-Encoding") - end - - proxy_file(response, env) - } - begin - HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| - return request_proc.call(resp) + get_ytimg_pool("i9").client &.get(url, headers) do |resp| + return self.proxy_image(env, resp) end rescue ex end @@ -165,8 +111,7 @@ module Invidious::Routes::Images if name == "maxres.jpg" build_thumbnails(id).each do |thumb| thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" - # This can likely be optimized into a (small) pool sometime in the future. - if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 + if get_ytimg_pool("i9").client &.head(thumbnail_resource_path, headers).status_code == 200 name = thumb[:url] + ".jpg" break end @@ -181,29 +126,28 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end + begin + get_ytimg_pool("i").client &.get(url, headers) do |resp| + return self.proxy_image(env, resp) end + rescue ex + end + end - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - return env.response.headers.delete("Transfer-Encoding") + private def self.proxy_image(env, response) + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value end + end - proxy_file(response, env) - } + env.response.headers["Access-Control-Allow-Origin"] = "*" - begin - # This can likely be optimized into a (small) pool sometime in the future. - HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end - rescue ex + if response.status_code >= 300 + return env.response.headers.delete("Transfer-Encoding") end + + return proxy_file(response, env) end end diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 24693662..26852d06 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -42,7 +42,7 @@ module Invidious::Routes::VideoPlayback headers["Range"] = "bytes=#{range_for_head}" end - client = make_client(URI.parse(host), region, force_resolve = true) + client = make_client(URI.parse(host), region, force_resolve: true) response = HTTP::Client::Response.new(500) error = "" 5.times do @@ -57,7 +57,7 @@ module Invidious::Routes::VideoPlayback if new_host != host host = new_host client.close - client = make_client(URI.parse(new_host), region, force_resolve = true) + client = make_client(URI.parse(new_host), region, force_resolve: true) end url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" @@ -71,7 +71,7 @@ module Invidious::Routes::VideoPlayback fvip = "3" host = "https://r#{fvip}---#{mn}.googlevideo.com" - client = make_client(URI.parse(host), region, force_resolve = true) + client = make_client(URI.parse(host), region, force_resolve: true) rescue ex error = ex.message end @@ -196,7 +196,7 @@ module Invidious::Routes::VideoPlayback break else client.close - client = make_client(URI.parse(host), region, force_resolve = true) + client = make_client(URI.parse(host), region, force_resolve: true) end end diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index ba05da19..9009062f 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -243,17 +243,16 @@ module Invidious::Routing # Channels get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home + get "/api/v1/channels/:ucid/latest", {{namespace}}::Channels, :latest + get "/api/v1/channels/:ucid/videos", {{namespace}}::Channels, :videos get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams get "/api/v1/channels/:ucid/podcasts", {{namespace}}::Channels, :podcasts get "/api/v1/channels/:ucid/releases", {{namespace}}::Channels, :releases - + get "/api/v1/channels/:ucid/playlists", {{namespace}}::Channels, :playlists + get "/api/v1/channels/:ucid/community", {{namespace}}::Channels, :community get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels - - {% for route in {"videos", "latest", "playlists", "community", "search"} %} - get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}} - get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}} - {% end %} + get "/api/v1/channels/:ucid/search", {{namespace}}::Channels, :search # Posts get "/api/v1/post/:id", {{namespace}}::Channels, :post @@ -271,11 +270,6 @@ module Invidious::Routing # Authenticated - # The notification APIs cannot be extracted yet! They require the *local* notifications constant defined in invidious.cr - # - # Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications - # Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications - get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr index bf968734..bc2715cf 100644 --- a/src/invidious/search/filters.cr +++ b/src/invidious/search/filters.cr @@ -75,7 +75,7 @@ module Invidious::Search @type : Type = Type::All, @duration : Duration = Duration::None, @features : Features = Features::None, - @sort : Sort = Sort::Relevance + @sort : Sort = Sort::Relevance, ) end diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr index c8e8cf7f..94a92e23 100644 --- a/src/invidious/search/query.cr +++ b/src/invidious/search/query.cr @@ -47,7 +47,7 @@ module Invidious::Search def initialize( params : HTTP::Params, @type : Type = Type::Regular, - @region : String? = nil + @region : String? = nil, ) # Get the raw search query string (common to all search types). In # Regular search mode, also look for the `search_query` URL parameter diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index ae09e736..962f87bd 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -106,7 +106,7 @@ struct Video if formats = info.dig?("streamingData", "adaptiveFormats") return formats .as_a.map(&.as_h) - .sort_by! { |f| f["width"]?.try &.as_i || 0 } + .sort_by! { |f| f["width"]?.try &.as_i || f["audioTrack"]?.try { |a| a["audioIsDefault"]?.try { |v| v.as_bool ? -1 : 0 } } || 0 } else return [] of Hash(String, JSON::Any) end diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr index 484e61d2..c811cfe1 100644 --- a/src/invidious/videos/caption.cr +++ b/src/invidious/videos/caption.cr @@ -123,6 +123,7 @@ module Invidious::Videos "Esperanto", "Estonian", "Filipino", + "Filipino (auto-generated)", "Finnish", "French", "French (auto-generated)", diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index fb8935d9..915c9baf 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -53,10 +53,6 @@ end def extract_video_info(video_id : String) # Init client config for the API client_config = YoutubeAPI::ClientConfig.new - # Use the WEB_CREATOR when po_token is configured because it fully only works on this client - if CONFIG.po_token - client_config.client_type = YoutubeAPI::ClientType::WebCreator - end # Fetch data from the player endpoint player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config) @@ -106,15 +102,8 @@ def extract_video_info(video_id : String) new_player_response = nil - # Second try in case WEB_CREATOR doesn't work with po_token. - # Only trigger if reason found and po_token configured. - if reason && CONFIG.po_token - client_config.client_type = YoutubeAPI::ClientType::WebEmbeddedPlayer - new_player_response = try_fetch_streaming_data(video_id, client_config) - end - - # Don't use Android client if po_token is passed because po_token doesn't - # work for Android client. + # Don't use Android test suite client if po_token is passed because po_token doesn't + # work for Android test suite client. if reason.nil? && CONFIG.po_token.nil? # Fetch the video streams using an Android client in order to get the # decrypted URLs and maybe fix throttling issues (#2194). See the @@ -124,14 +113,6 @@ def extract_video_info(video_id : String) new_player_response = try_fetch_streaming_data(video_id, client_config) end - # Last hope - # Only trigger if reason found or didn't work wth Android client. - # TvHtml5ScreenEmbed now requires sig helper for it to work but doesn't work with po_token. - if reason && CONFIG.po_token.nil? - client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed - new_player_response = try_fetch_streaming_data(video_id, client_config) - end - # Replace player response and reset reason if !new_player_response.nil? # Preserve captions & storyboard data before replacement @@ -235,8 +216,17 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp") .try { |t| Time.parse_rfc3339(t.as_s) } + premiere_timestamp ||= player_response.dig?( + "playabilityStatus", "liveStreamability", + "liveStreamabilityRenderer", "offlineSlate", + "liveStreamOfflineSlateRenderer", "scheduledStartTime" + ) + .try &.as_s.to_i64 + .try { |t| Time.unix(t) } + live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow") - .try &.as_bool || false + .try &.as_bool + live_now ||= video_details.dig?("isLive").try &.as_bool || false post_live_dvr = video_details.dig?("isPostLiveDvr") .try &.as_bool || false diff --git a/src/invidious/videos/storyboard.cr b/src/invidious/videos/storyboard.cr index a72c2f55..bd0eef59 100644 --- a/src/invidious/videos/storyboard.cr +++ b/src/invidious/videos/storyboard.cr @@ -20,7 +20,7 @@ module Invidious::Videos def initialize( *, @url, @width, @height, @count, @interval, - @rows, @columns, @images_count + @rows, @columns, @images_count, ) authority = /(i\d?).ytimg.com/.match!(@url.host.not_nil!)[1]? diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index ca612083..c4a73aa7 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -1,17 +1,6 @@ -def add_yt_headers(request) - request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" - request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" - - request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" - request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" - request.headers["Accept-Language"] ||= "en-us,en;q=0.5" - - # Preserve original cookies and add new YT consent cookie for EU servers - request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}" - if !CONFIG.cookies.empty? - request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" - end -end +# Mapping of subdomain => YoutubeConnectionPool +# This is needed as we may need to access arbitrary subdomains of ytimg +private YTIMG_POOLS = {} of String => YoutubeConnectionPool struct YoutubeConnectionPool property! url : URI @@ -26,15 +15,15 @@ struct YoutubeConnectionPool def client(&) conn = pool.checkout + # Proxy needs to be reinstated every time we get a client from the pool + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + begin response = yield conn rescue ex conn.close - conn = HTTP::Client.new(url) + conn = make_client(url, force_resolve: true) - conn.family = CONFIG.force_resolve - conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC - conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" response = yield conn ensure pool.release(conn) @@ -44,25 +33,45 @@ struct YoutubeConnectionPool end private def build_pool - DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do - conn = HTTP::Client.new(url) - conn.family = CONFIG.force_resolve - conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC - conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" - conn + options = DB::Pool::Options.new( + initial_pool_size: 0, + max_pool_size: capacity, + max_idle_pool_size: capacity, + checkout_timeout: timeout + ) + + DB::Pool(HTTP::Client).new(options) do + next make_client(url, force_resolve: true) end end end -def make_client(url : URI, region = nil, force_resolve : Bool = false) +def add_yt_headers(request) + request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" + request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" + + request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" + request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + request.headers["Accept-Language"] ||= "en-us,en;q=0.5" + + # Preserve original cookies and add new YT consent cookie for EU servers + request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}" + if !CONFIG.cookies.empty? + request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" + end +end + +def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false) client = HTTP::Client.new(url) + client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy # Force the usage of a specific configured IP Family if force_resolve client.family = CONFIG.force_resolve + client.family = Socket::Family::INET if client.family == Socket::Family::UNSPEC end - client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" + client.before_request { |r| add_yt_headers(r) } if url.host.try &.ends_with?("youtube.com") || force_youtube_headers client.read_timeout = 10.seconds client.connect_timeout = 10.seconds @@ -70,10 +79,38 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false) end def make_client(url : URI, region = nil, force_resolve : Bool = false, &) - client = make_client(url, region, force_resolve) + client = make_client(url, region, force_resolve: force_resolve) begin yield client ensure client.close end end + +def make_configured_http_proxy_client + # This method is only called when configuration for an HTTP proxy are set + config_proxy = CONFIG.http_proxy.not_nil! + + return HTTP::Proxy::Client.new( + config_proxy.host, + config_proxy.port, + + username: config_proxy.user, + password: config_proxy.password, + ) +end + +# Fetches a HTTP pool for the specified subdomain of ytimg.com +# +# Creates a new one when the specified pool for the subdomain does not exist +def get_ytimg_pool(subdomain) + if pool = YTIMG_POOLS[subdomain]? + return pool + else + LOGGER.info("ytimg_pool: Creating a new HTTP pool for \"https://#{subdomain}.ytimg.com\"") + pool = YoutubeConnectionPool.new(URI.parse("https://#{subdomain}.ytimg.com"), capacity: CONFIG.pool_size) + YTIMG_POOLS[subdomain] = pool + + return pool + end +end diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 4074de86..edd7bf1b 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -21,6 +21,7 @@ private ITEM_PARSERS = { Parsers::ItemSectionRendererParser, Parsers::ContinuationItemRendererParser, Parsers::HashtagRendererParser, + Parsers::LockupViewModelParser, } private alias InitialData = Hash(String, JSON::Any) @@ -66,6 +67,8 @@ private module Parsers author_id = author_fallback.id end + author_thumbnail = item_contents.dig?("channelThumbnailSupportedRenderers", "channelThumbnailWithLinkRenderer", "thumbnail", "thumbnails", 0, "url").try &.as_s + author_verified = has_verified_badge?(item_contents["ownerBadges"]?) # For live videos (and possibly recently premiered videos) there is no published information. @@ -147,6 +150,7 @@ private module Parsers length_seconds: length_seconds, premiere_timestamp: premiere_timestamp, author_verified: author_verified, + author_thumbnail: author_thumbnail, badges: badges, }) end @@ -467,9 +471,9 @@ private module Parsers # Parses an InnerTube richItemRenderer into a SearchVideo. # Returns nil when the given object isn't a RichItemRenderer # - # A richItemRenderer seems to be a simple wrapper for a videoRenderer, used - # by the result page for hashtags and for the podcast tab on channels. - # It is located inside a continuationItems container for hashtags. + # A richItemRenderer seems to be a simple wrapper for a various other types, + # used on the hashtags result page and the channel podcast tab. It is located + # itself inside a richGridRenderer container. # module RichItemRendererParser def self.process(item : JSON::Any, author_fallback : AuthorFallback) @@ -482,6 +486,8 @@ private module Parsers child = VideoRendererParser.process(item_contents, author_fallback) child ||= ReelItemRendererParser.process(item_contents, author_fallback) child ||= PlaylistRendererParser.process(item_contents, author_fallback) + child ||= LockupViewModelParser.process(item_contents, author_fallback) + child ||= ShortsLockupViewModelParser.process(item_contents, author_fallback) return child end @@ -496,6 +502,9 @@ private module Parsers # reelItemRenderer items are used in the new (2022) channel layout, # in the "shorts" tab. # + # NOTE: As of 10/2024, it might have been fully replaced by shortsLockupViewModel + # TODO: Confirm that hypothesis + # module ReelItemRendererParser def self.process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item["reelItemRenderer"]? @@ -573,6 +582,137 @@ private module Parsers length_seconds: duration, premiere_timestamp: Time.unix(0), author_verified: false, + author_thumbnail: nil, + badges: VideoBadges::None, + }) + end + + def self.parser_name + return {{@type.name}} + end + end + + # Parses an InnerTube lockupViewModel into a SearchPlaylist. + # Returns nil when the given object is not a lockupViewModel. + # + # This structure is present since November 2024 on the "podcasts" and + # "playlists" tabs of the channel page. It is usually encapsulated in either + # a richItemRenderer or a richGridRenderer. + # + module LockupViewModelParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["lockupViewModel"]? + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + playlist_id = item_contents["contentId"].as_s + + thumbnail_view_model = item_contents.dig( + "contentImage", "collectionThumbnailViewModel", + "primaryThumbnail", "thumbnailViewModel" + ) + + thumbnail = thumbnail_view_model.dig("image", "sources", 0, "url").as_s + + # This complicated sequences tries to extract the following data structure: + # "overlays": [{ + # "thumbnailOverlayBadgeViewModel": { + # "thumbnailBadges": [{ + # "thumbnailBadgeViewModel": { + # "text": "430 episodes", + # "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT" + # } + # }] + # } + # }] + # + # NOTE: this simplistic `.to_i` conversion might not work on larger + # playlists and hasn't been tested. + video_count = thumbnail_view_model.dig("overlays").as_a + .compact_map(&.dig?("thumbnailOverlayBadgeViewModel", "thumbnailBadges").try &.as_a) + .flatten + .find(nil, &.dig?("thumbnailBadgeViewModel", "text").try { |node| + {"episodes", "videos"}.any? { |str| node.as_s.ends_with?(str) } + }) + .try &.dig("thumbnailBadgeViewModel", "text").as_s.to_i(strict: false) + + metadata = item_contents.dig("metadata", "lockupMetadataViewModel") + title = metadata.dig("title", "content").as_s + + # TODO: Retrieve "updated" info from metadata parts + # rows = metadata.dig("metadata", "contentMetadataViewModel", "metadataRows").as_a + # parts_text = rows.map(&.dig?("metadataParts", "text", "content").try &.as_s) + # One of these parts should contain a string like: "Updated 2 days ago" + + # TODO: Maybe add a button to access the first video of the playlist? + # item_contents.dig("rendererContext", "commandContext", "onTap", "innertubeCommand", "watchEndpoint") + # Available fields: "videoId", "playlistId", "params" + + return SearchPlaylist.new({ + title: title, + id: playlist_id, + author: author_fallback.name, + ucid: author_fallback.id, + video_count: video_count || -1, + videos: [] of SearchPlaylistVideo, + thumbnail: thumbnail, + author_verified: false, + }) + end + + def self.parser_name + return {{@type.name}} + end + end + + # Parses an InnerTube shortsLockupViewModel into a SearchVideo. + # Returns nil when the given object is not a shortsLockupViewModel. + # + # This structure is present since around October 2024 on the "shorts" tab of + # the channel page and likely replaces the reelItemRenderer structure. It is + # usually (always?) encapsulated in a richItemRenderer. + # + module ShortsLockupViewModelParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["shortsLockupViewModel"]? + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + # TODO: Maybe add support for "oardefault.jpg" thumbnails? + # thumbnail = item_contents.dig("thumbnail", "sources", 0, "url").as_s + # Gives: https://i.ytimg.com/vi/{video_id}/oardefault.jpg?... + + video_id = item_contents.dig( + "onTap", "innertubeCommand", "reelWatchEndpoint", "videoId" + ).as_s + + title = item_contents.dig("overlayMetadata", "primaryText", "content").as_s + + view_count = short_text_to_number( + item_contents.dig("overlayMetadata", "secondaryText", "content").as_s + ) + + # Approximate to one minute, as "shorts" generally don't exceed that. + # NOTE: The actual duration is not provided by Youtube anymore. + # TODO: Maybe use -1 as an error value and handle that on the frontend? + duration = 60_i32 + + SearchVideo.new({ + title: title, + id: video_id, + author: author_fallback.name, + ucid: author_fallback.id, + published: Time.unix(0), + views: view_count, + description_html: "", + length_seconds: duration, + premiere_timestamp: Time.unix(0), + author_verified: false, + author_thumbnail: nil, badges: VideoBadges::None, }) end @@ -889,7 +1029,7 @@ end def extract_items( initial_data : InitialData, author_fallback : String? = nil, - author_id_fallback : String? = nil + author_id_fallback : String? = nil, ) : {Array(SearchItem), String?} items = [] of SearchItem continuation = nil diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index baa3cd92..ec080d8c 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -211,7 +211,7 @@ module YoutubeAPI def initialize( *, @client_type = ClientType::Web, - @region = "US" + @region = "US", ) end @@ -300,9 +300,8 @@ module YoutubeAPI end if client_config.screen == "EMBED" - # embedUrl https://www.google.com allow loading almost all video that are configured not embeddable client_context["thirdParty"] = { - "embedUrl" => "https://www.google.com/", + "embedUrl" => "https://www.youtube.com/embed/#{video_id}", } of String => String | Int64 end @@ -371,7 +370,7 @@ module YoutubeAPI browse_id : String, *, # Force the following parameters to be passed by name params : String, - client_config : ClientConfig | Nil = nil + client_config : ClientConfig | Nil = nil, ) # JSON Request data, required by the API data = { @@ -465,7 +464,7 @@ module YoutubeAPI video_id : String, *, # Force the following parameters to be passed by name params : String, - client_config : ClientConfig | Nil = nil + client_config : ClientConfig | Nil = nil, ) # Playback context, separate because it can be different between clients playback_ctx = { @@ -558,7 +557,7 @@ module YoutubeAPI def search( search_query : String, params : String, - client_config : ClientConfig | Nil = nil + client_config : ClientConfig | Nil = nil, ) # JSON Request data, required by the API data = { @@ -584,7 +583,7 @@ module YoutubeAPI def get_transcript( params : String, - client_config : ClientConfig | Nil = nil + client_config : ClientConfig | Nil = nil, ) : Hash(String, JSON::Any) data = { "context" => self.make_context(client_config), @@ -606,7 +605,7 @@ module YoutubeAPI def _post_json( endpoint : String, data : Hash, - client_config : ClientConfig | Nil + client_config : ClientConfig | Nil, ) : Hash(String, JSON::Any) # Use the default client config if nil is passed client_config ||= DEFAULT_CLIENT_CONFIG @@ -638,6 +637,11 @@ module YoutubeAPI # Send the POST request body = YT_POOL.client() do |client| client.post(url, headers: headers, body: data.to_json) do |response| + if response.status_code != 200 + raise InfoException.new("Error: non 200 status code. Youtube API returned \ + status code #{response.status_code}. See <a href=\"https://docs.invidious.io/youtube-errors-explained/\"> \ + https://docs.invidious.io/youtube-errors-explained/</a> for troubleshooting.") + end self._decompress(response.body_io, response.headers["Content-Encoding"]?) end end |