diff options
29 files changed, 230 insertions, 290 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ca0dc96..057e4d61 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,16 +38,16 @@ jobs: matrix: stable: [true] crystal: - - 1.6.2 - 1.7.3 - 1.8.2 - 1.9.2 + - 1.10.1 include: - crystal: nightly stable: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -87,7 +87,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build Docker run: docker-compose build --build-arg release=0 @@ -103,18 +103,18 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: arm64 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Build Docker ARM64 image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: docker/Dockerfile.arm64 diff --git a/.github/workflows/container-release.yml b/.github/workflows/container-release.yml index c2756fcc..e44ac200 100644 --- a/.github/workflows/container-release.yml +++ b/.github/workflows/container-release.yml @@ -11,7 +11,6 @@ on: - invidious.service - .git* - .editorconfig - - screenshots/* - .github/ISSUE_TEMPLATE/* - kubernetes/** @@ -22,7 +21,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Crystal uses: crystal-lang/install-crystal@v1.8.0 @@ -38,42 +37,64 @@ jobs: fi - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: arm64 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: quay.io username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_PASSWORD }} + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: quay.io/invidious/invidious + tags: | + type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + labels: | + quay.expires-after=12w + - name: Build and push Docker AMD64 image for Push Event - if: github.ref == 'refs/heads/master' - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: docker/Dockerfile platforms: linux/amd64 - labels: quay.expires-after=12w + labels: ${{ steps.meta.outputs.labels }} push: true - tags: quay.io/invidious/invidious:${{ github.sha }},quay.io/invidious/invidious:latest + tags: ${{ steps.meta.outputs.tags }} build-args: | "release=1" + - name: Docker meta + id: meta-arm64 + uses: docker/metadata-action@v5 + with: + images: quay.io/invidious/invidious + flavor: | + suffix=-arm64 + tags: | + type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + labels: | + quay.expires-after=12w + - name: Build and push Docker ARM64 image for Push Event - if: github.ref == 'refs/heads/master' - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: docker/Dockerfile.arm64 platforms: linux/arm64/v8 - labels: quay.expires-after=12w + labels: ${{ steps.meta-arm64.outputs.labels }} push: true - tags: quay.io/invidious/invidious:${{ github.sha }}-arm64,quay.io/invidious/invidious:latest-arm64 + tags: ${{ steps.meta-arm64.outputs.tags }} build-args: | "release=1" diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index a7e218a2..16d3269b 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -10,13 +10,13 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v5 + - uses: actions/stale@v8 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 365 days-before-pr-stale: 90 days-before-close: 30 - exempt-pr-labels: blocked + exempt-pr-labels: blocked,exempt-stale stale-issue-message: 'This issue has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely outdated. If you think this issue is still relevant and applicable, you just have to post a comment and it will be unmarked.' stale-pr-message: 'This pull request has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely abandoned or outdated. If you think this pull request is still relevant and applicable, you just have to post a comment and it will be unmarked.' stale-issue-label: "stale" @@ -145,18 +145,7 @@ Weblate also allows you to log-in with major SSO providers like Github, Gitlab, ## Projects using Invidious -- [FreeTube](https://github.com/FreeTubeApp/FreeTube): A libre software YouTube app for privacy. -- [CloudTube](https://sr.ht/~cadence/tube/): A JavaScript-rich alternate YouTube player. -- [PeerTubeify](https://gitlab.com/Cha_de_L/peertubeify): On YouTube, displays a link to the same video on PeerTube, if it exists. -- [MusicPiped](https://github.com/deep-gaurav/MusicPiped): A material design music player that streams music from YouTube. -- [HoloPlay](https://github.com/stephane-r/holoplay-pwa): Progressive Web App connecting on Invidious API's with search, playlists and favorites. -- [WatchTube](https://github.com/WatchTubeTeam/WatchTube): Powerful YouTube client for Apple Watch. -- [Yattee](https://github.com/yattee/yattee): Alternative YouTube frontend for iPhone, iPad, Mac and Apple TV. -- [TubiTui](https://codeberg.org/777/TubiTui): A lightweight, libre, TUI-based YouTube client. -- [Ytfzf](https://github.com/pystardust/ytfzf): A posix script to find and watch youtube videos from the terminal. (Without API). -- [Playlet](https://github.com/iBicha/playlet): Unofficial Youtube client for Roku TV. -- [Clipious](https://github.com/lamarios/clipious): Unofficial Invidious client for Android. - +A list of projects and extensions for or utilizing Invidious can be found in the documentation: https://docs.invidious.io/applications/ ## Liability diff --git a/assets/js/player.js b/assets/js/player.js index 16bb2752..71c5e7da 100644 --- a/assets/js/player.js +++ b/assets/js/player.js @@ -747,6 +747,17 @@ if (navigator.vendor === 'Apple Computer, Inc.' && video_data.params.listen) { }); } +// Safari screen timeout on looped video playback fix +if (navigator.vendor === 'Apple Computer, Inc.' && !video_data.params.listen && video_data.params.video_loop) { + player.loop(false); + player.ready(function () { + player.on('ended', function () { + player.currentTime(0); + player.play(); + }); + }); +} + // Watch on Invidious link if (location.pathname.startsWith('/embed/')) { const Button = videojs.getComponent('Button'); diff --git a/config/config.example.yml b/config/config.example.yml index b44fcc0e..38085a20 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -393,27 +393,6 @@ jobs: # ----------------------------- -# Captcha API -# ----------------------------- - -## -## URL of the captcha solving service. -## -## Accepted values: any URL -## Default: https://api.anti-captcha.com -## -#captcha_api_url: https://api.anti-captcha.com - -## -## API key for the captcha solving service. -## -## Accepted values: a string -## Default: <none> -## -#captcha_key: - - -# ----------------------------- # Miscellaneous # ----------------------------- diff --git a/docker-compose.yml b/docker-compose.yml index d879919a..42a5c06b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,8 @@ services: timeout: 5s retries: 2 depends_on: - - invidious-db + invidious-db: + condition: service_healthy invidious-db: image: docker.io/library/postgres:14 diff --git a/docker/Dockerfile b/docker/Dockerfile index c9644ca6..ace096bf 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -33,7 +33,7 @@ RUN if [[ "${release}" == 1 ]] ; then \ fi FROM alpine:3.18 -RUN apk add --no-cache librsvg ttf-opensans tini +RUN apk add --no-cache rsvg-convert ttf-opensans tini WORKDIR /invidious RUN addgroup -g 1000 -S invidious && \ adduser -u 1000 -S invidious -G invidious diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index d9a4eeaf..602f3ab2 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -33,7 +33,7 @@ RUN if [[ "${release}" == 1 ]] ; then \ fi FROM alpine:3.18 -RUN apk add --no-cache librsvg ttf-opensans tini +RUN apk add --no-cache rsvg-convert ttf-opensans tini WORKDIR /invidious RUN addgroup -g 1000 -S invidious && \ adduser -u 1000 -S invidious -G invidious diff --git a/spec/invidious/helpers_spec.cr b/spec/invidious/helpers_spec.cr index 142e1653..9fbb6d6f 100644 --- a/spec/invidious/helpers_spec.cr +++ b/spec/invidious/helpers_spec.cr @@ -3,18 +3,6 @@ require "../spec_helper" CONFIG = Config.from_yaml(File.open("config/config.example.yml")) Spectator.describe "Helper" do - describe "#produce_channel_videos_url" do - it "correctly produces url for requesting page `x` of a channel's videos" do - # expect(produce_channel_videos_url(ucid: "UCXuqSBlHAE6Xw-yeJA0Tunw")).to eq("/browse_ajax?continuation=4qmFsgI8EhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaIEVnWjJhV1JsYjNNd0FqZ0JZQUZxQUxnQkFDQUFlZ0V4&gl=US&hl=en") - # - # expect(produce_channel_videos_url(ucid: "UCXuqSBlHAE6Xw-yeJA0Tunw", sort_by: "popular")).to eq("/browse_ajax?continuation=4qmFsgJAEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaJEVnWjJhV1JsYjNNd0FqZ0JZQUZxQUxnQkFDQUFlZ0V4R0FFPQ%3D%3D&gl=US&hl=en") - - # expect(produce_channel_videos_url(ucid: "UCXuqSBlHAE6Xw-yeJA0Tunw", page: 20)).to eq("/browse_ajax?continuation=4qmFsgJAEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaJEVnWjJhV1JsYjNNd0FqZ0JZQUZxQUxnQkFDQUFlZ0l5TUE9PQ%3D%3D&gl=US&hl=en") - - # expect(produce_channel_videos_url(ucid: "UC-9-kyTW8ZkZNDHQJ6FgpwQ", page: 20, sort_by: "popular")).to eq("/browse_ajax?continuation=4qmFsgJAEhhVQy05LWt5VFc4WmtaTkRIUUo2Rmdwd1EaJEVnWjJhV1JsYjNNd0FqZ0JZQUZxQUxnQkFDQUFlZ0l5TUJnQg%3D%3D&gl=US&hl=en") - end - end - describe "#produce_channel_search_continuation" do it "correctly produces token for searching a specific channel" do expect(produce_channel_search_continuation("UCXuqSBlHAE6Xw-yeJA0Tunw", "", 100)).to eq("4qmFsgJqEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaIEVnWnpaV0Z5WTJnd0FUZ0JZQUY2QkVkS2IxaTRBUUE9WgCaAilicm93c2UtZmVlZFVDWHVxU0JsSEFFNlh3LXllSkEwVHVud3NlYXJjaA%3D%3D") diff --git a/spec/invidious/search/yt_filters_spec.cr b/spec/invidious/search/yt_filters_spec.cr index bf7f21e7..8abed5ce 100644 --- a/spec/invidious/search/yt_filters_spec.cr +++ b/spec/invidious/search/yt_filters_spec.cr @@ -12,45 +12,45 @@ end # page of Youtube with any browser devtools HTML inspector. DATE_FILTERS = { - Invidious::Search::Filters::Date::Hour => "EgIIAQ%3D%3D", - Invidious::Search::Filters::Date::Today => "EgIIAg%3D%3D", - Invidious::Search::Filters::Date::Week => "EgIIAw%3D%3D", - Invidious::Search::Filters::Date::Month => "EgIIBA%3D%3D", - Invidious::Search::Filters::Date::Year => "EgIIBQ%3D%3D", + Invidious::Search::Filters::Date::Hour => "EgIIAfABAQ%3D%3D", + Invidious::Search::Filters::Date::Today => "EgIIAvABAQ%3D%3D", + Invidious::Search::Filters::Date::Week => "EgIIA_ABAQ%3D%3D", + Invidious::Search::Filters::Date::Month => "EgIIBPABAQ%3D%3D", + Invidious::Search::Filters::Date::Year => "EgIIBfABAQ%3D%3D", } TYPE_FILTERS = { - Invidious::Search::Filters::Type::Video => "EgIQAQ%3D%3D", - Invidious::Search::Filters::Type::Channel => "EgIQAg%3D%3D", - Invidious::Search::Filters::Type::Playlist => "EgIQAw%3D%3D", - Invidious::Search::Filters::Type::Movie => "EgIQBA%3D%3D", + Invidious::Search::Filters::Type::Video => "EgIQAfABAQ%3D%3D", + Invidious::Search::Filters::Type::Channel => "EgIQAvABAQ%3D%3D", + Invidious::Search::Filters::Type::Playlist => "EgIQA_ABAQ%3D%3D", + Invidious::Search::Filters::Type::Movie => "EgIQBPABAQ%3D%3D", } DURATION_FILTERS = { - Invidious::Search::Filters::Duration::Short => "EgIYAQ%3D%3D", - Invidious::Search::Filters::Duration::Medium => "EgIYAw%3D%3D", - Invidious::Search::Filters::Duration::Long => "EgIYAg%3D%3D", + Invidious::Search::Filters::Duration::Short => "EgIYAfABAQ%3D%3D", + Invidious::Search::Filters::Duration::Medium => "EgIYA_ABAQ%3D%3D", + Invidious::Search::Filters::Duration::Long => "EgIYAvABAQ%3D%3D", } FEATURE_FILTERS = { - Invidious::Search::Filters::Features::Live => "EgJAAQ%3D%3D", - Invidious::Search::Filters::Features::FourK => "EgJwAQ%3D%3D", - Invidious::Search::Filters::Features::HD => "EgIgAQ%3D%3D", - Invidious::Search::Filters::Features::Subtitles => "EgIoAQ%3D%3D", - Invidious::Search::Filters::Features::CCommons => "EgIwAQ%3D%3D", - Invidious::Search::Filters::Features::ThreeSixty => "EgJ4AQ%3D%3D", - Invidious::Search::Filters::Features::VR180 => "EgPQAQE%3D", - Invidious::Search::Filters::Features::ThreeD => "EgI4AQ%3D%3D", - Invidious::Search::Filters::Features::HDR => "EgPIAQE%3D", - Invidious::Search::Filters::Features::Location => "EgO4AQE%3D", - Invidious::Search::Filters::Features::Purchased => "EgJIAQ%3D%3D", + Invidious::Search::Filters::Features::Live => "EgJAAfABAQ%3D%3D", + Invidious::Search::Filters::Features::FourK => "EgJwAfABAQ%3D%3D", + Invidious::Search::Filters::Features::HD => "EgIgAfABAQ%3D%3D", + Invidious::Search::Filters::Features::Subtitles => "EgIoAfABAQ%3D%3D", + Invidious::Search::Filters::Features::CCommons => "EgIwAfABAQ%3D%3D", + Invidious::Search::Filters::Features::ThreeSixty => "EgJ4AfABAQ%3D%3D", + Invidious::Search::Filters::Features::VR180 => "EgPQAQHwAQE%3D", + Invidious::Search::Filters::Features::ThreeD => "EgI4AfABAQ%3D%3D", + Invidious::Search::Filters::Features::HDR => "EgPIAQHwAQE%3D", + Invidious::Search::Filters::Features::Location => "EgO4AQHwAQE%3D", + Invidious::Search::Filters::Features::Purchased => "EgJIAfABAQ%3D%3D", } SORT_FILTERS = { - Invidious::Search::Filters::Sort::Relevance => "", - Invidious::Search::Filters::Sort::Date => "CAI%3D", - Invidious::Search::Filters::Sort::Views => "CAM%3D", - Invidious::Search::Filters::Sort::Rating => "CAE%3D", + Invidious::Search::Filters::Sort::Relevance => "8AEB", + Invidious::Search::Filters::Sort::Date => "CALwAQE%3D", + Invidious::Search::Filters::Sort::Views => "CAPwAQE%3D", + Invidious::Search::Filters::Sort::Rating => "CAHwAQE%3D", } Spectator.describe Invidious::Search::Filters do diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index 0054f8f2..8b60a728 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -18,8 +18,8 @@ record AboutChannel, def get_about_info(ucid, locale) : AboutChannel begin - # "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"} - initdata = YoutubeAPI.browse(browse_id: ucid, params: "EgVhYm91dA==") + # Fetch channel information from channel home page + initdata = YoutubeAPI.browse(browse_id: ucid, params: "") rescue raise InfoException.new("Could not get channel info.") end diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index c3d6124f..be739673 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -93,7 +93,7 @@ struct ChannelVideo def to_tuple {% begin %} { - {{*@type.instance_vars.map(&.name)}} + {{@type.instance_vars.map(&.name).splat}} } {% end %} end diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index beb86e08..351790d7 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -62,12 +62,6 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so return continuation end -# Used in bypass_captcha_job.cr -def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2) - return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" -end - module Invidious::Channel::Tabs extend self diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 429d9246..09c2168b 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -48,7 +48,7 @@ struct ConfigPreferences def to_tuple {% begin %} { - {{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}} + {{(@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }).splat}} } {% end %} end @@ -133,10 +133,6 @@ class Config # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] property cookies : HTTP::Cookies = HTTP::Cookies.new - # Key for Anti-Captcha - property captcha_key : String? = nil - # API URL for Anti-Captcha - property captcha_api_url : String = "https://api.anti-captcha.com" # Playlist length limit property playlist_length_limit : Int32 = 500 diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr index 6e5a975d..21b789bc 100644 --- a/src/invidious/helpers/errors.cr +++ b/src/invidious/helpers/errors.cr @@ -3,7 +3,7 @@ # ------------------- macro error_template(*args) - error_template_helper(env, {{*args}}) + error_template_helper(env, {{args.splat}}) end def github_details(summary : String, content : String) @@ -95,7 +95,7 @@ end # ------------------- macro error_atom(*args) - error_atom_helper(env, {{*args}}) + error_atom_helper(env, {{args.splat}}) end def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception) @@ -121,7 +121,7 @@ end # ------------------- macro error_json(*args) - error_json_helper(env, {{*args}}) + error_json_helper(env, {{args.splat}}) end def error_json_helper( diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 23ff0da9..6dc9860e 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -208,3 +208,20 @@ def proxy_file(response, env) IO.copy response.body_io, env.response end end + +# Fetch the playback requests tracker from the statistics endpoint. +# +# Creates a new tracker when unavailable. +def get_playback_statistic + if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]) && tracker.as(Hash).empty? + tracker = { + "totalRequests" => 0_i64, + "successfulRequests" => 0_i64, + "ratio" => 0_f64, + } + + Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"] = tracker + end + + return tracker.as(Hash(String, Int64 | Float64)) +end diff --git a/src/invidious/jobs/bypass_captcha_job.cr b/src/invidious/jobs/bypass_captcha_job.cr deleted file mode 100644 index 71f8a938..00000000 --- a/src/invidious/jobs/bypass_captcha_job.cr +++ /dev/null @@ -1,135 +0,0 @@ -class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob - def begin - loop do - begin - random_video = PG_DB.query_one?("select id, ucid from (select id, ucid from channel_videos limit 1000) as s ORDER BY RANDOM() LIMIT 1", as: {id: String, ucid: String}) - if !random_video - random_video = {id: "zj82_v2R6ts", ucid: "UCK87Lox575O_HCHBWaBSyGA"} - end - {"/watch?v=#{random_video["id"]}&gl=US&hl=en&has_verified=1&bpctr=9999999999", produce_channel_videos_url(ucid: random_video["ucid"])}.each do |path| - response = YT_POOL.client &.get(path) - if response.body.includes?("To continue with your YouTube experience, please fill out the form below.") - html = XML.parse_html(response.body) - form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil! - site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"] - s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"] - - inputs = {} of String => String - form.xpath_nodes(%(.//input[@name])).map do |node| - inputs[node["name"]] = node["value"] - end - - headers = response.cookies.add_request_headers(HTTP::Headers.new) - - response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/createTask", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "task" => { - "type" => "NoCaptchaTaskProxyless", - "websiteURL" => "https://www.youtube.com#{path}", - "websiteKey" => site_key, - "recaptchaDataSValue" => s_value, - }, - }.to_json).body) - - raise response["error"].as_s if response["error"]? - task_id = response["taskId"].as_i - - loop do - sleep 10.seconds - - response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/getTaskResult", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "taskId" => task_id, - }.to_json).body) - - if response["status"]?.try &.== "ready" - break - elsif response["errorId"]?.try &.as_i != 0 - raise response["errorDescription"].as_s - end - end - - inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s - headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || "" - response = YT_POOL.client &.post("/das_captcha", headers, form: inputs) - - response.cookies - .select { |cookie| cookie.name != "PREF" } - .each { |cookie| CONFIG.cookies << cookie } - - # Persist cookies between runs - File.write("config/config.yml", CONFIG.to_yaml) - elsif response.headers["Location"]?.try &.includes?("/sorry/index") - location = response.headers["Location"].try { |u| URI.parse(u) } - headers = HTTP::Headers{":authority" => location.host.not_nil!} - response = YT_POOL.client &.get(location.request_target, headers) - - html = XML.parse_html(response.body) - form = html.xpath_node(%(//form[@action="index"])).not_nil! - site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"] - s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"] - - inputs = {} of String => String - form.xpath_nodes(%(.//input[@name])).map do |node| - inputs[node["name"]] = node["value"] - end - - captcha_client = HTTPClient.new(URI.parse(CONFIG.captcha_api_url)) - captcha_client.family = CONFIG.force_resolve || Socket::Family::INET - response = JSON.parse(captcha_client.post("/createTask", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "task" => { - "type" => "NoCaptchaTaskProxyless", - "websiteURL" => location.to_s, - "websiteKey" => site_key, - "recaptchaDataSValue" => s_value, - }, - }.to_json).body) - - captcha_client.close - - raise response["error"].as_s if response["error"]? - task_id = response["taskId"].as_i - - loop do - sleep 10.seconds - - response = JSON.parse(captcha_client.post("/getTaskResult", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "taskId" => task_id, - }.to_json).body) - - if response["status"]?.try &.== "ready" - break - elsif response["errorId"]?.try &.as_i != 0 - raise response["errorDescription"].as_s - end - end - - inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s - headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || "" - response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs) - headers = HTTP::Headers{ - "Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0], - } - cookies = HTTP::Cookies.from_client_headers(headers) - - cookies.each { |cookie| CONFIG.cookies << cookie } - - # Persist cookies between runs - File.write("config/config.yml", CONFIG.to_yaml) - end - end - rescue ex - LOGGER.error("BypassCaptchaJob: #{ex.message}") - ensure - sleep 1.minute - Fiber.yield - end - end - end -end diff --git a/src/invidious/jobs/statistics_refresh_job.cr b/src/invidious/jobs/statistics_refresh_job.cr index a113bd77..72d1ce88 100644 --- a/src/invidious/jobs/statistics_refresh_job.cr +++ b/src/invidious/jobs/statistics_refresh_job.cr @@ -18,6 +18,13 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob "updatedAt" => Time.utc.to_unix, "lastChannelRefreshedAt" => 0_i64, }, + + # + # "totalRequests" => 0_i64, + # "successfulRequests" => 0_i64 + # "ratio" => 0_i64 + # + "playback" => {} of String => Int64 | Float64, } private getter db : DB::Database @@ -30,7 +37,7 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob loop do refresh_stats - sleep 1.minute + sleep 10.minute Fiber.yield end end @@ -56,5 +63,8 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob "updatedAt" => Time.utc.to_unix, "lastChannelRefreshedAt" => Invidious::Database::Statistics.channel_last_update.try &.to_unix || 0_i64, } + + # Reset playback requests tracker + STATISTICS["playback"] = {} of String => Int64 | Float64 end end diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index bad47709..12942906 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -6,6 +6,22 @@ module Invidious::Routes::API::V1::Misc if !CONFIG.statistics_enabled return {"software" => SOFTWARE}.to_json else + # Calculate playback success rate + if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]?) + tracker = tracker.as(Hash(String, Int64 | Float64)) + + if !tracker.empty? + total_requests = tracker["totalRequests"] + success_count = tracker["successfulRequests"] + + if total_requests.zero? + tracker["ratio"] = 1_i64 + else + tracker["ratio"] = (success_count / (total_requests)).round(2) + end + end + end + return Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json end end diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 40bca008..e20a7139 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -407,14 +407,23 @@ module Invidious::Routes::Feeds end spawn do - rss = XML.parse_html(body) - rss.xpath_nodes("//feed/entry").each do |entry| - id = entry.xpath_node("videoid").not_nil!.content - author = entry.xpath_node("author/name").not_nil!.content - published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) - updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) - - video = get_video(id, force_refresh: true) + # TODO: unify this with the other almost identical looking parts in this and channels.cr somehow? + namespaces = { + "yt" => "http://www.youtube.com/xml/schemas/2015", + "default" => "http://www.w3.org/2005/Atom", + } + rss = XML.parse(body) + rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry| + id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content + author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content + published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content) + updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content) + + begin + video = get_video(id, force_refresh: true) + rescue + next # skip this video since it raised an exception (e.g. it is a scheduled live event) + end if CONFIG.enable_user_notifications # Deliver notifications to `/api/v1/auth/notifications` diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 9641e01a..ec18f3b8 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -42,7 +42,7 @@ module Invidious::Routes::VideoPlayback headers["Range"] = "bytes=#{range_for_head}" end - client = make_client(URI.parse(host), region) + client = make_client(URI.parse(host), region, force_resolve = true) response = HTTP::Client::Response.new(500) error = "" 5.times do @@ -57,7 +57,7 @@ module Invidious::Routes::VideoPlayback if new_host != host host = new_host client.close - client = make_client(URI.parse(new_host), region) + client = make_client(URI.parse(new_host), region, force_resolve = true) end url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" @@ -71,7 +71,7 @@ module Invidious::Routes::VideoPlayback fvip = "3" host = "https://r#{fvip}---#{mn}.googlevideo.com" - client = make_client(URI.parse(host), region) + client = make_client(URI.parse(host), region, force_resolve = true) rescue ex error = ex.message end @@ -80,9 +80,14 @@ module Invidious::Routes::VideoPlayback # Remove the Range header added previously. headers.delete("Range") if range_header.nil? + playback_statistics = get_playback_statistic() + playback_statistics["totalRequests"] += 1 + if response.status_code >= 400 env.response.content_type = "text/plain" haltf env, response.status_code + else + playback_statistics["successfulRequests"] += 1 end if url.includes? "&file=seg.ts" @@ -191,7 +196,7 @@ module Invidious::Routes::VideoPlayback break else client.close - client = make_client(URI.parse(host), region) + client = make_client(URI.parse(host), region, force_resolve = true) end end diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr index c2b5c758..bf968734 100644 --- a/src/invidious/search/filters.cr +++ b/src/invidious/search/filters.cr @@ -300,9 +300,9 @@ module Invidious::Search object["9:varint"] = ((page - 1) * 20).to_i64 end - # If the object is empty, return an empty string, - # otherwise encode to protobuf then to base64 - return "" if object.empty? + # Prevent censoring of self harm topics + # See https://github.com/iv-org/invidious/issues/4398 + object["30:varint"] = 1.to_i64 return object .try { |i| Protodec::Any.cast_json(i) } diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 9fbd1374..a8f02056 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -227,8 +227,22 @@ struct Video info.dig?("streamingData", "hlsManifestUrl").try &.as_s end - def dash_manifest_url - info.dig?("streamingData", "dashManifestUrl").try &.as_s + def dash_manifest_url : String? + raw_dash_url = info.dig?("streamingData", "dashManifestUrl").try &.as_s + return nil if raw_dash_url.nil? + + # Use manifest v5 parameter to reduce file size + # See https://github.com/iv-org/invidious/issues/4186 + dash_url = URI.parse(raw_dash_url) + dash_query = dash_url.query || "" + + if dash_query.empty? + dash_url.path = "#{dash_url.path}/mpd_version/5" + else + dash_url.query = "#{dash_query}&mpd_version=5" + end + + return dash_url.to_s end def genre_url : String? diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 551ce2cb..77520dbe 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -78,6 +78,11 @@ def extract_video_info(video_id : String, proxy_region : String? = nil) # YouTube may return a different video player response than expected. # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 # Line to be reverted if one day we solve the video not available issue. + + # Although technically not a call to /videoplayback the fact that YouTube is returning the + # wrong video means that we should count it as a failure. + get_playback_statistic()["totalRequests"] += 1 + return { "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), "reason" => JSON::Any.new("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. <a href=\"https://github.com/iv-org/invidious/issues/3822\">Click here for more info about the issue.</a>"), diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index 031b46da..6d227cfc 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -82,11 +82,19 @@ </div> <div class="video-card-row flexible"> - <div class="flex-left"><a href="/channel/<%= item.ucid %>"> - <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> - <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> - </p> - </a></div> + <div class="flex-left"> + <% if !item.ucid.to_s.empty? %> + <a href="/channel/<%= item.ucid %>"> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + </a> + <% else %> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + <% end %> + </div> </div> <% when Category %> <% else %> @@ -160,11 +168,19 @@ </div> <div class="video-card-row flexible"> - <div class="flex-left"><a href="/channel/<%= item.ucid %>"> - <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> - <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> - </p> - </a></div> + <div class="flex-left"> + <% if !item.ucid.to_s.empty? %> + <a href="/channel/<%= item.ucid %>"> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + </a> + <% else %> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + <% end %> + </div> <%= rendered "components/video-context-buttons" %> </div> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 07474896..1b020321 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -346,7 +346,7 @@ we're going to need to do it here in order to allow for translations. <h5 class="pure-g"> <div class="pure-u-14-24"> - <% if rv["ucid"]? %> + <% if !rv["ucid"].empty? %> <b style="width:100%"><a href="/channel/<%= rv["ucid"] %>"><%= rv["author"]? %><% if rv["author_verified"]? == "true" %> <i class="icon ion ion-md-checkmark-circle"></i><% end %></a></b> <% else %> <b style="width:100%"><%= rv["author"]? %><% if rv["author_verified"]? == "true" %> <i class="icon ion ion-md-checkmark-circle"></i><% end %></b> diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index e9eb726c..81cfb272 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -1,7 +1,6 @@ def add_yt_headers(request) - if request.headers["User-Agent"] == "Crystal" - request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" - end + request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" + request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" @@ -27,7 +26,7 @@ struct YoutubeConnectionPool def client(region = nil, &block) if region - conn = make_client(url, region) + conn = make_client(url, region, force_resolve = true) response = yield conn else conn = pool.checkout @@ -37,7 +36,7 @@ struct YoutubeConnectionPool conn.close conn = HTTP::Client.new(url) - conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET + conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" response = yield conn @@ -52,7 +51,7 @@ struct YoutubeConnectionPool private def build_pool DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do conn = HTTP::Client.new(url) - conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET + conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" conn @@ -60,9 +59,14 @@ struct YoutubeConnectionPool end end -def make_client(url : URI, region = nil) +def make_client(url : URI, region = nil, force_resolve : Bool = false) client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure) - client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC + + # Some services do not support IPv6. + if force_resolve + client.family = CONFIG.force_resolve + end + client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" client.read_timeout = 10.seconds client.connect_timeout = 10.seconds @@ -81,8 +85,8 @@ def make_client(url : URI, region = nil) return client end -def make_client(url : URI, region = nil, &block) - client = make_client(url, region) +def make_client(url : URI, region = nil, force_resolve : Bool = false, &block) + client = make_client(url, region, force_resolve) begin yield client ensure diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 56325cf7..0e72957e 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -822,9 +822,9 @@ module HelperExtractors end # Retrieves the ID required for querying the InnerTube browse endpoint. - # Raises when it's unable to do so + # Returns an empty string when it's unable to do so def self.get_browse_id(container) - return container.dig("navigationEndpoint", "browseEndpoint", "browseId").as_s + return container.dig?("navigationEndpoint", "browseEndpoint", "browseId").try &.as_s || "" end end |