diff options
30 files changed, 251 insertions, 287 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ca0dc96..057e4d61 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,16 +38,16 @@ jobs: matrix: stable: [true] crystal: - - 1.6.2 - 1.7.3 - 1.8.2 - 1.9.2 + - 1.10.1 include: - crystal: nightly stable: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -87,7 +87,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build Docker run: docker-compose build --build-arg release=0 @@ -103,18 +103,18 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: arm64 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Build Docker ARM64 image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: docker/Dockerfile.arm64 diff --git a/.github/workflows/container-release.yml b/.github/workflows/container-release.yml index c2756fcc..e44ac200 100644 --- a/.github/workflows/container-release.yml +++ b/.github/workflows/container-release.yml @@ -11,7 +11,6 @@ on: - invidious.service - .git* - .editorconfig - - screenshots/* - .github/ISSUE_TEMPLATE/* - kubernetes/** @@ -22,7 +21,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Crystal uses: crystal-lang/install-crystal@v1.8.0 @@ -38,42 +37,64 @@ jobs: fi - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: arm64 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: quay.io username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_PASSWORD }} + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: quay.io/invidious/invidious + tags: | + type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + labels: | + quay.expires-after=12w + - name: Build and push Docker AMD64 image for Push Event - if: github.ref == 'refs/heads/master' - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: docker/Dockerfile platforms: linux/amd64 - labels: quay.expires-after=12w + labels: ${{ steps.meta.outputs.labels }} push: true - tags: quay.io/invidious/invidious:${{ github.sha }},quay.io/invidious/invidious:latest + tags: ${{ steps.meta.outputs.tags }} build-args: | "release=1" + - name: Docker meta + id: meta-arm64 + uses: docker/metadata-action@v5 + with: + images: quay.io/invidious/invidious + flavor: | + suffix=-arm64 + tags: | + type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + labels: | + quay.expires-after=12w + - name: Build and push Docker ARM64 image for Push Event - if: github.ref == 'refs/heads/master' - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: docker/Dockerfile.arm64 platforms: linux/arm64/v8 - labels: quay.expires-after=12w + labels: ${{ steps.meta-arm64.outputs.labels }} push: true - tags: quay.io/invidious/invidious:${{ github.sha }}-arm64,quay.io/invidious/invidious:latest-arm64 + tags: ${{ steps.meta-arm64.outputs.tags }} build-args: | "release=1" diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index a7e218a2..16d3269b 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -10,13 +10,13 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v5 + - uses: actions/stale@v8 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 365 days-before-pr-stale: 90 days-before-close: 30 - exempt-pr-labels: blocked + exempt-pr-labels: blocked,exempt-stale stale-issue-message: 'This issue has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely outdated. If you think this issue is still relevant and applicable, you just have to post a comment and it will be unmarked.' stale-pr-message: 'This pull request has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely abandoned or outdated. If you think this pull request is still relevant and applicable, you just have to post a comment and it will be unmarked.' stale-issue-label: "stale" @@ -145,18 +145,7 @@ Weblate also allows you to log-in with major SSO providers like Github, Gitlab, ## Projects using Invidious -- [FreeTube](https://github.com/FreeTubeApp/FreeTube): A libre software YouTube app for privacy. -- [CloudTube](https://sr.ht/~cadence/tube/): A JavaScript-rich alternate YouTube player. -- [PeerTubeify](https://gitlab.com/Cha_de_L/peertubeify): On YouTube, displays a link to the same video on PeerTube, if it exists. -- [MusicPiped](https://github.com/deep-gaurav/MusicPiped): A material design music player that streams music from YouTube. -- [HoloPlay](https://github.com/stephane-r/holoplay-pwa): Progressive Web App connecting on Invidious API's with search, playlists and favorites. -- [WatchTube](https://github.com/WatchTubeTeam/WatchTube): Powerful YouTube client for Apple Watch. -- [Yattee](https://github.com/yattee/yattee): Alternative YouTube frontend for iPhone, iPad, Mac and Apple TV. -- [TubiTui](https://codeberg.org/777/TubiTui): A lightweight, libre, TUI-based YouTube client. -- [Ytfzf](https://github.com/pystardust/ytfzf): A posix script to find and watch youtube videos from the terminal. (Without API). -- [Playlet](https://github.com/iBicha/playlet): Unofficial Youtube client for Roku TV. -- [Clipious](https://github.com/lamarios/clipious): Unofficial Invidious client for Android. - +A list of projects and extensions for or utilizing Invidious can be found in the documentation: https://docs.invidious.io/applications/ ## Liability diff --git a/assets/js/player.js b/assets/js/player.js index 16bb2752..71c5e7da 100644 --- a/assets/js/player.js +++ b/assets/js/player.js @@ -747,6 +747,17 @@ if (navigator.vendor === 'Apple Computer, Inc.' && video_data.params.listen) { }); } +// Safari screen timeout on looped video playback fix +if (navigator.vendor === 'Apple Computer, Inc.' && !video_data.params.listen && video_data.params.video_loop) { + player.loop(false); + player.ready(function () { + player.on('ended', function () { + player.currentTime(0); + player.play(); + }); + }); +} + // Watch on Invidious link if (location.pathname.startsWith('/embed/')) { const Button = videojs.getComponent('Button'); diff --git a/config/config.example.yml b/config/config.example.yml index b44fcc0e..38085a20 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -393,27 +393,6 @@ jobs: # ----------------------------- -# Captcha API -# ----------------------------- - -## -## URL of the captcha solving service. -## -## Accepted values: any URL -## Default: https://api.anti-captcha.com -## -#captcha_api_url: https://api.anti-captcha.com - -## -## API key for the captcha solving service. -## -## Accepted values: a string -## Default: <none> -## -#captcha_key: - - -# ----------------------------- # Miscellaneous # ----------------------------- diff --git a/docker-compose.yml b/docker-compose.yml index d879919a..42a5c06b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,8 @@ services: timeout: 5s retries: 2 depends_on: - - invidious-db + invidious-db: + condition: service_healthy invidious-db: image: docker.io/library/postgres:14 diff --git a/docker/Dockerfile b/docker/Dockerfile index c9644ca6..ace096bf 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -33,7 +33,7 @@ RUN if [[ "${release}" == 1 ]] ; then \ fi FROM alpine:3.18 -RUN apk add --no-cache librsvg ttf-opensans tini +RUN apk add --no-cache rsvg-convert ttf-opensans tini WORKDIR /invidious RUN addgroup -g 1000 -S invidious && \ adduser -u 1000 -S invidious -G invidious diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index d9a4eeaf..602f3ab2 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -33,7 +33,7 @@ RUN if [[ "${release}" == 1 ]] ; then \ fi FROM alpine:3.18 -RUN apk add --no-cache librsvg ttf-opensans tini +RUN apk add --no-cache rsvg-convert ttf-opensans tini WORKDIR /invidious RUN addgroup -g 1000 -S invidious && \ adduser -u 1000 -S invidious -G invidious diff --git a/spec/invidious/helpers_spec.cr b/spec/invidious/helpers_spec.cr index 142e1653..9fbb6d6f 100644 --- a/spec/invidious/helpers_spec.cr +++ b/spec/invidious/helpers_spec.cr @@ -3,18 +3,6 @@ require "../spec_helper" CONFIG = Config.from_yaml(File.open("config/config.example.yml")) Spectator.describe "Helper" do - describe "#produce_channel_videos_url" do - it "correctly produces url for requesting page `x` of a channel's videos" do - # expect(produce_channel_videos_url(ucid: "UCXuqSBlHAE6Xw-yeJA0Tunw")).to eq("/browse_ajax?continuation=4qmFsgI8EhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaIEVnWjJhV1JsYjNNd0FqZ0JZQUZxQUxnQkFDQUFlZ0V4&gl=US&hl=en") - # - # expect(produce_channel_videos_url(ucid: "UCXuqSBlHAE6Xw-yeJA0Tunw", sort_by: "popular")).to eq("/browse_ajax?continuation=4qmFsgJAEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaJEVnWjJhV1JsYjNNd0FqZ0JZQUZxQUxnQkFDQUFlZ0V4R0FFPQ%3D%3D&gl=US&hl=en") - - # expect(produce_channel_videos_url(ucid: "UCXuqSBlHAE6Xw-yeJA0Tunw", page: 20)).to eq("/browse_ajax?continuation=4qmFsgJAEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaJEVnWjJhV1JsYjNNd0FqZ0JZQUZxQUxnQkFDQUFlZ0l5TUE9PQ%3D%3D&gl=US&hl=en") - - # expect(produce_channel_videos_url(ucid: "UC-9-kyTW8ZkZNDHQJ6FgpwQ", page: 20, sort_by: "popular")).to eq("/browse_ajax?continuation=4qmFsgJAEhhVQy05LWt5VFc4WmtaTkRIUUo2Rmdwd1EaJEVnWjJhV1JsYjNNd0FqZ0JZQUZxQUxnQkFDQUFlZ0l5TUJnQg%3D%3D&gl=US&hl=en") - end - end - describe "#produce_channel_search_continuation" do it "correctly produces token for searching a specific channel" do expect(produce_channel_search_continuation("UCXuqSBlHAE6Xw-yeJA0Tunw", "", 100)).to eq("4qmFsgJqEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaIEVnWnpaV0Z5WTJnd0FUZ0JZQUY2QkVkS2IxaTRBUUE9WgCaAilicm93c2UtZmVlZFVDWHVxU0JsSEFFNlh3LXllSkEwVHVud3NlYXJjaA%3D%3D") diff --git a/spec/invidious/search/yt_filters_spec.cr b/spec/invidious/search/yt_filters_spec.cr index bf7f21e7..8abed5ce 100644 --- a/spec/invidious/search/yt_filters_spec.cr +++ b/spec/invidious/search/yt_filters_spec.cr @@ -12,45 +12,45 @@ end # page of Youtube with any browser devtools HTML inspector. DATE_FILTERS = { - Invidious::Search::Filters::Date::Hour => "EgIIAQ%3D%3D", - Invidious::Search::Filters::Date::Today => "EgIIAg%3D%3D", - Invidious::Search::Filters::Date::Week => "EgIIAw%3D%3D", - Invidious::Search::Filters::Date::Month => "EgIIBA%3D%3D", - Invidious::Search::Filters::Date::Year => "EgIIBQ%3D%3D", + Invidious::Search::Filters::Date::Hour => "EgIIAfABAQ%3D%3D", + Invidious::Search::Filters::Date::Today => "EgIIAvABAQ%3D%3D", + Invidious::Search::Filters::Date::Week => "EgIIA_ABAQ%3D%3D", + Invidious::Search::Filters::Date::Month => "EgIIBPABAQ%3D%3D", + Invidious::Search::Filters::Date::Year => "EgIIBfABAQ%3D%3D", } TYPE_FILTERS = { - Invidious::Search::Filters::Type::Video => "EgIQAQ%3D%3D", - Invidious::Search::Filters::Type::Channel => "EgIQAg%3D%3D", - Invidious::Search::Filters::Type::Playlist => "EgIQAw%3D%3D", - Invidious::Search::Filters::Type::Movie => "EgIQBA%3D%3D", + Invidious::Search::Filters::Type::Video => "EgIQAfABAQ%3D%3D", + Invidious::Search::Filters::Type::Channel => "EgIQAvABAQ%3D%3D", + Invidious::Search::Filters::Type::Playlist => "EgIQA_ABAQ%3D%3D", + Invidious::Search::Filters::Type::Movie => "EgIQBPABAQ%3D%3D", } DURATION_FILTERS = { - Invidious::Search::Filters::Duration::Short => "EgIYAQ%3D%3D", - Invidious::Search::Filters::Duration::Medium => "EgIYAw%3D%3D", - Invidious::Search::Filters::Duration::Long => "EgIYAg%3D%3D", + Invidious::Search::Filters::Duration::Short => "EgIYAfABAQ%3D%3D", + Invidious::Search::Filters::Duration::Medium => "EgIYA_ABAQ%3D%3D", + Invidious::Search::Filters::Duration::Long => "EgIYAvABAQ%3D%3D", } FEATURE_FILTERS = { - Invidious::Search::Filters::Features::Live => "EgJAAQ%3D%3D", - Invidious::Search::Filters::Features::FourK => "EgJwAQ%3D%3D", - Invidious::Search::Filters::Features::HD => "EgIgAQ%3D%3D", - Invidious::Search::Filters::Features::Subtitles => "EgIoAQ%3D%3D", - Invidious::Search::Filters::Features::CCommons => "EgIwAQ%3D%3D", - Invidious::Search::Filters::Features::ThreeSixty => "EgJ4AQ%3D%3D", - Invidious::Search::Filters::Features::VR180 => "EgPQAQE%3D", - Invidious::Search::Filters::Features::ThreeD => "EgI4AQ%3D%3D", - Invidious::Search::Filters::Features::HDR => "EgPIAQE%3D", - Invidious::Search::Filters::Features::Location => "EgO4AQE%3D", - Invidious::Search::Filters::Features::Purchased => "EgJIAQ%3D%3D", + Invidious::Search::Filters::Features::Live => "EgJAAfABAQ%3D%3D", + Invidious::Search::Filters::Features::FourK => "EgJwAfABAQ%3D%3D", + Invidious::Search::Filters::Features::HD => "EgIgAfABAQ%3D%3D", + Invidious::Search::Filters::Features::Subtitles => "EgIoAfABAQ%3D%3D", + Invidious::Search::Filters::Features::CCommons => "EgIwAfABAQ%3D%3D", + Invidious::Search::Filters::Features::ThreeSixty => "EgJ4AfABAQ%3D%3D", + Invidious::Search::Filters::Features::VR180 => "EgPQAQHwAQE%3D", + Invidious::Search::Filters::Features::ThreeD => "EgI4AfABAQ%3D%3D", + Invidious::Search::Filters::Features::HDR => "EgPIAQHwAQE%3D", + Invidious::Search::Filters::Features::Location => "EgO4AQHwAQE%3D", + Invidious::Search::Filters::Features::Purchased => "EgJIAfABAQ%3D%3D", } SORT_FILTERS = { - Invidious::Search::Filters::Sort::Relevance => "", - Invidious::Search::Filters::Sort::Date => "CAI%3D", - Invidious::Search::Filters::Sort::Views => "CAM%3D", - Invidious::Search::Filters::Sort::Rating => "CAE%3D", + Invidious::Search::Filters::Sort::Relevance => "8AEB", + Invidious::Search::Filters::Sort::Date => "CALwAQE%3D", + Invidious::Search::Filters::Sort::Views => "CAPwAQE%3D", + Invidious::Search::Filters::Sort::Rating => "CAHwAQE%3D", } Spectator.describe Invidious::Search::Filters do diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index c3d6124f..be739673 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -93,7 +93,7 @@ struct ChannelVideo def to_tuple {% begin %} { - {{*@type.instance_vars.map(&.name)}} + {{@type.instance_vars.map(&.name).splat}} } {% end %} end diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index beb86e08..351790d7 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -62,12 +62,6 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so return continuation end -# Used in bypass_captcha_job.cr -def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2) - return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" -end - module Invidious::Channel::Tabs extend self diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 429d9246..09c2168b 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -48,7 +48,7 @@ struct ConfigPreferences def to_tuple {% begin %} { - {{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}} + {{(@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }).splat}} } {% end %} end @@ -133,10 +133,6 @@ class Config # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] property cookies : HTTP::Cookies = HTTP::Cookies.new - # Key for Anti-Captcha - property captcha_key : String? = nil - # API URL for Anti-Captcha - property captcha_api_url : String = "https://api.anti-captcha.com" # Playlist length limit property playlist_length_limit : Int32 = 500 diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr index 6e5a975d..21b789bc 100644 --- a/src/invidious/helpers/errors.cr +++ b/src/invidious/helpers/errors.cr @@ -3,7 +3,7 @@ # ------------------- macro error_template(*args) - error_template_helper(env, {{*args}}) + error_template_helper(env, {{args.splat}}) end def github_details(summary : String, content : String) @@ -95,7 +95,7 @@ end # ------------------- macro error_atom(*args) - error_atom_helper(env, {{*args}}) + error_atom_helper(env, {{args.splat}}) end def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception) @@ -121,7 +121,7 @@ end # ------------------- macro error_json(*args) - error_json_helper(env, {{*args}}) + error_json_helper(env, {{args.splat}}) end def error_json_helper( diff --git a/src/invidious/jobs/bypass_captcha_job.cr b/src/invidious/jobs/bypass_captcha_job.cr deleted file mode 100644 index 71f8a938..00000000 --- a/src/invidious/jobs/bypass_captcha_job.cr +++ /dev/null @@ -1,135 +0,0 @@ -class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob - def begin - loop do - begin - random_video = PG_DB.query_one?("select id, ucid from (select id, ucid from channel_videos limit 1000) as s ORDER BY RANDOM() LIMIT 1", as: {id: String, ucid: String}) - if !random_video - random_video = {id: "zj82_v2R6ts", ucid: "UCK87Lox575O_HCHBWaBSyGA"} - end - {"/watch?v=#{random_video["id"]}&gl=US&hl=en&has_verified=1&bpctr=9999999999", produce_channel_videos_url(ucid: random_video["ucid"])}.each do |path| - response = YT_POOL.client &.get(path) - if response.body.includes?("To continue with your YouTube experience, please fill out the form below.") - html = XML.parse_html(response.body) - form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil! - site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"] - s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"] - - inputs = {} of String => String - form.xpath_nodes(%(.//input[@name])).map do |node| - inputs[node["name"]] = node["value"] - end - - headers = response.cookies.add_request_headers(HTTP::Headers.new) - - response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/createTask", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "task" => { - "type" => "NoCaptchaTaskProxyless", - "websiteURL" => "https://www.youtube.com#{path}", - "websiteKey" => site_key, - "recaptchaDataSValue" => s_value, - }, - }.to_json).body) - - raise response["error"].as_s if response["error"]? - task_id = response["taskId"].as_i - - loop do - sleep 10.seconds - - response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/getTaskResult", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "taskId" => task_id, - }.to_json).body) - - if response["status"]?.try &.== "ready" - break - elsif response["errorId"]?.try &.as_i != 0 - raise response["errorDescription"].as_s - end - end - - inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s - headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || "" - response = YT_POOL.client &.post("/das_captcha", headers, form: inputs) - - response.cookies - .select { |cookie| cookie.name != "PREF" } - .each { |cookie| CONFIG.cookies << cookie } - - # Persist cookies between runs - File.write("config/config.yml", CONFIG.to_yaml) - elsif response.headers["Location"]?.try &.includes?("/sorry/index") - location = response.headers["Location"].try { |u| URI.parse(u) } - headers = HTTP::Headers{":authority" => location.host.not_nil!} - response = YT_POOL.client &.get(location.request_target, headers) - - html = XML.parse_html(response.body) - form = html.xpath_node(%(//form[@action="index"])).not_nil! - site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"] - s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"] - - inputs = {} of String => String - form.xpath_nodes(%(.//input[@name])).map do |node| - inputs[node["name"]] = node["value"] - end - - captcha_client = HTTPClient.new(URI.parse(CONFIG.captcha_api_url)) - captcha_client.family = CONFIG.force_resolve || Socket::Family::INET - response = JSON.parse(captcha_client.post("/createTask", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "task" => { - "type" => "NoCaptchaTaskProxyless", - "websiteURL" => location.to_s, - "websiteKey" => site_key, - "recaptchaDataSValue" => s_value, - }, - }.to_json).body) - - captcha_client.close - - raise response["error"].as_s if response["error"]? - task_id = response["taskId"].as_i - - loop do - sleep 10.seconds - - response = JSON.parse(captcha_client.post("/getTaskResult", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "taskId" => task_id, - }.to_json).body) - - if response["status"]?.try &.== "ready" - break - elsif response["errorId"]?.try &.as_i != 0 - raise response["errorDescription"].as_s - end - end - - inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s - headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || "" - response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs) - headers = HTTP::Headers{ - "Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0], - } - cookies = HTTP::Cookies.from_client_headers(headers) - - cookies.each { |cookie| CONFIG.cookies << cookie } - - # Persist cookies between runs - File.write("config/config.yml", CONFIG.to_yaml) - end - end - rescue ex - LOGGER.error("BypassCaptchaJob: #{ex.message}") - ensure - sleep 1.minute - Fiber.yield - end - end - end -end diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index b42ecd1a..12942906 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -191,6 +191,8 @@ module Invidious::Routes::API::V1::Misc json.object do json.field "ucid", sub_endpoint["browseId"].as_s if sub_endpoint["browseId"]? json.field "videoId", sub_endpoint["videoId"].as_s if sub_endpoint["videoId"]? + json.field "playlistId", sub_endpoint["playlistId"].as_s if sub_endpoint["playlistId"]? + json.field "startTimeSeconds", sub_endpoint["startTimeSeconds"].as_i if sub_endpoint["startTimeSeconds"]? json.field "params", params.try &.as_s json.field "pageType", pageType end diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index 9fb283c2..2922b060 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -32,11 +32,14 @@ module Invidious::Routes::API::V1::Search begin client = HTTP::Client.new("suggestqueries-clients6.youtube.com") - url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&xssi=t&gs_ri=youtube&ds=yt" + client.before_request { |r| add_yt_headers(r) } + + url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt" response = client.get(url).body + client.close - body = JSON.parse(response[5..-1]).as_a + body = JSON.parse(response[19..-2]).as_a suggestions = body[1].as_a[0..-2] JSON.build do |json| diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 1017ac9d..9281f4dd 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -363,4 +363,47 @@ module Invidious::Routes::API::V1::Videos end end end + + def self.clips(env) + locale = env.get("preferences").as(Preferences).locale + + env.response.content_type = "application/json" + + clip_id = env.params.url["id"] + region = env.params.query["region"]? + proxy = {"1", "true"}.any? &.== env.params.query["local"]? + + response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}") + return error_json(400, "Invalid clip ID") if response["error"]? + + video_id = response.dig?("endpoint", "watchEndpoint", "videoId").try &.as_s + return error_json(400, "Invalid clip ID") if video_id.nil? + + start_time = nil + end_time = nil + clip_title = nil + + if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s + start_time, end_time, clip_title = parse_clip_parameters(params) + end + + begin + video = get_video(video_id, region: region) + rescue ex : NotFoundException + return error_json(404, ex) + rescue ex + return error_json(500, ex) + end + + return JSON.build do |json| + json.object do + json.field "startTime", start_time + json.field "endTime", end_time + json.field "clipTitle", clip_title + json.field "video" do + Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy) + end + end + end + end end diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 40bca008..e20a7139 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -407,14 +407,23 @@ module Invidious::Routes::Feeds end spawn do - rss = XML.parse_html(body) - rss.xpath_nodes("//feed/entry").each do |entry| - id = entry.xpath_node("videoid").not_nil!.content - author = entry.xpath_node("author/name").not_nil!.content - published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) - updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) - - video = get_video(id, force_refresh: true) + # TODO: unify this with the other almost identical looking parts in this and channels.cr somehow? + namespaces = { + "yt" => "http://www.youtube.com/xml/schemas/2015", + "default" => "http://www.w3.org/2005/Atom", + } + rss = XML.parse(body) + rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry| + id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content + author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content + published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content) + updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content) + + begin + video = get_video(id, force_refresh: true) + rescue + next # skip this video since it raised an exception (e.g. it is a scheduled live event) + end if CONFIG.enable_user_notifications # Deliver notifications to `/api/v1/auth/notifications` diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 1d5aa914..ec18f3b8 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -42,7 +42,7 @@ module Invidious::Routes::VideoPlayback headers["Range"] = "bytes=#{range_for_head}" end - client = make_client(URI.parse(host), region) + client = make_client(URI.parse(host), region, force_resolve = true) response = HTTP::Client::Response.new(500) error = "" 5.times do @@ -57,7 +57,7 @@ module Invidious::Routes::VideoPlayback if new_host != host host = new_host client.close - client = make_client(URI.parse(new_host), region) + client = make_client(URI.parse(new_host), region, force_resolve = true) end url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" @@ -71,7 +71,7 @@ module Invidious::Routes::VideoPlayback fvip = "3" host = "https://r#{fvip}---#{mn}.googlevideo.com" - client = make_client(URI.parse(host), region) + client = make_client(URI.parse(host), region, force_resolve = true) rescue ex error = ex.message end @@ -196,7 +196,7 @@ module Invidious::Routes::VideoPlayback break else client.close - client = make_client(URI.parse(host), region) + client = make_client(URI.parse(host), region, force_resolve = true) end end diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 3d935f0a..aabe8dfc 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -275,6 +275,12 @@ module Invidious::Routes::Watch return error_template(400, "Invalid clip ID") if response["error"]? if video_id = response.dig?("endpoint", "watchEndpoint", "videoId") + if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s + start_time, end_time, _ = parse_clip_parameters(params) + env.params.query["start"] = start_time.to_s if start_time != nil + env.params.query["end"] = end_time.to_s if end_time != nil + end + return env.redirect "/watch?v=#{video_id}&#{env.params.query}" else return error_template(404, "The requested clip doesn't exist") diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index d6bd991c..ba05da19 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -235,6 +235,7 @@ module Invidious::Routing get "/api/v1/captions/:id", {{namespace}}::Videos, :captions get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations get "/api/v1/comments/:id", {{namespace}}::Videos, :comments + get "/api/v1/clips/:id", {{namespace}}::Videos, :clips # Feeds get "/api/v1/trending", {{namespace}}::Feeds, :trending diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr index c2b5c758..bf968734 100644 --- a/src/invidious/search/filters.cr +++ b/src/invidious/search/filters.cr @@ -300,9 +300,9 @@ module Invidious::Search object["9:varint"] = ((page - 1) * 20).to_i64 end - # If the object is empty, return an empty string, - # otherwise encode to protobuf then to base64 - return "" if object.empty? + # Prevent censoring of self harm topics + # See https://github.com/iv-org/invidious/issues/4398 + object["30:varint"] = 1.to_i64 return object .try { |i| Protodec::Any.cast_json(i) } diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 9fbd1374..a8f02056 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -227,8 +227,22 @@ struct Video info.dig?("streamingData", "hlsManifestUrl").try &.as_s end - def dash_manifest_url - info.dig?("streamingData", "dashManifestUrl").try &.as_s + def dash_manifest_url : String? + raw_dash_url = info.dig?("streamingData", "dashManifestUrl").try &.as_s + return nil if raw_dash_url.nil? + + # Use manifest v5 parameter to reduce file size + # See https://github.com/iv-org/invidious/issues/4186 + dash_url = URI.parse(raw_dash_url) + dash_query = dash_url.query || "" + + if dash_query.empty? + dash_url.path = "#{dash_url.path}/mpd_version/5" + else + dash_url.query = "#{dash_query}&mpd_version=5" + end + + return dash_url.to_s end def genre_url : String? diff --git a/src/invidious/videos/clip.cr b/src/invidious/videos/clip.cr new file mode 100644 index 00000000..29c57182 --- /dev/null +++ b/src/invidious/videos/clip.cr @@ -0,0 +1,22 @@ +require "json" + +# returns start_time, end_time and clip_title +def parse_clip_parameters(params) : {Float64?, Float64?, String?} + decoded_protobuf = params.try { |i| URI.decode_www_form(i) } + .try { |i| Base64.decode(i) } + .try { |i| IO::Memory.new(i) } + .try { |i| Protodec::Any.parse(i) } + + start_time = decoded_protobuf + .try(&.["50:0:embedded"]["2:1:varint"].as_i64) + .try { |i| i/1000 } + + end_time = decoded_protobuf + .try(&.["50:0:embedded"]["3:2:varint"].as_i64) + .try { |i| i/1000 } + + clip_title = decoded_protobuf + .try(&.["50:0:embedded"]["4:3:string"].as_s) + + return start_time, end_time, clip_title +end diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index 031b46da..6d227cfc 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -82,11 +82,19 @@ </div> <div class="video-card-row flexible"> - <div class="flex-left"><a href="/channel/<%= item.ucid %>"> - <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> - <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> - </p> - </a></div> + <div class="flex-left"> + <% if !item.ucid.to_s.empty? %> + <a href="/channel/<%= item.ucid %>"> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + </a> + <% else %> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + <% end %> + </div> </div> <% when Category %> <% else %> @@ -160,11 +168,19 @@ </div> <div class="video-card-row flexible"> - <div class="flex-left"><a href="/channel/<%= item.ucid %>"> - <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> - <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> - </p> - </a></div> + <div class="flex-left"> + <% if !item.ucid.to_s.empty? %> + <a href="/channel/<%= item.ucid %>"> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + </a> + <% else %> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + <% end %> + </div> <%= rendered "components/video-context-buttons" %> </div> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 07474896..1b020321 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -346,7 +346,7 @@ we're going to need to do it here in order to allow for translations. <h5 class="pure-g"> <div class="pure-u-14-24"> - <% if rv["ucid"]? %> + <% if !rv["ucid"].empty? %> <b style="width:100%"><a href="/channel/<%= rv["ucid"] %>"><%= rv["author"]? %><% if rv["author_verified"]? == "true" %> <i class="icon ion ion-md-checkmark-circle"></i><% end %></a></b> <% else %> <b style="width:100%"><%= rv["author"]? %><% if rv["author_verified"]? == "true" %> <i class="icon ion ion-md-checkmark-circle"></i><% end %></b> diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 03ff0ee4..81cfb272 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -1,7 +1,6 @@ def add_yt_headers(request) - if request.headers["User-Agent"] == "Crystal" - request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" - end + request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" + request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" @@ -27,7 +26,7 @@ struct YoutubeConnectionPool def client(region = nil, &block) if region - conn = make_client(url, region) + conn = make_client(url, region, force_resolve = true) response = yield conn else conn = pool.checkout @@ -60,9 +59,14 @@ struct YoutubeConnectionPool end end -def make_client(url : URI, region = nil) +def make_client(url : URI, region = nil, force_resolve : Bool = false) client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure) - client.family = CONFIG.force_resolve + + # Some services do not support IPv6. + if force_resolve + client.family = CONFIG.force_resolve + end + client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" client.read_timeout = 10.seconds client.connect_timeout = 10.seconds @@ -81,8 +85,8 @@ def make_client(url : URI, region = nil) return client end -def make_client(url : URI, region = nil, &block) - client = make_client(url, region) +def make_client(url : URI, region = nil, force_resolve : Bool = false, &block) + client = make_client(url, region, force_resolve) begin yield client ensure diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 56325cf7..0e72957e 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -822,9 +822,9 @@ module HelperExtractors end # Retrieves the ID required for querying the InnerTube browse endpoint. - # Raises when it's unable to do so + # Returns an empty string when it's unable to do so def self.get_browse_id(container) - return container.dig("navigationEndpoint", "browseEndpoint", "browseId").as_s + return container.dig?("navigationEndpoint", "browseEndpoint", "browseId").try &.as_s || "" end end |