diff options
60 files changed, 2226 insertions, 2014 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..fde6506e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,60 @@ +name: Invidious CI + +on: + push: + branches: + - "master" + - "api-only" + pull_request: + branches: "*" + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Install Crystal + uses: oprypin/install-crystal@v1.2.4 + + - name: Cache Shards + uses: actions/cache@v2 + with: + path: ./lib + key: shards-${{ hashFiles('shard.lock') }} + + - name: Install Shards + run: | + if ! shards check; then + shards install + fi + + - name: Run tests + run: crystal spec + + - name: Run lint + run: | + if ! crystal tool format --check; then + crystal tool format + git diff + exit 1 + fi + + - name: Build + run: crystal build --warnings all --error-on-warnings --error-trace src/invidious.cr + + build-docker: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Build Docker + run: docker-compose up -d + + - name: Test Docker + run: while curl -Isf http://localhost:3000; do sleep 1; done + diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 00000000..e452274b --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,22 @@ +# Documentation: https://github.com/marketplace/actions/close-stale-issues + +name: "Stale issue handler" +on: + workflow_dispatch: + schedule: + - cron: "0 */12 * * *" + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + days-before-stale: 365 + days-before-close: 30 + stale-issue-message: 'This issue has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely outdated. If you think this issue is still relevant and applicable, you just have to post a comment and it will be unmarked.' + stale-pr-message: 'This pull request has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely outdated. If you think this pull request is still relevant and applicable, you just have to post a comment and it will be unmarked.' + stale-issue-label: "stale" + stale-pr-label: "stale" + ascending: true @@ -6,4 +6,4 @@ /.vscode/ /invidious /sentry -shard.lock +/config/config.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f443e815..00000000 --- a/.travis.yml +++ /dev/null @@ -1,47 +0,0 @@ -dist: bionic - -# Work around broken Travis Crystal image -addons: - apt: - packages: - - gcc - - pkg-config - - git - - tzdata - - libpcre3-dev - - libevent-dev - - libyaml-dev - - libgmp-dev - - libssl-dev - - libxml2-dev - -jobs: - include: - - stage: build - # TODO: Shallowly clone again once the .git folder is no longer required for building - git: - depth: false - language: crystal - crystal: latest - before_install: - - crystal --version - - shards update - - shards install - install: - - crystal build --warnings all --error-on-warnings src/invidious.cr - script: - - crystal tool format --check - - crystal spec - - - stage: build_docker - # TODO: Shallowly clone again once the .git folder is no longer required for building - git: - depth: false - language: minimal - services: - - docker - install: - - docker-compose build - script: - - docker-compose up -d - - while curl -Isf http://localhost:3000; do sleep 1; done @@ -1,12 +1,12 @@ # Invidious -[](https://travis-ci.org/github/iv-org/invidious) [](https://hosted.weblate.org/engage/invidious/) +[](https://github.com/iv-org/invidious/actions) [](https://hosted.weblate.org/engage/invidious/) ## Invidious is an alternative front-end to YouTube ## Invidious instances: -[Public Invidious instances are listed here.](https://github.com/iv-org/invidious/wiki/Invidious-Instances) +[Public Invidious instances are listed here.](https://github.com/iv-org/documentation/blob/master/Invidious-Instances.md) ## Invidious features: @@ -24,7 +24,7 @@ - Set default player options (speed, quality, autoplay, loop) - Support for Reddit comments in place of YouTube comments - Import/Export subscriptions, watch history, preferences -- [Developer API](https://github.com/iv-org/invidious/wiki/API) +- [Developer API](https://github.com/iv-org/documentation/blob/master/API.md) - Does not use any of the official YouTube APIs - Does not require JavaScript to play videos - No need to create a Google account to save subscriptions @@ -33,6 +33,12 @@ - No CLA - [Multilingual](https://hosted.weblate.org/projects/invidious/#languages) (translated into many languages) +## Donate: + +Bitcoin (BTC): [bc1qfhe7rq3lqzuayzjxzyt9waz9ytrs09kla3tsgr](bitcoin:bc1qfhe7rq3lqzuayzjxzyt9waz9ytrs09kla3tsgr) + +Monero (XMR): [41nMCtek197boJtiUvGnTFYMatrLEpnpkQDmUECqx5Es2uX3sTKKWVhSL76suXsG3LXqkEJBrCZBgPTwJrDp1FrZJfycGPR](monero:41nMCtek197boJtiUvGnTFYMatrLEpnpkQDmUECqx5Es2uX3sTKKWVhSL76suXsG3LXqkEJBrCZBgPTwJrDp1FrZJfycGPR) + ## Screenshots: | Player | Preferences | Subscriptions | @@ -143,14 +149,14 @@ $ sudo systemctl enable --now invidious.service #### Logrotate: ```bash -$ sudo echo "/home/invidious/invidious/invidious.log { +$ echo "/home/invidious/invidious/invidious.log { rotate 4 weekly notifempty missingok compress minsize 1048576 -}" | tee /etc/logrotate.d/invidious.logrotate +}" | sudo tee /etc/logrotate.d/invidious.logrotate $ sudo chmod 0644 /etc/logrotate.d/invidious.logrotate ``` @@ -185,7 +191,7 @@ $ crystal build src/invidious.cr --release ## Post-install configuration: -Detailed configuration available in the [configuration guide](https://github.com/iv-org/invidious/wiki/Configuration). +Detailed configuration available in the [configuration guide](https://github.com/iv-org/documentation/blob/master/Configuration.md). If you use a reverse proxy, you **must** configure invidious to properly serve request through it: @@ -197,7 +203,7 @@ If you use a reverse proxy, you **must** configure invidious to properly serve r ## Update Invidious -Instructions are available in the [updating guide](https://github.com/iv-org/invidious/wiki/Updating). +Instructions are available in the [updating guide](https://github.com/iv-org/documentation/blob/master/Updating.md). ## Usage: @@ -228,11 +234,11 @@ $ ./sentry ## Documentation -[Documentation](https://github.com/iv-org/invidious/wiki) can be found in the wiki. +The [documentation](https://github.com/iv-org/documentation) can be found in its own repository. ## Extensions -[Extensions](https://github.com/iv-org/invidious/wiki/Extensions) can be found in the wiki, as well as documentation for integrating it into other projects. +[Extensions](https://github.com/iv-org/documentation/blob/master/Extensions.md) can be found in the wiki, as well as documentation for integrating it into other projects. ## Made with Invidious @@ -255,10 +261,6 @@ $ ./sentry - Log in with an account you have elsewhere, or register an account and start translating at [Hosted Weblate](https://hosted.weblate.org/engage/invidious/). -## Donate: - -Liberapay: https://liberapay.com/iv-org/ - ## Contact Feel free to join our [Matrix room](https://matrix.to/#/#invidious:matrix.org), or #invidious on freenode. Both platforms are bridged together. diff --git a/assets/css/darktheme.css b/assets/css/darktheme.css deleted file mode 100644 index 92da15b6..00000000 --- a/assets/css/darktheme.css +++ /dev/null @@ -1,37 +0,0 @@ -a:hover, -a:active { - color: rgb(0, 182, 240); -} - -a { - color: #a0a0a0; - text-decoration: none; -} - -body { - background-color: rgba(35, 35, 35, 1); - color: #f0f0f0; -} - -.pure-form legend { - color: #f0f0f0; -} - -.pure-menu-heading { - color: #f0f0f0; -} - -input, -select, -textarea { - color: rgba(35, 35, 35, 1); -} - -.pure-form input[type="file"] { - color: #f0f0f0; -} - -.navbar > .searchbar input { - background-color: inherit; - color: inherit; -} diff --git a/assets/css/default.css b/assets/css/default.css index b7a77be6..e403e606 100644 --- a/assets/css/default.css +++ b/assets/css/default.css @@ -20,7 +20,7 @@ body { height: auto; } -.channel-owner { +body a.channel-owner { background-color: #008bec; color: #fff; border-radius: 9px; @@ -101,15 +101,19 @@ div { padding-right: 10px; } +body a.pure-button { + color: rgba(0,0,0,.8); +} + button.pure-button-primary, -a.pure-button-primary, +body a.pure-button-primary, .channel-owner:hover { background-color: #a0a0a0; color: rgba(35, 35, 35, 1); } button.pure-button-primary:hover, -a.pure-button-primary:hover { +body a.pure-button-primary:hover { background-color: rgba(0, 182, 240, 1); color: #fff; } @@ -282,7 +286,7 @@ input[type="search"]::-webkit-search-cancel-button { text-align: center; } -.footer a { +body .footer a { color: inherit; text-decoration: underline; } @@ -485,3 +489,142 @@ video.video-js { margin-top: -0.81666em; margin-left: -1.5em; } + +/* + * Light theme + */ + +.light-theme a:hover, +.light-theme a:active { + color: #075A9E !important; +} + +.light-theme a.pure-button-primary:hover { + color: #fff !important; +} + +.light-theme a { + color: #335d7a; + text-decoration: none; +} + +/* All links that do not fit with the default color goes here */ +.light-theme a:not([data-id]) > .icon, +.light-theme .pure-u-lg-1-5 > .h-box > a[href^="/watch?"], +.light-theme .playlist-restricted > ol > li > a { + color: #303030; +} + +.light-theme .pure-menu-heading { + color: #565d64; +} + +@media (prefers-color-scheme: light) { + .no-theme a:hover, + .no-theme a:active { + color: #075A9E !important; + } + + .no-theme a.pure-button-primary:hover { + color: #fff !important; + } + + .no-theme a { + color: #335d7a; + text-decoration: none; + } + + /* All links that do not fit with the default color goes here */ + .no-theme a:not([data-id]) > .icon, + .no-theme .pure-u-lg-1-5 > .h-box > a[href^="/watch?"], + .no-theme .playlist-restricted > ol > li > a { + color: #303030; + } + + .light-theme .pure-menu-heading { + color: #565d64; + } +} + +/* + * Dark theme + */ + +.dark-theme a:hover, +.dark-theme a:active { + color: rgb(0, 182, 240); +} + +.dark-theme a { + color: #a0a0a0; + text-decoration: none; +} + +body.dark-theme { + background-color: rgba(35, 35, 35, 1); + color: #f0f0f0; +} + +.dark-theme .pure-form legend { + color: #f0f0f0; +} + +.dark-theme .pure-menu-heading { + color: #f0f0f0; +} + +.dark-theme input, +.dark-theme select, +.dark-theme textarea { + color: rgba(35, 35, 35, 1); +} + +.dark-theme .pure-form input[type="file"] { + color: #f0f0f0; +} + +.dark-theme .navbar > .searchbar input { + background-color: inherit; + color: inherit; +} + +@media (prefers-color-scheme: dark) { + .no-theme a:hover, + .no-theme a:active { + color: rgb(0, 182, 240); + } + + .no-theme a { + color: #a0a0a0; + text-decoration: none; + } + + body.no-theme { + background-color: rgba(35, 35, 35, 1); + color: #f0f0f0; + } + + .no-theme .pure-form legend { + color: #f0f0f0; + } + + .no-theme .pure-menu-heading { + color: #f0f0f0; + } + + .no-theme input, + .no-theme select, + .no-theme textarea { + color: rgba(35, 35, 35, 1); + } + + .no-theme .pure-form input[type="file"] { + color: #f0f0f0; + } + + .no-theme .navbar > .searchbar input { + background-color: inherit; + color: inherit; + } +} + diff --git a/assets/css/lighttheme.css b/assets/css/lighttheme.css deleted file mode 100644 index 73706bb7..00000000 --- a/assets/css/lighttheme.css +++ /dev/null @@ -1,16 +0,0 @@ -a:hover, -a:active { - color: #167ac6 !important; -} - -a { - color: #61809b; - text-decoration: none; -} - -/* All links that do not fit with the default color goes here */ -a:not([data-id]) > .icon, -.pure-u-lg-1-5 > .h-box > a[href^="/watch?"], -.playlist-restricted > ol > li > a { - color: #303030; -} diff --git a/assets/css/videojs-vtt-thumbnails-fix.css b/assets/css/videojs-vtt-thumbnails-fix.css deleted file mode 100644 index 8b62cf0c..00000000 --- a/assets/css/videojs-vtt-thumbnails-fix.css +++ /dev/null @@ -1,3 +0,0 @@ -.video-js .vjs-vtt-thumbnail-display { - max-width: 158px; -} diff --git a/assets/js/embed.js b/assets/js/embed.js index 99d2fc53..9d0be0ea 100644 --- a/assets/js/embed.js +++ b/assets/js/embed.js @@ -31,6 +31,11 @@ function get_playlist(plid, retries) { player.on('ended', function () { var url = new URL('https://example.com/embed/' + xhr.response.nextVideo); + url.searchParams.set('list', plid); + if (!plid.startsWith('RD')) { + url.searchParams.set('index', xhr.response.index); + } + if (video_data.params.autoplay || video_data.params.continue_autoplay) { url.searchParams.set('autoplay', '1'); } @@ -47,10 +52,6 @@ function get_playlist(plid, retries) { url.searchParams.set('local', video_data.params.local); } - url.searchParams.set('list', plid); - if (!plid.startsWith('RD')) { - url.searchParams.set('index', xhr.response.index); - } location.assign(url.pathname + url.search); }); } diff --git a/assets/js/player.js b/assets/js/player.js index f79fbbf3..fcba43d8 100644 --- a/assets/js/player.js +++ b/assets/js/player.js @@ -41,8 +41,16 @@ var shareOptions = { embedCode: "<iframe id='ivplayer' width='640' height='360' src='" + embed_url + "' style='border:none;'></iframe>" } +videojs.Hls.xhr.beforeRequest = function(options) { + if (options.uri.indexOf('local=true') === -1) { + options.uri = options.uri + '?local=true'; + } + return options; +}; + var player = videojs('player', options); + if (location.pathname.startsWith('/embed/')) { player.overlay({ overlays: [{ @@ -146,6 +154,35 @@ if (video_data.params.autoplay) { if (!video_data.params.listen && video_data.params.quality === 'dash') { player.httpSourceSelector(); + + if (video_data.params.quality_dash != "auto") { + player.ready(() => { + player.on("loadedmetadata", () => { + const qualityLevels = Array.from(player.qualityLevels()).sort((a, b) => a.height - b.height); + let targetQualityLevel; + switch (video_data.params.quality_dash) { + case "best": + targetQualityLevel = qualityLevels.length - 1; + break; + case "worst": + targetQualityLevel = 0; + break; + default: + const targetHeight = Number.parseInt(video_data.params.quality_dash, 10); + for (let i = 0; i < qualityLevels.length; i++) { + if (qualityLevels[i].height <= targetHeight) { + targetQualityLevel = i; + } else { + break; + } + } + } + for (let i = 0; i < qualityLevels.length; i++) { + qualityLevels[i].enabled = (i == targetQualityLevel); + } + }); + }); + } } player.vttThumbnails({ @@ -502,4 +539,6 @@ window.addEventListener('keydown', e => { }()); // Since videojs-share can sometimes be blocked, we defer it until last -player.share(shareOptions); +if (player.share) { + player.share(shareOptions); +} diff --git a/assets/js/themes.js b/assets/js/themes.js index c600073d..543b849e 100644 --- a/assets/js/themes.js +++ b/assets/js/themes.js @@ -2,7 +2,7 @@ var toggle_theme = document.getElementById('toggle_theme'); toggle_theme.href = 'javascript:void(0);'; toggle_theme.addEventListener('click', function () { - var dark_mode = document.getElementById('dark_theme').media === 'none'; + var dark_mode = document.body.classList.contains("light-theme"); var url = '/toggle_theme?redirect=false'; var xhr = new XMLHttpRequest(); @@ -22,7 +22,7 @@ window.addEventListener('storage', function (e) { } }); -window.addEventListener('load', function () { +window.addEventListener('DOMContentLoaded', function () { window.localStorage.setItem('dark_mode', document.getElementById('dark_mode_pref').textContent); // Update localStorage if dark mode preference changed on preferences page update_mode(window.localStorage.dark_mode); @@ -50,13 +50,18 @@ function scheme_switch (e) { } function set_mode (bool) { - document.getElementById('dark_theme').media = !bool ? 'none' : ''; - document.getElementById('light_theme').media = bool ? 'none' : ''; - if (bool) { + // dark toggle_theme.children[0].setAttribute('class', 'icon ion-ios-sunny'); + document.body.classList.remove('no-theme'); + document.body.classList.remove('light-theme'); + document.body.classList.add('dark-theme'); } else { + // light toggle_theme.children[0].setAttribute('class', 'icon ion-ios-moon'); + document.body.classList.remove('no-theme'); + document.body.classList.remove('dark-theme'); + document.body.classList.add('light-theme'); } } diff --git a/assets/js/watch.js b/assets/js/watch.js index e9ad2ddc..05530f3d 100644 --- a/assets/js/watch.js +++ b/assets/js/watch.js @@ -154,6 +154,11 @@ function get_playlist(plid, retries) { player.on('ended', function () { var url = new URL('https://example.com/watch?v=' + xhr.response.nextVideo); + url.searchParams.set('list', plid); + if (!plid.startsWith('RD')) { + url.searchParams.set('index', xhr.response.index); + } + if (video_data.params.autoplay || video_data.params.continue_autoplay) { url.searchParams.set('autoplay', '1'); } @@ -170,10 +175,6 @@ function get_playlist(plid, retries) { url.searchParams.set('local', video_data.params.local); } - url.searchParams.set('list', plid); - if (!plid.startsWith('RD')) { - url.searchParams.set('index', xhr.response.index); - } location.assign(url.pathname + url.search); }); } diff --git a/config/config.yml b/config/config.example.yml index e83a7515..e83a7515 100644 --- a/config/config.yml +++ b/config/config.example.yml diff --git a/docker/Dockerfile b/docker/Dockerfile index 96f844fe..d93f2868 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,7 +2,8 @@ FROM crystallang/crystal:0.35.1-alpine AS builder RUN apk add --no-cache curl sqlite-static WORKDIR /invidious COPY ./shard.yml ./shard.yml -RUN shards update && shards install && \ +COPY ./shard.lock ./shard.lock +RUN shards install && \ # TODO: Document build instructions # See https://github.com/omarroth/boringssl-alpine/blob/master/APKBUILD, # https://github.com/omarroth/lsquic-alpine/blob/master/APKBUILD, @@ -23,11 +24,13 @@ WORKDIR /invidious RUN addgroup -g 1000 -S invidious && \ adduser -u 1000 -S invidious -G invidious COPY ./assets/ ./assets/ -COPY --chown=invidious ./config/config.yml ./config/config.yml +COPY --chown=invidious ./config/config.* ./config/ +RUN mv -n config/config.example.yml config/config.yml RUN sed -i 's/host: \(127.0.0.1\|localhost\)/host: postgres/' config/config.yml COPY ./config/sql/ ./config/sql/ COPY ./locales/ ./locales/ COPY --from=builder /invidious/invidious . +EXPOSE 3000 USER invidious CMD [ "/invidious/invidious" ] diff --git a/screenshots/01_player.png b/screenshots/01_player.png Binary files differindex 63e6dbba..4ab8c4e5 100644 --- a/screenshots/01_player.png +++ b/screenshots/01_player.png diff --git a/screenshots/02_preferences.png b/screenshots/02_preferences.png Binary files differindex 1cd29add..bca77802 100644 --- a/screenshots/02_preferences.png +++ b/screenshots/02_preferences.png diff --git a/screenshots/03_subscriptions.png b/screenshots/03_subscriptions.png Binary files differnew file mode 100644 index 00000000..3da369eb --- /dev/null +++ b/screenshots/03_subscriptions.png diff --git a/screenshots/04_description.png b/screenshots/04_description.png Binary files differindex f8ec2564..949925f6 100644 --- a/screenshots/04_description.png +++ b/screenshots/04_description.png diff --git a/screenshots/05_preferences.png b/screenshots/05_preferences.png Binary files differindex dc6d4a42..ea7f142f 100644 --- a/screenshots/05_preferences.png +++ b/screenshots/05_preferences.png diff --git a/screenshots/06_subscriptions.png b/screenshots/06_subscriptions.png Binary files differindex 0da82f55..15a31f20 100644 --- a/screenshots/06_subscriptions.png +++ b/screenshots/06_subscriptions.png diff --git a/shard.lock b/shard.lock new file mode 100644 index 00000000..5dbac470 --- /dev/null +++ b/shard.lock @@ -0,0 +1,42 @@ +version: 2.0 +shards: + db: + git: https://github.com/crystal-lang/crystal-db.git + version: 0.10.0 + + exception_page: + git: https://github.com/crystal-loot/exception_page.git + version: 0.1.4 + + kemal: + git: https://github.com/kemalcr/kemal.git + version: 0.27.0 + + kilt: + git: https://github.com/jeromegn/kilt.git + version: 0.4.0 + + lsquic: + git: https://github.com/iv-org/lsquic.cr.git + version: 2.18.1-1 + + pg: + git: https://github.com/will/crystal-pg.git + version: 0.22.1 + + pool: + git: https://github.com/ysbaddaden/pool.git + version: 0.2.3 + + protodec: + git: https://github.com/omarroth/protodec.git + version: 0.1.3 + + radix: + git: https://github.com/luislavena/radix.git + version: 0.3.9 + + sqlite3: + git: https://github.com/crystal-lang/crystal-sqlite3.git + version: 0.17.0 + @@ -11,19 +11,19 @@ targets: dependencies: pg: github: will/crystal-pg - version: ~> 0.21.1 + version: ~> 0.22.1 sqlite3: github: crystal-lang/crystal-sqlite3 - version: ~> 0.16.0 + version: ~> 0.17.0 kemal: github: kemalcr/kemal - commit: dfe7dca08f4c9a9456d6132af5f6b59fcd6865e4 + version: ~> 0.27.0 pool: github: ysbaddaden/pool version: ~> 0.2.3 protodec: github: omarroth/protodec - version: ~> 0.1.2 + version: ~> 0.1.3 lsquic: github: iv-org/lsquic.cr version: ~> 2.18.1-1 diff --git a/spec/helpers_spec.cr b/spec/helpers_spec.cr index a8a3c6ce..d297759e 100644 --- a/spec/helpers_spec.cr +++ b/spec/helpers_spec.cr @@ -12,7 +12,7 @@ require "../src/invidious/search" require "../src/invidious/trending" require "../src/invidious/users" -CONFIG = Config.from_yaml(File.open("config/config.yml")) +CONFIG = Config.from_yaml(File.open("config/config.example.yml")) describe "Helper" do describe "#produce_channel_videos_url" do @@ -87,13 +87,13 @@ describe "Helper" do describe "#produce_comment_continuation" do it "correctly produces a continuation token for comments" do - produce_comment_continuation("_cE8xSu6swE", "ADSJ_i2qvJeFtL0htmS5_K5Ctj3eGFVBMWL9Wd42o3kmUL6_mAzdLp85-liQZL0mYr_16BhaggUqX652Sv9JqV6VXinShSP-ZT6rL4NolPBaPXVtJsO5_rA_qE3GubAuLFw9uzIIXU2-HnpXbdgPLWTFavfX206hqWmmpHwUOrmxQV_OX6tYkM3ux3rPAKCDrT8eWL7MU3bLiNcnbgkW8o0h8KYLL_8BPa8LcHbTv8pAoNkjerlX1x7K4pqxaXPoyz89qNlnh6rRx6AXgAzzoHH1dmcyQ8CIBeOHg-m4i8ZxdX4dP88XWrIFg-jJGhpGP8JUMDgZgavxVx225hUEYZMyrLGler5em4FgbG62YWC51moLDLeYEA").should eq("EiYSC19jRTh4U3U2c3dFwAEByAEB4AEBogINKP___________wFAABgGMowDCvYCQURTSl9pMnF2SmVGdEwwaHRtUzVfSzVDdGozZUdGVkJNV0w5V2Q0Mm8za21VTDZfbUF6ZExwODUtbGlRWkwwbVlyXzE2QmhhZ2dVcVg2NTJTdjlKcVY2VlhpblNoU1AtWlQ2ckw0Tm9sUEJhUFhWdEpzTzVfckFfcUUzR3ViQXVMRnc5dXpJSVhVMi1IbnBYYmRnUExXVEZhdmZYMjA2aHFXbW1wSHdVT3JteFFWX09YNnRZa00zdXgzclBBS0NEclQ4ZVdMN01VM2JMaU5jbmJna1c4bzBoOEtZTExfOEJQYThMY0hiVHY4cEFvTmtqZXJsWDF4N0s0cHF4YVhQb3l6ODlxTmxuaDZyUng2QVhnQXp6b0hIMWRtY3lROENJQmVPSGctbTRpOFp4ZFg0ZFA4OFhXcklGZy1qSkdocEdQOEpVTURnWmdhdnhWeDIyNWhVRVlaTXlyTEdsZXI1ZW00RmdiRzYyWVdDNTFtb0xETGVZRUEiDyILX2NFOHhTdTZzd0UwACgU") + produce_comment_continuation("_cE8xSu6swE", "ADSJ_i2qvJeFtL0htmS5_K5Ctj3eGFVBMWL9Wd42o3kmUL6_mAzdLp85-liQZL0mYr_16BhaggUqX652Sv9JqV6VXinShSP-ZT6rL4NolPBaPXVtJsO5_rA_qE3GubAuLFw9uzIIXU2-HnpXbdgPLWTFavfX206hqWmmpHwUOrmxQV_OX6tYkM3ux3rPAKCDrT8eWL7MU3bLiNcnbgkW8o0h8KYLL_8BPa8LcHbTv8pAoNkjerlX1x7K4pqxaXPoyz89qNlnh6rRx6AXgAzzoHH1dmcyQ8CIBeOHg-m4i8ZxdX4dP88XWrIFg-jJGhpGP8JUMDgZgavxVx225hUEYZMyrLGler5em4FgbG62YWC51moLDLeYEA").should eq("EkMSC19jRTh4U3U2c3dFyAEA4AEBogINKP___________wFAAMICHQgEGhdodHRwczovL3d3dy55b3V0dWJlLmNvbSIAGAYyjAMK9gJBRFNKX2kycXZKZUZ0TDBodG1TNV9LNUN0ajNlR0ZWQk1XTDlXZDQybzNrbVVMNl9tQXpkTHA4NS1saVFaTDBtWXJfMTZCaGFnZ1VxWDY1MlN2OUpxVjZWWGluU2hTUC1aVDZyTDROb2xQQmFQWFZ0SnNPNV9yQV9xRTNHdWJBdUxGdzl1eklJWFUyLUhucFhiZGdQTFdURmF2ZlgyMDZocVdtbXBId1VPcm14UVZfT1g2dFlrTTN1eDNyUEFLQ0RyVDhlV0w3TVUzYkxpTmNuYmdrVzhvMGg4S1lMTF84QlBhOExjSGJUdjhwQW9Oa2plcmxYMXg3SzRwcXhhWFBveXo4OXFObG5oNnJSeDZBWGdBenpvSEgxZG1jeVE4Q0lCZU9IZy1tNGk4WnhkWDRkUDg4WFdySUZnLWpKR2hwR1A4SlVNRGdaZ2F2eFZ4MjI1aFVFWVpNeXJMR2xlcjVlbTRGZ2JHNjJZV0M1MW1vTERMZVlFQSIPIgtfY0U4eFN1NnN3RTAAKBQ%3D") - produce_comment_continuation("_cE8xSu6swE", "ADSJ_i1yz21HI4xrtsYXVC-2_kfZ6kx1yjYQumXAAxqH3CAd7ZxKxfLdZS1__fqhCtOASRbbpSBGH_tH1J96Dxux-Qfjk-lUbupMqv08Q3aHzGu7p70VoUMHhI2-GoJpnbpmcOxkGzeIuenRS_ym2Y8fkDowhqLPFgsS0n4djnZ2UmC17F3Ch3N1S1UYf1ZVOc991qOC1iW9kJDzyvRQTWCPsJUPneSaAKW-Rr97pdesOkR4i8cNvHZRnQKe2HEfsvlJOb2C3lF1dJBfJeNfnQYeh5hv6_fZN7bt3-JL1Xk3Qc9NXNxmmbDpwAC_yFR8dthFfUJdyIO9Nu1D79MLYeR-H5HxqUJokkJiGIz4lTE_CXXbhAI").should eq("EiYSC19jRTh4U3U2c3dFwAEByAEB4AEBogINKP___________wFAABgGMokDCvMCQURTSl9pMXl6MjFISTR4cnRzWVhWQy0yX2tmWjZreDF5allRdW1YQUF4cUgzQ0FkN1p4S3hmTGRaUzFfX2ZxaEN0T0FTUmJicFNCR0hfdEgxSjk2RHh1eC1RZmprLWxVYnVwTXF2MDhRM2FIekd1N3A3MFZvVU1IaEkyLUdvSnBuYnBtY094a0d6ZUl1ZW5SU195bTJZOGZrRG93aHFMUEZnc1MwbjRkam5aMlVtQzE3RjNDaDNOMVMxVVlmMVpWT2M5OTFxT0MxaVc5a0pEenl2UlFUV0NQc0pVUG5lU2FBS1ctUnI5N3BkZXNPa1I0aThjTnZIWlJuUUtlMkhFZnN2bEpPYjJDM2xGMWRKQmZKZU5mblFZZWg1aHY2X2ZaTjdidDMtSkwxWGszUWM5TlhOeG1tYkRwd0FDX3lGUjhkdGhGZlVKZHlJTzlOdTFENzlNTFllUi1INUh4cVVKb2trSmlHSXo0bFRFX0NYWGJoQUkiDyILX2NFOHhTdTZzd0UwACgU") + produce_comment_continuation("_cE8xSu6swE", "ADSJ_i1yz21HI4xrtsYXVC-2_kfZ6kx1yjYQumXAAxqH3CAd7ZxKxfLdZS1__fqhCtOASRbbpSBGH_tH1J96Dxux-Qfjk-lUbupMqv08Q3aHzGu7p70VoUMHhI2-GoJpnbpmcOxkGzeIuenRS_ym2Y8fkDowhqLPFgsS0n4djnZ2UmC17F3Ch3N1S1UYf1ZVOc991qOC1iW9kJDzyvRQTWCPsJUPneSaAKW-Rr97pdesOkR4i8cNvHZRnQKe2HEfsvlJOb2C3lF1dJBfJeNfnQYeh5hv6_fZN7bt3-JL1Xk3Qc9NXNxmmbDpwAC_yFR8dthFfUJdyIO9Nu1D79MLYeR-H5HxqUJokkJiGIz4lTE_CXXbhAI").should eq("EkMSC19jRTh4U3U2c3dFyAEA4AEBogINKP___________wFAAMICHQgEGhdodHRwczovL3d3dy55b3V0dWJlLmNvbSIAGAYyiQMK8wJBRFNKX2kxeXoyMUhJNHhydHNZWFZDLTJfa2ZaNmt4MXlqWVF1bVhBQXhxSDNDQWQ3WnhLeGZMZFpTMV9fZnFoQ3RPQVNSYmJwU0JHSF90SDFKOTZEeHV4LVFmamstbFVidXBNcXYwOFEzYUh6R3U3cDcwVm9VTUhoSTItR29KcG5icG1jT3hrR3plSXVlblJTX3ltMlk4ZmtEb3docUxQRmdzUzBuNGRqbloyVW1DMTdGM0NoM04xUzFVWWYxWlZPYzk5MXFPQzFpVzlrSkR6eXZSUVRXQ1BzSlVQbmVTYUFLVy1Scjk3cGRlc09rUjRpOGNOdkhaUm5RS2UySEVmc3ZsSk9iMkMzbEYxZEpCZkplTmZuUVllaDVodjZfZlpON2J0My1KTDFYazNRYzlOWE54bW1iRHB3QUNfeUZSOGR0aEZmVUpkeUlPOU51MUQ3OU1MWWVSLUg1SHhxVUpva2tKaUdJejRsVEVfQ1hYYmhBSSIPIgtfY0U4eFN1NnN3RTAAKBQ%3D") - produce_comment_continuation("29-q7YnyUmY", "").should eq("EiYSCzI5LXE3WW55VW1ZwAEByAEB4AEBogINKP___________wFAABgGMhUKACIPIgsyOS1xN1lueVVtWTAAKBQ%3D") + produce_comment_continuation("29-q7YnyUmY", "").should eq("EkMSCzI5LXE3WW55VW1ZyAEA4AEBogINKP___________wFAAMICHQgEGhdodHRwczovL3d3dy55b3V0dWJlLmNvbSIAGAYyFQoAIg8iCzI5LXE3WW55VW1ZMAAoFA%3D%3D") - produce_comment_continuation("CvFH_6DNRCY", "").should eq("EiYSC0N2RkhfNkROUkNZwAEByAEB4AEBogINKP___________wFAABgGMhUKACIPIgtDdkZIXzZETlJDWTAAKBQ%3D") + produce_comment_continuation("CvFH_6DNRCY", "").should eq("EkMSC0N2RkhfNkROUkNZyAEA4AEBogINKP___________wFAAMICHQgEGhdodHRwczovL3d3dy55b3V0dWJlLmNvbSIAGAYyFQoAIg8iC0N2RkhfNkROUkNZMAAoFA%3D%3D") end end diff --git a/src/invidious.cr b/src/invidious.cr index 284b238c..8d4c2e58 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -64,7 +64,7 @@ HTTP_CHUNK_SIZE = 10485760 # ~10MB CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }} CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }} -CURRENT_VERSION = {{ "#{`git describe --tags --abbrev=0`.strip}" }} +CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }} # This is used to determine the `?v=` on the end of file URLs (for cache busting). We # only need to expire modified assets, so we can use this to find the last commit that changes @@ -104,10 +104,11 @@ LOCALES = { "zh-TW" => load_locale("zh-TW"), } -YT_POOL = QUICPool.new(YT_URL, capacity: CONFIG.pool_size, timeout: 0.1) +YT_POOL = QUICPool.new(YT_URL, capacity: CONFIG.pool_size, timeout: 2.0) config = CONFIG -logger = Invidious::LogHandler.new +output = STDOUT +loglvl = LogLevel::Debug Kemal.config.extra_options do |parser| parser.banner = "Usage: invidious [arguments]" @@ -127,11 +128,14 @@ Kemal.config.extra_options do |parser| exit end end - parser.on("-o OUTPUT", "--output=OUTPUT", "Redirect output (default: STDOUT)") do |output| - FileUtils.mkdir_p(File.dirname(output)) - logger = Invidious::LogHandler.new(File.open(output, mode: "a")) + parser.on("-o OUTPUT", "--output=OUTPUT", "Redirect output (default: STDOUT)") do |output_arg| + FileUtils.mkdir_p(File.dirname(output_arg)) + output = File.open(output_arg, mode: "a") end - parser.on("-v", "--version", "Print version") do |output| + parser.on("-l LEVEL", "--log-level=LEVEL", "Log level, one of #{LogLevel.values} (default: #{loglvl})") do |loglvl_arg| + loglvl = LogLevel.parse(loglvl_arg) + end + parser.on("-v", "--version", "Print version") do puts SOFTWARE.to_pretty_json exit end @@ -139,6 +143,8 @@ end Kemal::CLI.new ARGV +logger = Invidious::LogHandler.new(output, loglvl) + # Check table integrity if CONFIG.check_tables check_enum(PG_DB, logger, "privacy", PlaylistPrivacy) @@ -162,13 +168,16 @@ end Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB, logger, config) Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB, logger, config) Invidious::Jobs.register Invidious::Jobs::SubscribeToFeedsJob.new(PG_DB, logger, config, HMAC_KEY) -Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB) Invidious::Jobs.register Invidious::Jobs::UpdateDecryptFunctionJob.new if config.statistics_enabled Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, config, SOFTWARE) end +if config.popular_enabled + Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB) +end + if config.captcha_key Invidious::Jobs.register Invidious::Jobs::BypassCaptchaJob.new(logger, config) end @@ -197,6 +206,7 @@ before_all do |env| extra_media_csp = "" if CONFIG.disabled?("local") || !preferences.local extra_media_csp += " https://*.googlevideo.com:443" + extra_media_csp += " https://*.youtube.com:443" end # TODO: Remove style-src's 'unsafe-inline', requires to remove all inline styles (<style> [..] </style>, style=" [..] ") env.response.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' data:; connect-src 'self'; manifest-src 'self'; media-src 'self' blob:#{extra_media_csp}" @@ -249,7 +259,7 @@ before_all do |env| headers["Cookie"] = env.request.headers["Cookie"] begin - user, sid = get_user(sid, headers, PG_DB, false) + user, sid = get_user(sid, headers, PG_DB, logger, false) csrf_token = generate_response(sid, { ":authorize_token", ":playlist_ajax", @@ -299,1394 +309,30 @@ Invidious::Routing.get "/licenses", Invidious::Routes::Licenses Invidious::Routing.get "/watch", Invidious::Routes::Watch Invidious::Routing.get "/embed/", Invidious::Routes::Embed::Index Invidious::Routing.get "/embed/:id", Invidious::Routes::Embed::Show - -# Playlists - -get "/feed/playlists" do |env| - env.redirect "/view_all_playlists" -end - -get "/view_all_playlists" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - referer = get_referer(env) - - if !user - next env.redirect "/" - end - - user = user.as(User) - - items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) - items_created.map! do |item| - item.author = "" - item - end - - items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) - items_saved.map! do |item| - item.author = "" - item - end - - templated "view_all_playlists" -end - -get "/create_playlist" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect "/" - end - - user = user.as(User) - sid = sid.as(String) - csrf_token = generate_response(sid, {":create_playlist"}, HMAC_KEY, PG_DB) - - templated "create_playlist" -end - -post "/create_playlist" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect "/" - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) - rescue ex - error_message = ex.message - env.response.status_code = 400 - next templated "error" - end - - title = env.params.body["title"]?.try &.as(String) - if !title || title.empty? - error_message = "Title cannot be empty." - next templated "error" - end - - privacy = PlaylistPrivacy.parse?(env.params.body["privacy"]?.try &.as(String) || "") - if !privacy - error_message = "Invalid privacy setting." - next templated "error" - end - - if PG_DB.query_one("SELECT count(*) FROM playlists WHERE author = $1", user.email, as: Int64) >= 100 - error_message = "User cannot have more than 100 playlists." - next templated "error" - end - - playlist = create_playlist(PG_DB, title, privacy, user) - - env.redirect "/playlist?list=#{playlist.id}" -end - -get "/subscribe_playlist" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - referer = get_referer(env) - - if !user - next env.redirect "/" - end - - user = user.as(User) - - playlist_id = env.params.query["list"] - playlist = get_playlist(PG_DB, playlist_id, locale) - subscribe_playlist(PG_DB, user, playlist) - - env.redirect "/playlist?list=#{playlist.id}" -end - -get "/delete_playlist" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect "/" - end - - user = user.as(User) - sid = sid.as(String) - - plid = env.params.query["list"]? - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) - if !playlist || playlist.author != user.email - next env.redirect referer - end - - csrf_token = generate_response(sid, {":delete_playlist"}, HMAC_KEY, PG_DB) - - templated "delete_playlist" -end - -post "/delete_playlist" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect "/" - end - - plid = env.params.query["list"]? - if !plid - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) - rescue ex - error_message = ex.message - env.response.status_code = 400 - next templated "error" - end - - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) - if !playlist || playlist.author != user.email - next env.redirect referer - end - - PG_DB.exec("DELETE FROM playlist_videos * WHERE plid = $1", plid) - PG_DB.exec("DELETE FROM playlists * WHERE id = $1", plid) - - env.redirect "/view_all_playlists" -end - -get "/edit_playlist" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect "/" - end - - user = user.as(User) - sid = sid.as(String) - - plid = env.params.query["list"]? - if !plid || !plid.starts_with?("IV") - next env.redirect referer - end - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - begin - playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) - if !playlist || playlist.author != user.email - next env.redirect referer - end - rescue ex - next env.redirect referer - end - - begin - videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale) - rescue ex - videos = [] of PlaylistVideo - end - - csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY, PG_DB) - - templated "edit_playlist" -end - -post "/edit_playlist" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect "/" - end - - plid = env.params.query["list"]? - if !plid - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) - rescue ex - error_message = ex.message - env.response.status_code = 400 - next templated "error" - end - - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) - if !playlist || playlist.author != user.email - next env.redirect referer - end - - title = env.params.body["title"]?.try &.delete("<>") || "" - privacy = PlaylistPrivacy.parse(env.params.body["privacy"]? || "Public") - description = env.params.body["description"]?.try &.delete("\r") || "" - - if title != playlist.title || - privacy != playlist.privacy || - description != playlist.description - updated = Time.utc - else - updated = playlist.updated - end - - PG_DB.exec("UPDATE playlists SET title = $1, privacy = $2, description = $3, updated = $4 WHERE id = $5", title, privacy, description, updated, plid) - - env.redirect "/playlist?list=#{plid}" -end - -get "/add_playlist_items" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect "/" - end - - user = user.as(User) - sid = sid.as(String) - - plid = env.params.query["list"]? - if !plid || !plid.starts_with?("IV") - next env.redirect referer - end - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - begin - playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) - if !playlist || playlist.author != user.email - next env.redirect referer - end - rescue ex - next env.redirect referer - end - - query = env.params.query["q"]? - if query - begin - search_query, count, items = process_search_query(query, page, user, region: nil) - videos = items.select { |item| item.is_a? SearchVideo }.map { |item| item.as(SearchVideo) } - rescue ex - videos = [] of SearchVideo - count = 0 - end - else - videos = [] of SearchVideo - count = 0 - end - - env.set "add_playlist_items", plid - templated "add_playlist_items" -end - -post "/playlist_ajax" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env, "/") - - redirect = env.params.query["redirect"]? - redirect ||= "true" - redirect = redirect == "true" - - if !user - if redirect - next env.redirect referer - else - error_message = {"error" => "No such user"}.to_json - env.response.status_code = 403 - next error_message - end - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) - rescue ex - if redirect - error_message = ex.message - env.response.status_code = 400 - next templated "error" - else - error_message = {"error" => ex.message}.to_json - env.response.status_code = 400 - next error_message - end - end - - if env.params.query["action_create_playlist"]? - action = "action_create_playlist" - elsif env.params.query["action_delete_playlist"]? - action = "action_delete_playlist" - elsif env.params.query["action_edit_playlist"]? - action = "action_edit_playlist" - elsif env.params.query["action_add_video"]? - action = "action_add_video" - video_id = env.params.query["video_id"] - elsif env.params.query["action_remove_video"]? - action = "action_remove_video" - elsif env.params.query["action_move_video_before"]? - action = "action_move_video_before" - else - next env.redirect referer - end - - begin - playlist_id = env.params.query["playlist_id"] - playlist = get_playlist(PG_DB, playlist_id, locale).as(InvidiousPlaylist) - raise "Invalid user" if playlist.author != user.email - rescue ex - if redirect - error_message = ex.message - env.response.status_code = 400 - next templated "error" - else - error_message = {"error" => ex.message}.to_json - env.response.status_code = 400 - next error_message - end - end - - if !user.password - # TODO: Playlist stub, sync with YouTube for Google accounts - # playlist_ajax(playlist_id, action, env.request.headers) - end - email = user.email - - case action - when "action_edit_playlist" - # TODO: Playlist stub - when "action_add_video" - if playlist.index.size >= 500 - env.response.status_code = 400 - if redirect - error_message = "Playlist cannot have more than 500 videos" - next templated "error" - else - error_message = {"error" => "Playlist cannot have more than 500 videos"}.to_json - next error_message - end - end - - video_id = env.params.query["video_id"] - - begin - video = get_video(video_id, PG_DB) - rescue ex - env.response.status_code = 500 - if redirect - error_message = ex.message - next templated "error" - else - error_message = {"error" => ex.message}.to_json - next error_message - end - end - - playlist_video = PlaylistVideo.new({ - title: video.title, - id: video.id, - author: video.author, - ucid: video.ucid, - length_seconds: video.length_seconds, - published: video.published, - plid: playlist_id, - live_now: video.live_now, - index: Random::Secure.rand(0_i64..Int64::MAX), - }) - - video_array = playlist_video.to_a - args = arg_array(video_array) - - PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array) - PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, playlist_id) - when "action_remove_video" - index = env.params.query["set_video_id"] - PG_DB.exec("DELETE FROM playlist_videos * WHERE index = $1", index) - PG_DB.exec("UPDATE playlists SET index = array_remove(index, $1), video_count = cardinality(index) - 1, updated = $2 WHERE id = $3", index, Time.utc, playlist_id) - when "action_move_video_before" - # TODO: Playlist stub - else - error_message = {"error" => "Unsupported action #{action}"}.to_json - env.response.status_code = 400 - next error_message - end - - if redirect - env.redirect referer - else - env.response.content_type = "application/json" - "{}" - end -end - -get "/playlist" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get?("user").try &.as(User) - referer = get_referer(env) - - plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") - if !plid - next env.redirect "/" - end - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - if plid.starts_with? "RD" - next env.redirect "/mix?list=#{plid}" - end - - begin - playlist = get_playlist(PG_DB, plid, locale) - rescue ex - error_message = ex.message - env.response.status_code = 500 - next templated "error" - end - - if playlist.privacy == PlaylistPrivacy::Private && playlist.author != user.try &.email - error_message = "This playlist is private." - env.response.status_code = 403 - next templated "error" - end - - begin - videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale) - rescue ex - videos = [] of PlaylistVideo - end - - if playlist.author == user.try &.email - env.set "remove_playlist_items", plid - end - - templated "playlist" -end - -get "/mix" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - rdid = env.params.query["list"]? - if !rdid - next env.redirect "/" - end - - continuation = env.params.query["continuation"]? - continuation ||= rdid.lchop("RD") - - begin - mix = fetch_mix(rdid, continuation, locale: locale) - rescue ex - error_message = ex.message - env.response.status_code = 500 - next templated "error" - end - - templated "mix" -end - -# Search - -get "/opensearch.xml" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - env.response.content_type = "application/opensearchdescription+xml" - - XML.build(indent: " ", encoding: "UTF-8") do |xml| - xml.element("OpenSearchDescription", xmlns: "http://a9.com/-/spec/opensearch/1.1/") do - xml.element("ShortName") { xml.text "Invidious" } - xml.element("LongName") { xml.text "Invidious Search" } - xml.element("Description") { xml.text "Search for videos, channels, and playlists on Invidious" } - xml.element("InputEncoding") { xml.text "UTF-8" } - xml.element("Image", width: 48, height: 48, type: "image/x-icon") { xml.text "#{HOST_URL}/favicon.ico" } - xml.element("Url", type: "text/html", method: "get", template: "#{HOST_URL}/search?q={searchTerms}") - end - end -end - -get "/results" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - query = env.params.query["search_query"]? - query ||= env.params.query["q"]? - query ||= "" - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - if query - env.redirect "/search?q=#{URI.encode_www_form(query)}&page=#{page}" - else - env.redirect "/" - end -end - -get "/search" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - region = env.params.query["region"]? - - query = env.params.query["search_query"]? - query ||= env.params.query["q"]? - query ||= "" - - if query.empty? - next env.redirect "/" - end - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - user = env.get? "user" - - begin - search_query, count, videos = process_search_query(query, page, user, region: nil) - rescue ex - error_message = ex.message - env.response.status_code = 500 - next templated "error" - end - - env.set "search", query - templated "search" -end +Invidious::Routing.get "/view_all_playlists", Invidious::Routes::Playlists, :index +Invidious::Routing.get "/create_playlist", Invidious::Routes::Playlists, :new +Invidious::Routing.post "/create_playlist", Invidious::Routes::Playlists, :create +Invidious::Routing.get "/subscribe_playlist", Invidious::Routes::Playlists, :subscribe +Invidious::Routing.get "/delete_playlist", Invidious::Routes::Playlists, :delete_page +Invidious::Routing.post "/delete_playlist", Invidious::Routes::Playlists, :delete +Invidious::Routing.get "/edit_playlist", Invidious::Routes::Playlists, :edit +Invidious::Routing.post "/edit_playlist", Invidious::Routes::Playlists, :update +Invidious::Routing.get "/add_playlist_items", Invidious::Routes::Playlists, :add_playlist_items_page +Invidious::Routing.post "/playlist_ajax", Invidious::Routes::Playlists, :playlist_ajax +Invidious::Routing.get "/playlist", Invidious::Routes::Playlists, :show +Invidious::Routing.get "/mix", Invidious::Routes::Playlists, :mix +Invidious::Routing.get "/opensearch.xml", Invidious::Routes::Search, :opensearch +Invidious::Routing.get "/results", Invidious::Routes::Search, :results +Invidious::Routing.get "/search", Invidious::Routes::Search, :search +Invidious::Routing.get "/login", Invidious::Routes::Login, :login_page +Invidious::Routing.post "/login", Invidious::Routes::Login, :login +Invidious::Routing.post "/signout", Invidious::Routes::Login, :signout +Invidious::Routing.get "/preferences", Invidious::Routes::UserPreferences, :show +Invidious::Routing.post "/preferences", Invidious::Routes::UserPreferences, :update +Invidious::Routing.get "/toggle_theme", Invidious::Routes::UserPreferences, :toggle_theme # Users -get "/login" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - if user - next env.redirect "/feed/subscriptions" - end - - if !config.login_enabled - error_message = "Login has been disabled by administrator." - env.response.status_code = 400 - next templated "error" - end - - referer = get_referer(env, "/feed/subscriptions") - - email = nil - password = nil - captcha = nil - - account_type = env.params.query["type"]? - account_type ||= "invidious" - - captcha_type = env.params.query["captcha"]? - captcha_type ||= "image" - - tfa = env.params.query["tfa"]? - prompt = nil - - templated "login" -end - -post "/login" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - referer = get_referer(env, "/feed/subscriptions") - - if !config.login_enabled - error_message = "Login has been disabled by administrator." - env.response.status_code = 403 - next templated "error" - end - - # https://stackoverflow.com/a/574698 - email = env.params.body["email"]?.try &.downcase.byte_slice(0, 254) - password = env.params.body["password"]? - - account_type = env.params.query["type"]? - account_type ||= "invidious" - - case account_type - when "google" - tfa_code = env.params.body["tfa"]?.try &.lchop("G-") - traceback = IO::Memory.new - - # See https://github.com/ytdl-org/youtube-dl/blob/2019.04.07/youtube_dl/extractor/youtube.py#L82 - begin - client = QUIC::Client.new(LOGIN_URL) - headers = HTTP::Headers.new - - login_page = client.get("/ServiceLogin") - headers = login_page.cookies.add_request_headers(headers) - - lookup_req = { - email, nil, [] of String, nil, "US", nil, nil, 2, false, true, - {nil, nil, - {2, 1, nil, 1, - "https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn", - nil, [] of String, 4}, - 1, - {nil, nil, [] of String}, - nil, nil, nil, true, - }, - email, - }.to_json - - traceback << "Getting lookup..." - - headers["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8" - headers["Google-Accounts-XSRF"] = "1" - - response = client.post("/_/signin/sl/lookup", headers, login_req(lookup_req)) - lookup_results = JSON.parse(response.body[5..-1]) - - traceback << "done, returned #{response.status_code}.<br/>" - - user_hash = lookup_results[0][2] - - if token = env.params.body["token"]? - answer = env.params.body["answer"]? - captcha = {token, answer} - else - captcha = nil - end - - challenge_req = { - user_hash, nil, 1, nil, - {1, nil, nil, nil, - {password, captcha, true}, - }, - {nil, nil, - {2, 1, nil, 1, - "https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn", - nil, [] of String, 4}, - 1, - {nil, nil, [] of String}, - nil, nil, nil, true, - }, - }.to_json - - traceback << "Getting challenge..." - - response = client.post("/_/signin/sl/challenge", headers, login_req(challenge_req)) - headers = response.cookies.add_request_headers(headers) - challenge_results = JSON.parse(response.body[5..-1]) - - traceback << "done, returned #{response.status_code}.<br/>" - - headers["Cookie"] = URI.decode_www_form(headers["Cookie"]) - - if challenge_results[0][3]?.try &.== 7 - error_message = translate(locale, "Account has temporarily been disabled") - env.response.status_code = 423 - next templated "error" - end - - if token = challenge_results[0][-1]?.try &.[-1]?.try &.as_h?.try &.["5001"]?.try &.[-1].as_a?.try &.[-1].as_s - account_type = "google" - captcha_type = "image" - prompt = nil - tfa = tfa_code - captcha = {tokens: [token], question: ""} - - next templated "login" - end - - if challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED" - error_message = translate(locale, "Incorrect password") - env.response.status_code = 401 - next templated "error" - end - - prompt_type = challenge_results[0][-1]?.try &.[0].as_a?.try &.[0][2]? - if {"TWO_STEP_VERIFICATION", "LOGIN_CHALLENGE"}.includes? prompt_type - traceback << "Handling prompt #{prompt_type}.<br/>" - case prompt_type - when "TWO_STEP_VERIFICATION" - prompt_type = 2 - else # "LOGIN_CHALLENGE" - prompt_type = 4 - end - - # Prefer Authenticator app and SMS over unsupported protocols - if !{6, 9, 12, 15}.includes?(challenge_results[0][-1][0][0][8].as_i) && prompt_type == 2 - tfa = challenge_results[0][-1][0].as_a.select { |auth_type| {6, 9, 12, 15}.includes? auth_type[8] }[0] - - traceback << "Selecting challenge #{tfa[8]}..." - select_challenge = {prompt_type, nil, nil, nil, {tfa[8]}}.to_json - - tl = challenge_results[1][2] - - tfa = client.post("/_/signin/selectchallenge?TL=#{tl}", headers, login_req(select_challenge)).body - tfa = tfa[5..-1] - tfa = JSON.parse(tfa)[0][-1] - - traceback << "done.<br/>" - else - traceback << "Using challenge #{challenge_results[0][-1][0][0][8]}.<br/>" - tfa = challenge_results[0][-1][0][0] - end - - if tfa[5] == "QUOTA_EXCEEDED" - error_message = translate(locale, "Quota exceeded, try again in a few hours") - env.response.status_code = 423 - next templated "error" - end - - if !tfa_code - account_type = "google" - captcha_type = "image" - - case tfa[8] - when 6, 9 - prompt = "Google verification code" - when 12 - prompt = "Login verification, recovery email: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}" - when 15 - prompt = "Login verification, security question: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}" - else - prompt = "Google verification code" - end - - tfa = nil - captcha = nil - next templated "login" - end - - tl = challenge_results[1][2] - - request_type = tfa[8] - case request_type - when 6 # Authenticator app - tfa_req = { - user_hash, nil, 2, nil, - {6, nil, nil, nil, nil, - {tfa_code, false}, - }, - }.to_json - when 9 # Voice or text message - tfa_req = { - user_hash, nil, 2, nil, - {9, nil, nil, nil, nil, nil, nil, nil, - {nil, tfa_code, false, 2}, - }, - }.to_json - when 12 # Recovery email - tfa_req = { - user_hash, nil, 4, nil, - {12, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - {tfa_code}, - }, - }.to_json - when 15 # Security question - tfa_req = { - user_hash, nil, 5, nil, - {15, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - {tfa_code}, - }, - }.to_json - else - error_message = translate(locale, "Unable to log in, make sure two-factor authentication (Authenticator or SMS) is turned on.") - env.response.status_code = 500 - next templated "error" - end - - traceback << "Submitting challenge..." - - response = client.post("/_/signin/challenge?hl=en&TL=#{tl}", headers, login_req(tfa_req)) - headers = response.cookies.add_request_headers(headers) - challenge_results = JSON.parse(response.body[5..-1]) - - if (challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED") || - (challenge_results[0][-1]?.try &.[5] == "INVALID_INPUT") - error_message = translate(locale, "Invalid TFA code") - env.response.status_code = 401 - next templated "error" - end - - traceback << "done.<br/>" - end - - traceback << "Logging in..." - - location = URI.parse(challenge_results[0][-1][2].to_s) - cookies = HTTP::Cookies.from_headers(headers) - - headers.delete("Content-Type") - headers.delete("Google-Accounts-XSRF") - - loop do - if !location || location.path == "/ManageAccount" - break - end - - # Occasionally there will be a second page after login confirming - # the user's phone number ("/b/0/SmsAuthInterstitial"), which we currently don't handle. - - if location.path.starts_with? "/b/0/SmsAuthInterstitial" - traceback << "Unhandled dialog /b/0/SmsAuthInterstitial." - end - - login = client.get(location.full_path, headers) - - headers = login.cookies.add_request_headers(headers) - location = login.headers["Location"]?.try { |u| URI.parse(u) } - end - - cookies = HTTP::Cookies.from_headers(headers) - sid = cookies["SID"]?.try &.value - if !sid - raise "Couldn't get SID." - end - - user, sid = get_user(sid, headers, PG_DB) - - # We are now logged in - traceback << "done.<br/>" - - host = URI.parse(env.request.headers["Host"]).host - - if Kemal.config.ssl || config.https_only - secure = true - else - secure = false - end - - cookies.each do |cookie| - if Kemal.config.ssl || config.https_only - cookie.secure = secure - else - cookie.secure = secure - end - - if cookie.extension - cookie.extension = cookie.extension.not_nil!.gsub(".youtube.com", host) - cookie.extension = cookie.extension.not_nil!.gsub("Secure; ", "") - end - env.response.cookies << cookie - end - - if env.request.cookies["PREFS"]? - preferences = env.get("preferences").as(Preferences) - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) - - cookie = env.request.cookies["PREFS"] - cookie.expires = Time.utc(1990, 1, 1) - env.response.cookies << cookie - end - - env.redirect referer - rescue ex - traceback.rewind - # error_message = translate(locale, "Login failed. This may be because two-factor authentication is not turned on for your account.") - error_message = %(#{ex.message}<br/>Traceback:<br/><div style="padding-left:2em" id="traceback">#{traceback.gets_to_end}</div>) - env.response.status_code = 500 - next templated "error" - end - when "invidious" - if !email - error_message = translate(locale, "User ID is a required field") - env.response.status_code = 401 - next templated "error" - end - - if !password - error_message = translate(locale, "Password is a required field") - env.response.status_code = 401 - next templated "error" - end - - user = PG_DB.query_one?("SELECT * FROM users WHERE email = $1", email, as: User) - - if user - if !user.password - error_message = translate(locale, "Please sign in using 'Log in with Google'") - env.response.status_code = 400 - next templated "error" - end - - if Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55)) - sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) - - if Kemal.config.ssl || config.https_only - secure = true - else - secure = false - end - - if config.domain - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{config.domain}", value: sid, expires: Time.utc + 2.years, - secure: secure, http_only: true) - else - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.utc + 2.years, - secure: secure, http_only: true) - end - else - error_message = translate(locale, "Wrong username or password") - env.response.status_code = 401 - next templated "error" - end - - # Since this user has already registered, we don't want to overwrite their preferences - if env.request.cookies["PREFS"]? - cookie = env.request.cookies["PREFS"] - cookie.expires = Time.utc(1990, 1, 1) - env.response.cookies << cookie - end - else - if !config.registration_enabled - error_message = "Registration has been disabled by administrator." - env.response.status_code = 400 - next templated "error" - end - - if password.empty? - error_message = translate(locale, "Password cannot be empty") - env.response.status_code = 401 - next templated "error" - end - - # See https://security.stackexchange.com/a/39851 - if password.bytesize > 55 - error_message = translate(locale, "Password should not be longer than 55 characters") - env.response.status_code = 400 - next templated "error" - end - - password = password.byte_slice(0, 55) - - if config.captcha_enabled - captcha_type = env.params.body["captcha_type"]? - answer = env.params.body["answer"]? - change_type = env.params.body["change_type"]? - - if !captcha_type || change_type - if change_type - captcha_type = change_type - end - captcha_type ||= "image" - - account_type = "invidious" - tfa = false - prompt = "" - - if captcha_type == "image" - captcha = generate_captcha(HMAC_KEY, PG_DB) - else - captcha = generate_text_captcha(HMAC_KEY, PG_DB) - end - - next templated "login" - end - - tokens = env.params.body.select { |k, v| k.match(/^token\[\d+\]$/) }.map { |k, v| v } - - answer ||= "" - captcha_type ||= "image" - - case captcha_type - when "image" - answer = answer.lstrip('0') - answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer) - - begin - validate_request(tokens[0], answer, env.request, HMAC_KEY, PG_DB, locale) - rescue ex - error_message = ex.message - env.response.status_code = 400 - next templated "error" - end - else # "text" - answer = Digest::MD5.hexdigest(answer.downcase.strip) - - found_valid_captcha = false - - error_message = translate(locale, "Erroneous CAPTCHA") - tokens.each_with_index do |token, i| - begin - validate_request(token, answer, env.request, HMAC_KEY, PG_DB, locale) - found_valid_captcha = true - rescue ex - error_message = ex.message - end - end - - if !found_valid_captcha - env.response.status_code = 500 - next templated "error" - end - end - end - - sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - user, sid = create_user(sid, email, password) - user_array = user.to_a - user_array[4] = user_array[4].to_json # User preferences - - args = arg_array(user_array) - - PG_DB.exec("INSERT INTO users VALUES (#{args})", args: user_array) - PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) - - view_name = "subscriptions_#{sha256(user.email)}" - PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") - - if Kemal.config.ssl || config.https_only - secure = true - else - secure = false - end - - if config.domain - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{config.domain}", value: sid, expires: Time.utc + 2.years, - secure: secure, http_only: true) - else - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.utc + 2.years, - secure: secure, http_only: true) - end - - if env.request.cookies["PREFS"]? - preferences = env.get("preferences").as(Preferences) - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) - - cookie = env.request.cookies["PREFS"] - cookie.expires = Time.utc(1990, 1, 1) - env.response.cookies << cookie - end - end - - env.redirect referer - else - env.redirect referer - end -end - -post "/signout" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) - rescue ex - error_message = ex.message - env.response.status_code = 400 - next templated "error" - end - - PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", sid) - - env.request.cookies.each do |cookie| - cookie.expires = Time.utc(1990, 1, 1) - env.response.cookies << cookie - end - - env.redirect referer -end - -get "/preferences" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - referer = get_referer(env) - - preferences = env.get("preferences").as(Preferences) - - templated "preferences" -end - -post "/preferences" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - referer = get_referer(env) - - video_loop = env.params.body["video_loop"]?.try &.as(String) - video_loop ||= "off" - video_loop = video_loop == "on" - - annotations = env.params.body["annotations"]?.try &.as(String) - annotations ||= "off" - annotations = annotations == "on" - - annotations_subscribed = env.params.body["annotations_subscribed"]?.try &.as(String) - annotations_subscribed ||= "off" - annotations_subscribed = annotations_subscribed == "on" - - autoplay = env.params.body["autoplay"]?.try &.as(String) - autoplay ||= "off" - autoplay = autoplay == "on" - - continue = env.params.body["continue"]?.try &.as(String) - continue ||= "off" - continue = continue == "on" - - continue_autoplay = env.params.body["continue_autoplay"]?.try &.as(String) - continue_autoplay ||= "off" - continue_autoplay = continue_autoplay == "on" - - listen = env.params.body["listen"]?.try &.as(String) - listen ||= "off" - listen = listen == "on" - - local = env.params.body["local"]?.try &.as(String) - local ||= "off" - local = local == "on" - - speed = env.params.body["speed"]?.try &.as(String).to_f32? - speed ||= CONFIG.default_user_preferences.speed - - player_style = env.params.body["player_style"]?.try &.as(String) - player_style ||= CONFIG.default_user_preferences.player_style - - quality = env.params.body["quality"]?.try &.as(String) - quality ||= CONFIG.default_user_preferences.quality - - volume = env.params.body["volume"]?.try &.as(String).to_i? - volume ||= CONFIG.default_user_preferences.volume - - comments = [] of String - 2.times do |i| - comments << (env.params.body["comments[#{i}]"]?.try &.as(String) || CONFIG.default_user_preferences.comments[i]) - end - - captions = [] of String - 3.times do |i| - captions << (env.params.body["captions[#{i}]"]?.try &.as(String) || CONFIG.default_user_preferences.captions[i]) - end - - related_videos = env.params.body["related_videos"]?.try &.as(String) - related_videos ||= "off" - related_videos = related_videos == "on" - - default_home = env.params.body["default_home"]?.try &.as(String) || CONFIG.default_user_preferences.default_home - - feed_menu = [] of String - 5.times do |index| - option = env.params.body["feed_menu[#{index}]"]?.try &.as(String) || "" - if !option.empty? - feed_menu << option - end - end - - locale = env.params.body["locale"]?.try &.as(String) - locale ||= CONFIG.default_user_preferences.locale - - dark_mode = env.params.body["dark_mode"]?.try &.as(String) - dark_mode ||= CONFIG.default_user_preferences.dark_mode - - thin_mode = env.params.body["thin_mode"]?.try &.as(String) - thin_mode ||= "off" - thin_mode = thin_mode == "on" - - max_results = env.params.body["max_results"]?.try &.as(String).to_i? - max_results ||= CONFIG.default_user_preferences.max_results - - sort = env.params.body["sort"]?.try &.as(String) - sort ||= CONFIG.default_user_preferences.sort - - latest_only = env.params.body["latest_only"]?.try &.as(String) - latest_only ||= "off" - latest_only = latest_only == "on" - - unseen_only = env.params.body["unseen_only"]?.try &.as(String) - unseen_only ||= "off" - unseen_only = unseen_only == "on" - - notifications_only = env.params.body["notifications_only"]?.try &.as(String) - notifications_only ||= "off" - notifications_only = notifications_only == "on" - - # Convert to JSON and back again to take advantage of converters used for compatability - preferences = Preferences.from_json({ - annotations: annotations, - annotations_subscribed: annotations_subscribed, - autoplay: autoplay, - captions: captions, - comments: comments, - continue: continue, - continue_autoplay: continue_autoplay, - dark_mode: dark_mode, - latest_only: latest_only, - listen: listen, - local: local, - locale: locale, - max_results: max_results, - notifications_only: notifications_only, - player_style: player_style, - quality: quality, - default_home: default_home, - feed_menu: feed_menu, - related_videos: related_videos, - sort: sort, - speed: speed, - thin_mode: thin_mode, - unseen_only: unseen_only, - video_loop: video_loop, - volume: volume, - }.to_json).to_json - - if user = env.get? "user" - user = user.as(User) - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences, user.email) - - if config.admins.includes? user.email - config.default_user_preferences.default_home = env.params.body["admin_default_home"]?.try &.as(String) || config.default_user_preferences.default_home - - admin_feed_menu = [] of String - 5.times do |index| - option = env.params.body["admin_feed_menu[#{index}]"]?.try &.as(String) || "" - if !option.empty? - admin_feed_menu << option - end - end - config.default_user_preferences.feed_menu = admin_feed_menu - - captcha_enabled = env.params.body["captcha_enabled"]?.try &.as(String) - captcha_enabled ||= "off" - config.captcha_enabled = captcha_enabled == "on" - - login_enabled = env.params.body["login_enabled"]?.try &.as(String) - login_enabled ||= "off" - config.login_enabled = login_enabled == "on" - - registration_enabled = env.params.body["registration_enabled"]?.try &.as(String) - registration_enabled ||= "off" - config.registration_enabled = registration_enabled == "on" - - statistics_enabled = env.params.body["statistics_enabled"]?.try &.as(String) - statistics_enabled ||= "off" - config.statistics_enabled = statistics_enabled == "on" - - CONFIG.default_user_preferences = config.default_user_preferences - File.write("config/config.yml", config.to_yaml) - end - else - if Kemal.config.ssl || config.https_only - secure = true - else - secure = false - end - - if config.domain - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{config.domain}", value: preferences, expires: Time.utc + 2.years, - secure: secure, http_only: true) - else - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: preferences, expires: Time.utc + 2.years, - secure: secure, http_only: true) - end - end - - env.redirect referer -end - -get "/toggle_theme" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - referer = get_referer(env, unroll: false) - - redirect = env.params.query["redirect"]? - redirect ||= "true" - redirect = redirect == "true" - - if user = env.get? "user" - user = user.as(User) - preferences = user.preferences - - case preferences.dark_mode - when "dark" - preferences.dark_mode = "light" - else - preferences.dark_mode = "dark" - end - - preferences = preferences.to_json - - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences, user.email) - else - preferences = env.get("preferences").as(Preferences) - - case preferences.dark_mode - when "dark" - preferences.dark_mode = "light" - else - preferences.dark_mode = "dark" - end - - preferences = preferences.to_json - - if Kemal.config.ssl || config.https_only - secure = true - else - secure = false - end - - if config.domain - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{config.domain}", value: preferences, expires: Time.utc + 2.years, - secure: secure, http_only: true) - else - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: preferences, expires: Time.utc + 2.years, - secure: secure, http_only: true) - end - end - - if redirect - env.redirect referer - else - env.response.content_type = "application/json" - "{}" - end -end - post "/watch_ajax" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? @@ -1702,9 +348,7 @@ post "/watch_ajax" do |env| if redirect next env.redirect referer else - error_message = {"error" => "No such user"}.to_json - env.response.status_code = 403 - next error_message + next error_json(403, "No such user") end end @@ -1721,13 +365,10 @@ post "/watch_ajax" do |env| begin validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) rescue ex - env.response.status_code = 400 if redirect - error_message = ex.message - next templated "error" + next error_template(400, ex) else - error_message = {"error" => ex.message}.to_json - next error_message + next error_json(400, ex) end end @@ -1747,9 +388,7 @@ post "/watch_ajax" do |env| when "action_mark_unwatched" PG_DB.exec("UPDATE users SET watched = array_remove(watched, $1) WHERE email = $2", id, user.email) else - error_message = {"error" => "Unsupported action #{action}"}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, "Unsupported action #{action}") end if redirect @@ -1779,9 +418,7 @@ get "/modify_notifications" do |env| if redirect next env.redirect referer else - error_message = {"error" => "No such user"}.to_json - env.response.status_code = 403 - next error_message + next error_json(403, "No such user") end end @@ -1854,9 +491,7 @@ post "/subscription_ajax" do |env| if redirect next env.redirect referer else - error_message = {"error" => "No such user"}.to_json - env.response.status_code = 403 - next error_message + next error_json(403, "No such user") end end @@ -1868,13 +503,9 @@ post "/subscription_ajax" do |env| validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) rescue ex if redirect - error_message = ex.message - env.response.status_code = 400 - next templated "error" + next error_template(400, ex) else - error_message = {"error" => ex.message}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, ex) end end @@ -1898,15 +529,13 @@ post "/subscription_ajax" do |env| case action when "action_create_subscription_to_channel" if !user.subscriptions.includes? channel_id - get_channel(channel_id, PG_DB, false, false) + get_channel(channel_id, PG_DB, logger, false, false) PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_append(subscriptions, $1) WHERE email = $2", channel_id, email) end when "action_remove_subscriptions" PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_remove(subscriptions, $1) WHERE email = $2", channel_id, email) else - error_message = {"error" => "Unsupported action #{action}"}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, "Unsupported action #{action}") end if redirect @@ -1935,7 +564,7 @@ get "/subscription_manager" do |env| headers = HTTP::Headers.new headers["Cookie"] = env.request.headers["Cookie"] - user, sid = get_user(sid, headers, PG_DB) + user, sid = get_user(sid, headers, PG_DB, logger) end action_takeout = env.params.query["action_takeout"]?.try &.to_i? @@ -2059,7 +688,7 @@ post "/data_control" do |env| user.subscriptions += body["subscriptions"].as_a.map { |a| a.as_s } user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) + user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, logger, false, false) PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) end @@ -2089,7 +718,7 @@ post "/data_control" do |env| PG_DB.exec("UPDATE playlists SET description = $1 WHERE id = $2", description, playlist.id) videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| - raise "Playlist cannot have more than 500 videos" if idx > 500 + raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500 video_id = video_id.try &.as_s? next if !video_id @@ -2121,13 +750,14 @@ post "/data_control" do |env| end end when "import_youtube" - subscriptions = XML.parse(body) - user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| - channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] + subscriptions = JSON.parse(body) + + user.subscriptions += subscriptions.as_a.compact_map do |entry| + entry["snippet"]["resourceId"]["channelId"].as_s end user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) + user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, logger, false, false) PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) when "import_freetube" @@ -2136,7 +766,7 @@ post "/data_control" do |env| end user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) + user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, logger, false, false) PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) when "import_newpipe_subscriptions" @@ -2155,7 +785,7 @@ post "/data_control" do |env| end user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) + user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, logger, false, false) PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) when "import_newpipe" @@ -2174,7 +804,7 @@ post "/data_control" do |env| user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map { |url| url.lchop("https://www.youtube.com/channel/") } user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) + user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, logger, false, false) PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) @@ -2226,51 +856,37 @@ post "/change_password" do |env| # We don't store passwords for Google accounts if !user.password - error_message = "Cannot change password for Google accounts" - env.response.status_code = 400 - next templated "error" + next error_template(400, "Cannot change password for Google accounts") end begin validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) rescue ex - error_message = ex.message - env.response.status_code = 400 - next templated "error" + next error_template(400, ex) end password = env.params.body["password"]? if !password - error_message = translate(locale, "Password is a required field") - env.response.status_code = 401 - next templated "error" + next error_template(401, "Password is a required field") end new_passwords = env.params.body.select { |k, v| k.match(/^new_password\[\d+\]$/) }.map { |k, v| v } if new_passwords.size <= 1 || new_passwords.uniq.size != 1 - error_message = translate(locale, "New passwords must match") - env.response.status_code = 400 - next templated "error" + next error_template(400, "New passwords must match") end new_password = new_passwords.uniq[0] if new_password.empty? - error_message = translate(locale, "Password cannot be empty") - env.response.status_code = 401 - next templated "error" + next error_template(401, "Password cannot be empty") end if new_password.bytesize > 55 - error_message = translate(locale, "Password should not be longer than 55 characters") - env.response.status_code = 400 - next templated "error" + next error_template(400, "Password cannot be longer than 55 characters") end if !Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55)) - error_message = translate(locale, "Incorrect password") - env.response.status_code = 401 - next templated "error" + next error_template(401, "Incorrect password") end new_password = Crypto::Bcrypt::Password.create(new_password, cost: 10) @@ -2315,9 +931,7 @@ post "/delete_account" do |env| begin validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) rescue ex - error_message = ex.message - env.response.status_code = 400 - next templated "error" + next error_template(400, ex) end view_name = "subscriptions_#{sha256(user.email)}" @@ -2369,9 +983,7 @@ post "/clear_watch_history" do |env| begin validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) rescue ex - error_message = ex.message - env.response.status_code = 400 - next templated "error" + next error_template(400, ex) end PG_DB.exec("UPDATE users SET watched = '{}' WHERE email = $1", user.email) @@ -2424,9 +1036,7 @@ post "/authorize_token" do |env| begin validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) rescue ex - error_message = ex.message - env.response.status_code = 400 - next templated "error" + next error_template(400, ex) end scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v } @@ -2489,9 +1099,7 @@ post "/token_ajax" do |env| if redirect next env.redirect referer else - error_message = {"error" => "No such user"}.to_json - env.response.status_code = 403 - next error_message + next error_json(403, "No such user") end end @@ -2503,13 +1111,9 @@ post "/token_ajax" do |env| validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) rescue ex if redirect - error_message = ex.message - env.response.status_code = 400 - next templated "error" + next error_template(400, ex) else - error_message = {"error" => ex.message}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, ex) end end @@ -2526,9 +1130,7 @@ post "/token_ajax" do |env| when .starts_with? "action_revoke_token" PG_DB.exec("DELETE FROM session_ids * WHERE id = $1 AND email = $2", session, user.email) else - error_message = {"error" => "Unsupported action #{action}"}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, "Unsupported action #{action}") end if redirect @@ -2541,15 +1143,26 @@ end # Feeds +get "/feed/playlists" do |env| + env.redirect "/view_all_playlists" +end + get "/feed/top" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? - env.redirect "/" + + message = translate(locale, "The Top feed has been removed from Invidious.") + templated "message" end get "/feed/popular" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? - templated "popular" + if config.popular_enabled + templated "popular" + else + message = translate(locale, "The Popular feed has been disabled by the administrator.") + templated "message" + end end get "/feed/trending" do |env| @@ -2564,9 +1177,7 @@ get "/feed/trending" do |env| begin trending, plid = fetch_trending(trending_type, region, locale) rescue ex - error_message = "#{ex.message}" - env.response.status_code = 500 - next templated "error" + next error_template(500, ex) end templated "trending" @@ -2596,7 +1207,7 @@ get "/feed/subscriptions" do |env| headers["Cookie"] = env.request.headers["Cookie"] if !user.password - user, sid = get_user(sid, headers, PG_DB) + user, sid = get_user(sid, headers, PG_DB, logger) end max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE) @@ -2661,9 +1272,7 @@ get "/feed/channel/:ucid" do |env| rescue ex : ChannelRedirect next env.redirect env.request.resource.gsub(ucid, ex.channel_id) rescue ex - error_message = ex.message - env.response.status_code = 500 - next error_message + next error_atom(500, ex) end response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}") @@ -2704,6 +1313,7 @@ get "/feed/channel/:ucid" do |env| xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}") xml.element("id") { xml.text "yt:channel:#{channel.ucid}" } xml.element("yt:channelId") { xml.text channel.ucid } + xml.element("icon") { xml.text channel.author_thumbnail } xml.element("title") { xml.text channel.author } xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{channel.ucid}") @@ -2901,7 +1511,7 @@ post "/feed/webhook/:token" do |env| signature = env.request.headers["X-Hub-Signature"].lchop("sha1=") if signature != OpenSSL::HMAC.hexdigest(:sha1, HMAC_KEY, body) - logger.puts("#{token} : Invalid signature") + logger.error("/feed/webhook/#{token} : Invalid signature") env.response.status_code = 200 next end @@ -3074,9 +1684,7 @@ get "/channel/:ucid" do |env| rescue ex : ChannelRedirect next env.redirect env.request.resource.gsub(ucid, ex.channel_id) rescue ex - error_message = ex.message - env.response.status_code = 500 - next templated "error" + next error_template(500, ex) end if channel.auto_generated @@ -3143,9 +1751,7 @@ get "/channel/:ucid/playlists" do |env| rescue ex : ChannelRedirect next env.redirect env.request.resource.gsub(ucid, ex.channel_id) rescue ex - error_message = ex.message - env.response.status_code = 500 - next templated "error" + next error_template(500, ex) end if channel.auto_generated @@ -3183,9 +1789,7 @@ get "/channel/:ucid/community" do |env| rescue ex : ChannelRedirect next env.redirect env.request.resource.gsub(ucid, ex.channel_id) rescue ex - error_message = ex.message - env.response.status_code = 500 - next templated "error" + next error_template(500, ex) end if !channel.tabs.includes? "community" @@ -3194,9 +1798,11 @@ get "/channel/:ucid/community" do |env| begin items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode)) - rescue ex + rescue ex : InfoException env.response.status_code = 500 error_message = ex.message + rescue ex + next error_template(500, ex) end env.set "search", "channel:#{channel.ucid} " @@ -3206,12 +1812,11 @@ end # API Endpoints get "/api/v1/stats" do |env| + locale = LOCALES[env.get("preferences").as(Preferences).locale]? env.response.content_type = "application/json" if !config.statistics_enabled - error_message = {"error" => "Statistics are not enabled."}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, "Statistics are not enabled.") end Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json @@ -3231,10 +1836,8 @@ get "/api/v1/storyboards/:id" do |env| begin video = get_video(id, PG_DB, region: region) rescue ex : VideoRedirect - error_message = {"error" => "Video is unavailable", "videoId" => ex.video_id}.to_json - env.response.status_code = 302 env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) - next error_message + next error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) rescue ex env.response.status_code = 500 next @@ -3280,14 +1883,14 @@ get "/api/v1/storyboards/:id" do |env| storyboard[:storyboard_count].times do |i| url = storyboard[:url] authority = /(i\d?).ytimg.com/.match(url).not_nil![1]? - url = storyboard[:url].gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "") + url = url.gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "") url = "#{HOST_URL}/sb/#{authority}/#{url}" storyboard[:storyboard_height].times do |j| storyboard[:storyboard_width].times do |k| str << <<-END_CUE #{start_time}.000 --> #{end_time}.000 - #{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width]},#{storyboard[:height]} + #{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]} END_CUE @@ -3319,10 +1922,8 @@ get "/api/v1/captions/:id" do |env| begin video = get_video(id, PG_DB, region: region) rescue ex : VideoRedirect - error_message = {"error" => "Video is unavailable", "videoId" => ex.video_id}.to_json - env.response.status_code = 302 env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) - next error_message + next error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) rescue ex env.response.status_code = 500 next @@ -3454,9 +2055,7 @@ get "/api/v1/comments/:id" do |env| begin comments = fetch_youtube_comments(id, PG_DB, continuation, format, locale, thin_mode, region, sort_by: sort_by) rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end next comments @@ -3499,13 +2098,7 @@ end get "/api/v1/insights/:id" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - id = env.params.url["id"] - env.response.content_type = "application/json" - - error_message = {"error" => "YouTube has removed publicly available analytics."}.to_json - env.response.status_code = 410 - error_message + next error_json(410, "YouTube has removed publicly available analytics.") end get "/api/v1/annotations/:id" do |env| @@ -3540,14 +2133,13 @@ get "/api/v1/annotations/:id" do |env| file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml") - client = make_client(ARCHIVE_URL) - location = client.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}") + location = make_client(ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}")) if !location.headers["Location"]? env.response.status_code = location.status_code end - response = make_client(URI.parse(location.headers["Location"])).get(location.headers["Location"]) + response = make_client(URI.parse(location.headers["Location"]), &.get(location.headers["Location"])) if response.body.empty? env.response.status_code = 404 @@ -3594,14 +2186,10 @@ get "/api/v1/videos/:id" do |env| begin video = get_video(id, PG_DB, region: region) rescue ex : VideoRedirect - error_message = {"error" => "Video is unavailable", "videoId" => ex.video_id}.to_json - env.response.status_code = 302 env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) - next error_message + next error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end video.to_json(locale) @@ -3618,9 +2206,7 @@ get "/api/v1/trending" do |env| begin trending, plid = fetch_trending(trending_type, region, locale) rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end videos = JSON.build do |json| @@ -3639,6 +2225,12 @@ get "/api/v1/popular" do |env| env.response.content_type = "application/json" + if !config.popular_enabled + error_message = {"error" => "Administrator has disabled this endpoint."}.to_json + env.response.status_code = 400 + next error_message + end + JSON.build do |json| json.array do popular_videos.each do |video| @@ -3652,7 +2244,8 @@ get "/api/v1/top" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? env.response.content_type = "application/json" - "[]" + env.response.status_code = 400 + {"error" => "The Top feed has been removed from Invidious."}.to_json end get "/api/v1/channels/:ucid" do |env| @@ -3667,14 +2260,10 @@ get "/api/v1/channels/:ucid" do |env| begin channel = get_about_info(ucid, locale) rescue ex : ChannelRedirect - error_message = {"error" => "Channel is unavailable", "authorId" => ex.channel_id}.to_json - env.response.status_code = 302 env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) - next error_message + next error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end page = 1 @@ -3685,9 +2274,7 @@ get "/api/v1/channels/:ucid" do |env| begin count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end end @@ -3802,22 +2389,16 @@ end begin channel = get_about_info(ucid, locale) rescue ex : ChannelRedirect - error_message = {"error" => "Channel is unavailable", "authorId" => ex.channel_id}.to_json - env.response.status_code = 302 env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) - next error_message + next error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end begin count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end JSON.build do |json| @@ -3841,9 +2422,7 @@ end begin videos = get_latest_videos(ucid) rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end JSON.build do |json| @@ -3871,14 +2450,10 @@ end begin channel = get_about_info(ucid, locale) rescue ex : ChannelRedirect - error_message = {"error" => "Channel is unavailable", "authorId" => ex.channel_id}.to_json - env.response.status_code = 302 env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) - next error_message + next error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end items, continuation = fetch_channel_playlists(channel.ucid, channel.author, channel.auto_generated, continuation, sort_by) @@ -3919,9 +2494,7 @@ end begin fetch_channel_community(ucid, continuation, locale, format, thin_mode) rescue ex - env.response.status_code = 400 - error_message = {"error" => ex.message}.to_json - next error_message + next error_json(500, ex) end end end @@ -3979,9 +2552,7 @@ get "/api/v1/search" do |env| begin search_params = produce_search_params(sort_by, date, content_type, duration, features) rescue ex - env.response.status_code = 400 - error_message = {"error" => ex.message}.to_json - next error_message + next error_json(400, ex) end count, search_results = search(query, page, search_params, region).as(Tuple) @@ -4024,9 +2595,7 @@ get "/api/v1/search/suggestions" do |env| end end rescue ex - env.response.status_code = 500 - error_message = {"error" => ex.message}.to_json - next error_message + next error_json(500, ex) end end @@ -4053,16 +2622,12 @@ end begin playlist = get_playlist(PG_DB, plid, locale) rescue ex - env.response.status_code = 404 - error_message = {"error" => "Playlist does not exist."}.to_json - next error_message + next error_json(404, "Playlist does not exist.") end user = env.get?("user").try &.as(User) if !playlist || playlist.privacy.private? && playlist.author != user.try &.email - env.response.status_code = 404 - error_message = {"error" => "Playlist does not exist."}.to_json - next error_message + next error_json(404, "Playlist does not exist.") end response = playlist.to_json(offset, locale, continuation: continuation) @@ -4106,9 +2671,7 @@ get "/api/v1/mixes/:rdid" do |env| mix.videos = mix.videos[index..-1] rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end response = JSON.build do |json| @@ -4264,7 +2827,7 @@ post "/api/v1/auth/subscriptions/:ucid" do |env| ucid = env.params.url["ucid"] if !user.subscriptions.includes? ucid - get_channel(ucid, PG_DB, false, false) + get_channel(ucid, PG_DB, logger, false, false) PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email) end @@ -4310,22 +2873,16 @@ post "/api/v1/auth/playlists" do |env| title = env.params.json["title"]?.try &.as(String).delete("<>").byte_slice(0, 150) if !title - error_message = {"error" => "Invalid title."}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, "Invalid title.") end privacy = env.params.json["privacy"]?.try { |privacy| PlaylistPrivacy.parse(privacy.as(String).downcase) } if !privacy - error_message = {"error" => "Invalid privacy setting."}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, "Invalid privacy setting.") end if PG_DB.query_one("SELECT count(*) FROM playlists WHERE author = $1", user.email, as: Int64) >= 100 - error_message = {"error" => "User cannot have more than 100 playlists."}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, "User cannot have more than 100 playlists.") end playlist = create_playlist(PG_DB, title, privacy, user) @@ -4347,15 +2904,11 @@ patch "/api/v1/auth/playlists/:plid" do |env| playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) if !playlist || playlist.author != user.email && playlist.privacy.private? - env.response.status_code = 404 - error_message = {"error" => "Playlist does not exist."}.to_json - next error_message + next error_json(404, "Playlist does not exist.") end if playlist.author != user.email - env.response.status_code = 403 - error_message = {"error" => "Invalid user"}.to_json - next error_message + next error_json(403, "Invalid user") end title = env.params.json["title"].try &.as(String).delete("<>").byte_slice(0, 150) || playlist.title @@ -4375,6 +2928,8 @@ patch "/api/v1/auth/playlists/:plid" do |env| end delete "/api/v1/auth/playlists/:plid" do |env| + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + env.response.content_type = "application/json" user = env.get("user").as(User) @@ -4382,15 +2937,11 @@ delete "/api/v1/auth/playlists/:plid" do |env| playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) if !playlist || playlist.author != user.email && playlist.privacy.private? - env.response.status_code = 404 - error_message = {"error" => "Playlist does not exist."}.to_json - next error_message + next error_json(404, "Playlist does not exist.") end if playlist.author != user.email - env.response.status_code = 403 - error_message = {"error" => "Invalid user"}.to_json - next error_message + next error_json(403, "Invalid user") end PG_DB.exec("DELETE FROM playlist_videos * WHERE plid = $1", plid) @@ -4409,36 +2960,26 @@ post "/api/v1/auth/playlists/:plid/videos" do |env| playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) if !playlist || playlist.author != user.email && playlist.privacy.private? - env.response.status_code = 404 - error_message = {"error" => "Playlist does not exist."}.to_json - next error_message + next error_json(404, "Playlist does not exist.") end if playlist.author != user.email - env.response.status_code = 403 - error_message = {"error" => "Invalid user"}.to_json - next error_message + next error_json(403, "Invalid user") end if playlist.index.size >= 500 - env.response.status_code = 400 - error_message = {"error" => "Playlist cannot have more than 500 videos"}.to_json - next error_message + next error_json(400, "Playlist cannot have more than 500 videos") end video_id = env.params.json["videoId"].try &.as(String) if !video_id - env.response.status_code = 403 - error_message = {"error" => "Invalid videoId"}.to_json - next error_message + next error_json(403, "Invalid videoId") end begin video = get_video(video_id, PG_DB) rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end playlist_video = PlaylistVideo.new({ @@ -4465,6 +3006,8 @@ post "/api/v1/auth/playlists/:plid/videos" do |env| end delete "/api/v1/auth/playlists/:plid/videos/:index" do |env| + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + env.response.content_type = "application/json" user = env.get("user").as(User) @@ -4473,21 +3016,15 @@ delete "/api/v1/auth/playlists/:plid/videos/:index" do |env| playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) if !playlist || playlist.author != user.email && playlist.privacy.private? - env.response.status_code = 404 - error_message = {"error" => "Playlist does not exist."}.to_json - next error_message + next error_json(404, "Playlist does not exist.") end if playlist.author != user.email - env.response.status_code = 403 - error_message = {"error" => "Invalid user"}.to_json - next error_message + next error_json(403, "Invalid user") end if !playlist.index.includes? index - env.response.status_code = 404 - error_message = {"error" => "Playlist does not contain index"}.to_json - next error_message + next error_json(404, "Playlist does not contain index") end PG_DB.exec("DELETE FROM playlist_videos * WHERE index = $1", index) @@ -4533,9 +3070,7 @@ post "/api/v1/auth/tokens/register" do |env| callback_url = env.params.json["callbackUrl"]?.try &.as(String) expire = env.params.json["expire"]?.try &.as(Int64) else - error_message = {"error" => "Invalid or missing header 'Content-Type'"}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, "Invalid or missing header 'Content-Type'") end if callback_url && callback_url.empty? @@ -4585,6 +3120,7 @@ post "/api/v1/auth/tokens/register" do |env| end post "/api/v1/auth/tokens/unregister" do |env| + locale = LOCALES[env.get("preferences").as(Preferences).locale]? env.response.content_type = "application/json" user = env.get("user").as(User) scopes = env.get("scopes").as(Array(String)) @@ -4598,9 +3134,7 @@ post "/api/v1/auth/tokens/unregister" do |env| elsif scopes_include_scope(scopes, "GET:tokens") PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", session) else - error_message = {"error" => "Cannot revoke session #{session}"}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, "Cannot revoke session #{session}") end env.response.status_code = 204 @@ -4924,6 +3458,7 @@ get "/videoplayback/*" do |env| end get "/videoplayback" do |env| + locale = LOCALES[env.get("preferences").as(Preferences).locale]? query_params = env.params.query fvip = query_params["fvip"]? || "3" @@ -4962,8 +3497,12 @@ get "/videoplayback" do |env| location = URI.parse(response.headers["Location"]) env.response.headers["Access-Control-Allow-Origin"] = "*" - host = "#{location.scheme}://#{location.host}" - client = make_client(URI.parse(host), region) + new_host = "#{location.scheme}://#{location.host}" + if new_host != host + host = new_host + client.close + client = make_client(URI.parse(new_host), region) + end url = "#{location.full_path}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" else @@ -4990,13 +3529,10 @@ get "/videoplayback" do |env| if url.includes? "&file=seg.ts" if CONFIG.disabled?("livestreams") - env.response.status_code = 403 - error_message = "Administrator has disabled this endpoint." - next templated "error" + next error_template(403, "Administrator has disabled this endpoint.") end begin - client = make_client(URI.parse(host), region) client.get(url, headers) do |response| response.headers.each do |key, value| if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) @@ -5024,9 +3560,7 @@ get "/videoplayback" do |env| else if query_params["title"]? && CONFIG.disabled?("downloads") || CONFIG.disabled?("dash") - env.response.status_code = 403 - error_message = "Administrator has disabled this endpoint." - next templated "error" + next error_template(403, "Administrator has disabled this endpoint.") end content_length = nil @@ -5039,8 +3573,6 @@ get "/videoplayback" do |env| chunk_end = chunk_start + HTTP_CHUNK_SIZE - 1 end - client = make_client(URI.parse(host), region) - # TODO: Record bytes written so we can restart after a chunk fails while true if !range_end && content_length @@ -5104,6 +3636,7 @@ get "/videoplayback" do |env| if ex.message != "Error reading socket: Connection reset by peer" break else + client.close client = make_client(URI.parse(host), region) end end @@ -5113,6 +3646,7 @@ get "/videoplayback" do |env| first_chunk = false end end + client.close end get "/ggpht/*" do |env| @@ -5367,14 +3901,9 @@ error 404 do |env| halt env, status_code: 302 end -error 500 do |env| - error_message = <<-END_HTML - Looks like you've found a bug in Invidious. Feel free to open a new issue - <a href="https://github.com/iv-org/invidious/issues">here</a> - or send an email to - <a href="mailto:#{CONFIG.admin_email}">#{CONFIG.admin_email}</a>. - END_HTML - templated "error" +error 500 do |env, ex| + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + error_template(500, ex) end static_headers do |response, filepath, filestat| diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index 656b9953..6907ff3d 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -144,7 +144,7 @@ class ChannelRedirect < Exception end end -def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, max_threads = 10) +def get_batch_channels(channels, db, logger, refresh = false, pull_all_videos = true, max_threads = 10) finished_channel = Channel(String | Nil).new spawn do @@ -160,7 +160,7 @@ def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, ma active_threads += 1 spawn do begin - get_channel(ucid, db, refresh, pull_all_videos) + get_channel(ucid, db, logger, refresh, pull_all_videos) finished_channel.send(ucid) rescue ex finished_channel.send(nil) @@ -181,10 +181,10 @@ def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, ma return final end -def get_channel(id, db, refresh = true, pull_all_videos = true) +def get_channel(id, db, logger, refresh = true, pull_all_videos = true) if channel = db.query_one?("SELECT * FROM channels WHERE id = $1", id, as: InvidiousChannel) if refresh && Time.utc - channel.updated > 10.minutes - channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) + channel = fetch_channel(id, db, logger, pull_all_videos: pull_all_videos) channel_array = channel.to_a args = arg_array(channel_array) @@ -192,7 +192,7 @@ def get_channel(id, db, refresh = true, pull_all_videos = true) ON CONFLICT (id) DO UPDATE SET author = $2, updated = $3", args: channel_array) end else - channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) + channel = fetch_channel(id, db, logger, pull_all_videos: pull_all_videos) channel_array = channel.to_a args = arg_array(channel_array) @@ -202,13 +202,17 @@ def get_channel(id, db, refresh = true, pull_all_videos = true) return channel end -def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) +def fetch_channel(ucid, db, logger, pull_all_videos = true, locale = nil) + logger.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}, locale = #{locale}") + + logger.trace("fetch_channel: #{ucid} : Downloading RSS feed") rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body + logger.trace("fetch_channel: #{ucid} : Parsing RSS feed") rss = XML.parse_html(rss) author = rss.xpath_node(%q(//feed/title)) if !author - raise translate(locale, "Deleted or invalid channel") + raise InfoException.new("Deleted or invalid channel") end author = author.content @@ -219,22 +223,29 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) auto_generated = true end + logger.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}") + page = 1 + logger.trace("fetch_channel: #{ucid} : Downloading channel videos page") response = get_channel_videos_response(ucid, page, auto_generated: auto_generated) videos = [] of SearchVideo begin initial_data = JSON.parse(response.body).as_a.find &.["response"]? - raise "Could not extract JSON" if !initial_data + raise InfoException.new("Could not extract channel JSON") if !initial_data + + logger.trace("fetch_channel: #{ucid} : Extracting videos from channel videos page initial_data") videos = extract_videos(initial_data.as_h, author, ucid) rescue ex if response.body.includes?("To continue with your YouTube experience, please fill out the form below.") || response.body.includes?("https://www.google.com/sorry/index") - raise "Could not extract channel info. Instance is likely blocked." + raise InfoException.new("Could not extract channel info. Instance is likely blocked.") end + raise ex end + logger.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed") rss.xpath_nodes("//feed/entry").each do |entry| video_id = entry.xpath_node("videoid").not_nil!.content title = entry.xpath_node("title").not_nil!.content @@ -268,6 +279,8 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) views: views, }) + logger.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video") + # We don't include the 'premiere_timestamp' here because channel pages don't include them, # meaning the above timestamp is always null was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ @@ -275,8 +288,13 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) updated = $4, ucid = $5, author = $6, length_seconds = $7, \ live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) - db.exec("UPDATE users SET notifications = array_append(notifications, $1), \ - feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert + if was_insert + logger.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions") + db.exec("UPDATE users SET notifications = array_append(notifications, $1), \ + feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) + else + logger.trace("fetch_channel: #{ucid} : video #{video_id} : Updated") + end end if pull_all_videos @@ -287,7 +305,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) loop do response = get_channel_videos_response(ucid, page, auto_generated: auto_generated) initial_data = JSON.parse(response.body).as_a.find &.["response"]? - raise "Could not extract JSON" if !initial_data + raise InfoException.new("Could not extract channel JSON") if !initial_data videos = extract_videos(initial_data.as_h, author, ucid) count = videos.size @@ -507,8 +525,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) end if response.status_code != 200 - error_message = translate(locale, "This channel does not exist.") - raise error_message + raise InfoException.new("This channel does not exist.") end ucid = response.body.match(/https:\/\/www.youtube.com\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/).not_nil!["ucid"] @@ -518,7 +535,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) body = initial_data["contents"]?.try &.["twoColumnBrowseResultsRenderer"]["tabs"].as_a.select { |tab| tab["tabRenderer"]?.try &.["selected"].as_bool.== true }[0]? if !body - raise "Could not extract community tab." + raise InfoException.new("Could not extract community tab.") end body = body["tabRenderer"]["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"] @@ -540,7 +557,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) body["response"]["continuationContents"]["backstageCommentsContinuation"]? if !body - raise "Could not extract continuation." + raise InfoException.new("Could not extract continuation.") end end @@ -551,7 +568,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) error_message = (message["text"]["simpleText"]? || message["text"]["runs"]?.try &.[0]?.try &.["text"]?) .try &.as_s || "" - raise error_message + raise InfoException.new(error_message) end response = JSON.build do |json| @@ -634,7 +651,8 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) else video_id = attachment["videoId"].as_s - json.field "title", attachment["title"]["simpleText"].as_s + video_title = attachment["title"]["simpleText"]? || attachment["title"]["runs"]?.try &.[0]?.try &.["text"]? + json.field "title", video_title json.field "videoId", video_id json.field "videoThumbnails" do generate_thumbnails(json, video_id) @@ -656,7 +674,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) json.field "published", published.to_unix json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale)) - view_count = attachment["viewCountText"]["simpleText"].as_s.gsub(/\D/, "").to_i64? || 0_i64 + view_count = attachment["viewCountText"]?.try &.["simpleText"].as_s.gsub(/\D/, "").to_i64? || 0_i64 json.field "viewCount", view_count json.field "viewCountText", translate(locale, "`x` views", number_to_short_text(view_count)) @@ -775,46 +793,41 @@ def extract_channel_community_cursor(continuation) cursor end -INITDATA_PREQUERY = "window[\"ytInitialData\"] = {" - def get_about_info(ucid, locale) - about = YT_POOL.client &.get("/channel/#{ucid}/about?gl=US&hl=en") - if about.status_code != 200 - about = YT_POOL.client &.get("/user/#{ucid}/about?gl=US&hl=en") + result = YT_POOL.client &.get("/channel/#{ucid}/about?gl=US&hl=en") + if result.status_code != 200 + result = YT_POOL.client &.get("/user/#{ucid}/about?gl=US&hl=en") end - if md = about.headers["location"]?.try &.match(/\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/) + if md = result.headers["location"]?.try &.match(/\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/) raise ChannelRedirect.new(channel_id: md["ucid"]) end - if about.status_code != 200 - error_message = translate(locale, "This channel does not exist.") - raise error_message + if result.status_code != 200 + raise InfoException.new("This channel does not exist.") + end + + about = XML.parse_html(result.body) + if about.xpath_node(%q(//div[contains(@class, "channel-empty-message")])) + raise InfoException.new("This channel does not exist.") end - initdata_pre = about.body.index(INITDATA_PREQUERY) - initdata_post = initdata_pre.nil? ? nil : about.body.index("};", initdata_pre) - if initdata_post.nil? - about = XML.parse_html(about.body) + initdata = extract_initial_data(result.body) + if initdata.empty? error_message = about.xpath_node(%q(//div[@class="yt-alert-content"])).try &.content.strip error_message ||= translate(locale, "Could not get channel info.") - raise error_message + raise InfoException.new(error_message) end - initdata_pre = initdata_pre.not_nil! + INITDATA_PREQUERY.size - 1 - - initdata = JSON.parse(about.body[initdata_pre, initdata_post - initdata_pre + 1]) - about = XML.parse_html(about.body) - if about.xpath_node(%q(//div[contains(@class, "channel-empty-message")])) - error_message = translate(locale, "This channel does not exist.") - raise error_message + if browse_endpoint = initdata["onResponseReceivedActions"]?.try &.[0]?.try &.["navigateAction"]?.try &.["endpoint"]?.try &.["browseEndpoint"]? + raise ChannelRedirect.new(channel_id: browse_endpoint["browseId"].to_s) end - author = about.xpath_node(%q(//meta[@name="title"])).not_nil!["content"] - author_url = about.xpath_node(%q(//link[@rel="canonical"])).not_nil!["href"] - author_thumbnail = about.xpath_node(%q(//link[@rel="image_src"])).not_nil!["href"] + author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s + author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s + author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s - ucid = about.xpath_node(%q(//meta[@itemprop="channelId"])).not_nil!["content"] + ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s # Raises a KeyError on failure. banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index 407cef78..8849c87f 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -88,11 +88,11 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so "cookie" => video.cookie, } - response = YT_POOL.client(region, &.post("/comment_service_ajax?action_get_comments=1&hl=en&gl=US", headers, form: post_req)) + response = YT_POOL.client(region, &.post("/comment_service_ajax?action_get_comments=1&hl=en&gl=US&pbj=1", headers, form: post_req)) response = JSON.parse(response.body) if !response["response"]["continuationContents"]? - raise translate(locale, "Could not fetch comments") + raise InfoException.new("Could not fetch comments") end response = response["response"]["continuationContents"] @@ -266,9 +266,11 @@ def fetch_reddit_comments(id, sort_by = "confidence") thread = result[0].data.as(RedditListing).children[0].data.as(RedditLink) else - raise "Got error code #{search_results.status_code}" + raise InfoException.new("Could not fetch comments") end + client.close + comments = result[1].data.as(RedditListing).children return comments, thread end @@ -581,13 +583,17 @@ def produce_comment_continuation(video_id, cursor = "", sort_by = "top") object = { "2:embedded" => { "2:string" => video_id, - "24:varint" => 1_i64, - "25:varint" => 1_i64, + "25:varint" => 0_i64, "28:varint" => 1_i64, "36:embedded" => { "5:varint" => -1_i64, "8:varint" => 0_i64, }, + "40:embedded" => { + "1:varint" => 4_i64, + "3:string" => "https://www.youtube.com", + "4:string" => "", + }, }, "3:varint" => 6_i64, "6:embedded" => { diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr new file mode 100644 index 00000000..4487ff8c --- /dev/null +++ b/src/invidious/helpers/errors.cr @@ -0,0 +1,103 @@ +# InfoExceptions are for displaying information to the user. +# +# An InfoException might or might not indicate that something went wrong. +# Historically Invidious didn't differentiate between these two options, so to +# maintain previous functionality InfoExceptions do not print backtraces. +class InfoException < Exception +end + +macro error_template(*args) + error_template_helper(env, config, locale, {{*args}}) +end + +def github_details(summary : String, content : String) + details = %(\n<details>) + details += %(\n<summary>#{summary}</summary>) + details += %(\n<p>) + details += %(\n \n```\n) + details += content.strip + details += %(\n```) + details += %(\n</p>) + details += %(\n</details>) + return HTML.escape(details) +end + +def error_template_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception) + if exception.is_a?(InfoException) + return error_template_helper(env, config, locale, status_code, exception.message || "") + end + env.response.status_code = status_code + issue_template = %(Title: `#{exception.message} (#{exception.class})`) + issue_template += %(\nDate: `#{Time::Format::ISO_8601_DATE_TIME.format(Time.utc)}`) + issue_template += %(\nRoute: `#{env.request.resource}`) + issue_template += %(\nVersion: `#{SOFTWARE["version"]} @ #{SOFTWARE["branch"]}`) + # issue_template += github_details("Preferences", env.get("preferences").as(Preferences).to_pretty_json) + issue_template += github_details("Backtrace", exception.inspect_with_backtrace) + error_message = <<-END_HTML + Looks like you've found a bug in Invidious. Please open a new issue + <a href="https://github.com/iv-org/invidious/issues">on GitHub</a> + and include the following text in your message: + <pre style="padding: 20px; background: rgba(0, 0, 0, 0.12345);">#{issue_template}</pre> + END_HTML + return templated "error" +end + +def error_template_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String) + env.response.status_code = status_code + error_message = translate(locale, message) + return templated "error" +end + +macro error_atom(*args) + error_atom_helper(env, config, locale, {{*args}}) +end + +def error_atom_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception) + if exception.is_a?(InfoException) + return error_atom_helper(env, config, locale, status_code, exception.message || "") + end + env.response.content_type = "application/atom+xml" + env.response.status_code = status_code + return "<error>#{exception.inspect_with_backtrace}</error>" +end + +def error_atom_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String) + env.response.content_type = "application/atom+xml" + env.response.status_code = status_code + return "<error>#{message}</error>" +end + +macro error_json(*args) + error_json_helper(env, config, locale, {{*args}}) +end + +def error_json_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception, additional_fields : Hash(String, Object) | Nil) + if exception.is_a?(InfoException) + return error_json_helper(env, config, locale, status_code, exception.message || "", additional_fields) + end + env.response.content_type = "application/json" + env.response.status_code = status_code + error_message = {"error" => exception.message, "errorBacktrace" => exception.inspect_with_backtrace} + if additional_fields + error_message = error_message.merge(additional_fields) + end + return error_message.to_json +end + +def error_json_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception) + return error_json_helper(env, config, locale, status_code, exception, nil) +end + +def error_json_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String, additional_fields : Hash(String, Object) | Nil) + env.response.content_type = "application/json" + env.response.status_code = status_code + error_message = {"error" => message} + if additional_fields + error_message = error_message.merge(additional_fields) + end + return error_message.to_json +end + +def error_json_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String) + error_json_helper(env, config, locale, status_code, message, nil) +end diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 62c24f3e..2da49abb 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -41,6 +41,7 @@ struct ConfigPreferences property notifications_only : Bool = false property player_style : String = "invidious" property quality : String = "hd720" + property quality_dash : String = "auto" property default_home : String = "Popular" property feed_menu : Array(String) = ["Popular", "Trending", "Subscriptions", "Playlists"] property related_videos : Bool = true @@ -60,7 +61,7 @@ struct ConfigPreferences end end -struct Config +class Config include YAML::Serializable property channel_threads : Int32 # Number of threads to use for crawling videos from channels (for updating subscriptions) @@ -71,6 +72,7 @@ struct Config property hmac_key : String? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions property domain : String? # Domain to be used for links to resources on the site where an absolute URL is required property use_pubsub_feeds : Bool | Int32 = false # Subscribe to channels using PubSubHubbub (requires domain, hmac_key) + property popular_enabled : Bool = true property captcha_enabled : Bool = true property login_enabled : Bool = true property registration_enabled : Bool = true @@ -93,8 +95,9 @@ struct Config property admin_email : String = "omarroth@protonmail.com" # Email for bug reports @[YAML::Field(converter: Preferences::StringToCookies)] - property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format - property captcha_key : String? = nil # Key for Anti-Captcha + property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format + property captcha_key : String? = nil # Key for Anti-Captcha + property captcha_api_url : String = "https://api.anti-captcha.com" # API URL for Anti-Captcha def disabled?(option) case disabled = CONFIG.disable_proxy @@ -334,7 +337,7 @@ def check_enum(db, logger, enum_name, struct_type = nil) return # TODO if !db.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool) - logger.puts("CREATE TYPE #{enum_name}") + logger.info("check_enum: CREATE TYPE #{enum_name}") db.using_connection do |conn| conn.as(PG::Connection).exec_all(File.read("config/sql/#{enum_name}.sql")) @@ -347,7 +350,7 @@ def check_table(db, logger, table_name, struct_type = nil) begin db.exec("SELECT * FROM #{table_name} LIMIT 0") rescue ex - logger.puts("CREATE TABLE #{table_name}") + logger.info("check_table: check_table: CREATE TABLE #{table_name}") db.using_connection do |conn| conn.as(PG::Connection).exec_all(File.read("config/sql/#{table_name}.sql")) @@ -367,7 +370,7 @@ def check_table(db, logger, table_name, struct_type = nil) if name != column_array[i]? if !column_array[i]? new_column = column_types.select { |line| line.starts_with? name }[0] - logger.puts("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") + logger.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}") db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") next end @@ -385,29 +388,29 @@ def check_table(db, logger, table_name, struct_type = nil) # There's a column we didn't expect if !new_column - logger.puts("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]}") + logger.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]}") db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") column_array = get_column_array(db, table_name) next end - logger.puts("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") + logger.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}") db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") - logger.puts("UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}") + logger.info("check_table: UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}") db.exec("UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}") - logger.puts("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") + logger.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") - logger.puts("ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}") + logger.info("check_table: ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}") db.exec("ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}") column_array = get_column_array(db, table_name) end else - logger.puts("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") + logger.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") end end @@ -417,7 +420,7 @@ def check_table(db, logger, table_name, struct_type = nil) column_array.each do |column| if !struct_array.includes? column - logger.puts("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE") + logger.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE") db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE") end end @@ -597,12 +600,7 @@ def create_notification_stream(env, topics, connection_channel) end def extract_initial_data(body) : Hash(String, JSON::Any) - initial_data = body.match(/(window\["ytInitialData"\]|var\s+ytInitialData)\s*=\s*(?<info>.*?);+\s*\n/).try &.["info"] || "{}" - if initial_data.starts_with?("JSON.parse(\"") - return JSON.parse(JSON.parse(%({"initial_data":"#{initial_data[12..-3]}"}))["initial_data"].as_s).as_h - else - return JSON.parse(initial_data).as_h - end + return JSON.parse(body.match(/(window\["ytInitialData"\]|var\s*ytInitialData)\s*=\s*(?<info>\{.*?\});/mx).try &.["info"] || "{}").as_h end def proxy_file(response, env) diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr index 52f0a22c..4e4d7306 100644 --- a/src/invidious/helpers/logger.cr +++ b/src/invidious/helpers/logger.cr @@ -1,66 +1,52 @@ require "logger" enum LogLevel + All + Trace Debug Info Warn Error + Fatal + Off end class Invidious::LogHandler < Kemal::BaseLogHandler - def initialize(@io : IO = STDOUT, @level = LogLevel::Warn) + def initialize(@io : IO = STDOUT, @level = LogLevel::Debug) end def call(context : HTTP::Server::Context) - time = Time.utc - call_next(context) - elapsed_text = elapsed_text(Time.utc - time) + elapsed_time = Time.measure { call_next(context) } + elapsed_text = elapsed_text(elapsed_time) - @io << time << ' ' << context.response.status_code << ' ' << context.request.method << ' ' << context.request.resource << ' ' << elapsed_text << '\n' - - if @io.is_a? File - @io.flush - end + info("#{context.response.status_code} #{context.request.method} #{context.request.resource} #{elapsed_text}") context end def puts(message : String) @io << message << '\n' - - if @io.is_a? File - @io.flush - end + @io.flush end - def write(message : String, level = @level) + def write(message : String) @io << message - - if @io.is_a? File - @io.flush - end + @io.flush end def set_log_level(level : String) - case level.downcase - when "debug" - set_log_level(LogLevel::Debug) - when "info" - set_log_level(LogLevel::Info) - when "warn" - set_log_level(LogLevel::Warn) - when "error" - set_log_level(LogLevel::Error) - end + @level = LogLevel.parse(level) end def set_log_level(level : LogLevel) @level = level end - {% for level in %w(debug info warn error) %} + {% for level in %w(trace debug info warn error fatal) %} def {{level.id}}(message : String) - puts(message, LogLevel::{{level.id.capitalize}}) + if LogLevel::{{level.id.capitalize}} >= @level + puts("#{Time.utc} [{{level.id}}] #{message}") + end end {% end %} diff --git a/src/invidious/helpers/proxy.cr b/src/invidious/helpers/proxy.cr index 4f415ba0..7a42ef41 100644 --- a/src/invidious/helpers/proxy.cr +++ b/src/invidious/helpers/proxy.cr @@ -108,7 +108,9 @@ def filter_proxies(proxies) proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) client.set_proxy(proxy) - client.head("/").status_code == 200 + status_ok = client.head("/").status_code == 200 + client.close + status_ok rescue ex false end @@ -132,6 +134,7 @@ def get_nova_proxies(country_code = "US") headers["Referer"] = "https://www.proxynova.com/proxy-server-list/country-#{country_code}/" response = client.get("/proxy-server-list/country-#{country_code}/", headers) + client.close document = XML.parse_html(response.body) proxies = [] of {ip: String, port: Int32, score: Float64} @@ -177,6 +180,7 @@ def get_spys_proxies(country_code = "US") } response = client.post("/free-proxy-list/#{country_code}/", headers, form: body) + client.close 20.times do if response.status_code == 200 break diff --git a/src/invidious/helpers/tokens.cr b/src/invidious/helpers/tokens.cr index 39aae367..a09ce90b 100644 --- a/src/invidious/helpers/tokens.cr +++ b/src/invidious/helpers/tokens.cr @@ -70,33 +70,33 @@ def validate_request(token, session, request, key, db, locale = nil) when JSON::Any token = token.as_h when Nil - raise translate(locale, "Hidden field \"token\" is a required field") + raise InfoException.new("Hidden field \"token\" is a required field") end expire = token["expire"]?.try &.as_i if expire.try &.< Time.utc.to_unix - raise translate(locale, "Token is expired, please try again") + raise InfoException.new("Token is expired, please try again") end if token["session"] != session - raise translate(locale, "Erroneous token") + raise InfoException.new("Erroneous token") end scopes = token["scopes"].as_a.map { |v| v.as_s } scope = "#{request.method}:#{request.path.lchop("/api/v1/auth/").lstrip("/")}" if !scopes_include_scope(scopes, scope) - raise translate(locale, "Invalid scope") + raise InfoException.new("Invalid scope") end if !Crypto::Subtle.constant_time_compare(token["signature"].to_s, sign_token(key, token)) - raise translate(locale, "Invalid signature") + raise InfoException.new("Invalid signature") end if token["nonce"]? && (nonce = db.query_one?("SELECT * FROM nonces WHERE nonce = $1", token["nonce"], as: {String, Time})) if nonce[1] > Time.utc db.exec("UPDATE nonces SET expire = $1 WHERE nonce = $2", Time.utc(1990, 1, 1), nonce[0]) else - raise translate(locale, "Erroneous token") + raise InfoException.new("Erroneous token") end end diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index a51f15ce..f068b5f2 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -83,6 +83,7 @@ def make_client(url : URI, region = nil) # TODO: Migrate any applicable endpoints to QUIC client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure) client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC + client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" client.read_timeout = 10.seconds client.connect_timeout = 10.seconds @@ -100,6 +101,15 @@ def make_client(url : URI, region = nil) return client end +def make_client(url : URI, region = nil, &block) + client = make_client(url, region) + begin + yield client + ensure + client.close + end +end + def decode_length_seconds(string) length_seconds = string.gsub(/[^0-9:]/, "").split(":").map &.to_i length_seconds = [0] * (3 - length_seconds.size) + length_seconds @@ -360,7 +370,7 @@ def subscribe_pubsub(topic, key, config) "hub.secret" => key.to_s, } - return make_client(PUBSUB_URL).post("/subscribe", form: body) + return make_client(PUBSUB_URL, &.post("/subscribe", form: body)) end def parse_range(range) diff --git a/src/invidious/jobs/bypass_captcha_job.cr b/src/invidious/jobs/bypass_captcha_job.cr index 8b69e01a..61f8eaf3 100644 --- a/src/invidious/jobs/bypass_captcha_job.cr +++ b/src/invidious/jobs/bypass_captcha_job.cr @@ -23,7 +23,8 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob headers = response.cookies.add_request_headers(HTTP::Headers.new) - response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/createTask", body: { + response = JSON.parse(HTTP::Client.post(config.captcha_api_url + "/createTask", + headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { "clientKey" => config.captcha_key, "task" => { "type" => "NoCaptchaTaskProxyless", @@ -39,7 +40,8 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob loop do sleep 10.seconds - response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/getTaskResult", body: { + response = JSON.parse(HTTP::Client.post(config.captcha_api_url + "/getTaskResult", + headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { "clientKey" => config.captcha_key, "taskId" => task_id, }.to_json).body) @@ -76,9 +78,10 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob inputs[node["name"]] = node["value"] end - captcha_client = HTTPClient.new(URI.parse("https://api.anti-captcha.com")) + captcha_client = HTTPClient.new(URI.parse(config.captcha_api_url)) captcha_client.family = config.force_resolve || Socket::Family::INET - response = JSON.parse(captcha_client.post("/createTask", body: { + response = JSON.parse(captcha_client.post("/createTask", + headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { "clientKey" => config.captcha_key, "task" => { "type" => "NoCaptchaTaskProxyless", @@ -88,13 +91,16 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob }, }.to_json).body) + captcha_client.close + raise response["error"].as_s if response["error"]? task_id = response["taskId"].as_i loop do sleep 10.seconds - response = JSON.parse(captcha_client.post("/getTaskResult", body: { + response = JSON.parse(captcha_client.post("/getTaskResult", + headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { "clientKey" => config.captcha_key, "taskId" => task_id, }.to_json).body) @@ -121,7 +127,7 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob end end rescue ex - logger.puts("Exception: #{ex.message}") + logger.error("BypassCaptchaJob: #{ex.message}") ensure sleep 1.minute Fiber.yield diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr index 75fc474d..6c858afa 100644 --- a/src/invidious/jobs/refresh_channels_job.cr +++ b/src/invidious/jobs/refresh_channels_job.cr @@ -7,37 +7,44 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob end def begin - max_threads = config.channel_threads - lim_threads = max_threads - active_threads = 0 + max_fibers = config.channel_threads + lim_fibers = max_fibers + active_fibers = 0 active_channel = Channel(Bool).new backoff = 1.seconds loop do + logger.debug("RefreshChannelsJob: Refreshing all channels") db.query("SELECT id FROM channels ORDER BY updated") do |rs| rs.each do id = rs.read(String) - if active_threads >= lim_threads + if active_fibers >= lim_fibers + logger.trace("RefreshChannelsJob: Fiber limit reached, waiting...") if active_channel.receive - active_threads -= 1 + logger.trace("RefreshChannelsJob: Fiber limit ok, continuing") + active_fibers -= 1 end end - active_threads += 1 + logger.trace("RefreshChannelsJob: #{id} : Spawning fiber") + active_fibers += 1 spawn do begin - channel = fetch_channel(id, db, config.full_refresh) + logger.trace("RefreshChannelsJob: #{id} fiber : Fetching channel") + channel = fetch_channel(id, db, logger, config.full_refresh) - lim_threads = max_threads + lim_fibers = max_fibers + + logger.trace("RefreshChannelsJob: #{id} fiber : Updating DB") db.exec("UPDATE channels SET updated = $1, author = $2, deleted = false WHERE id = $3", Time.utc, channel.author, id) rescue ex - logger.puts("#{id} : #{ex.message}") + logger.error("RefreshChannelsJob: #{id} : #{ex.message}") if ex.message == "Deleted or invalid channel" db.exec("UPDATE channels SET updated = $1, deleted = true WHERE id = $2", Time.utc, id) else - lim_threads = 1 - logger.puts("#{id} : backing off for #{backoff}s") + lim_fibers = 1 + logger.error("RefreshChannelsJob: #{id} fiber : backing off for #{backoff}s") sleep backoff if backoff < 1.days backoff += backoff @@ -45,13 +52,15 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob backoff = 1.days end end + ensure + logger.trace("RefreshChannelsJob: #{id} fiber : Done") + active_channel.send(true) end - - active_channel.send(true) end end end + logger.debug("RefreshChannelsJob: Done, sleeping for one minute") sleep 1.minute Fiber.yield end diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr index eebdf0f3..208569b8 100644 --- a/src/invidious/jobs/refresh_feeds_job.cr +++ b/src/invidious/jobs/refresh_feeds_job.cr @@ -7,8 +7,8 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob end def begin - max_threads = config.feed_threads - active_threads = 0 + max_fibers = config.feed_threads + active_fibers = 0 active_channel = Channel(Bool).new loop do @@ -17,27 +17,27 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob email = rs.read(String) view_name = "subscriptions_#{sha256(email)}" - if active_threads >= max_threads + if active_fibers >= max_fibers if active_channel.receive - active_threads -= 1 + active_fibers -= 1 end end - active_threads += 1 + active_fibers += 1 spawn do begin # Drop outdated views column_array = get_column_array(db, view_name) ChannelVideo.type_array.each_with_index do |name, i| if name != column_array[i]? - logger.puts("DROP MATERIALIZED VIEW #{view_name}") + logger.info("RefreshFeedsJob: DROP MATERIALIZED VIEW #{view_name}") db.exec("DROP MATERIALIZED VIEW #{view_name}") raise "view does not exist" end end if !db.query_one("SELECT pg_get_viewdef('#{view_name}')", as: String).includes? "WHERE ((cv.ucid = ANY (u.subscriptions))" - logger.puts("Materialized view #{view_name} is out-of-date, recreating...") + logger.info("RefreshFeedsJob: Materialized view #{view_name} is out-of-date, recreating...") db.exec("DROP MATERIALIZED VIEW #{view_name}") end @@ -49,18 +49,18 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob legacy_view_name = "subscriptions_#{sha256(email)[0..7]}" db.exec("SELECT * FROM #{legacy_view_name} LIMIT 0") - logger.puts("RENAME MATERIALIZED VIEW #{legacy_view_name}") + logger.info("RefreshFeedsJob: RENAME MATERIALIZED VIEW #{legacy_view_name}") db.exec("ALTER MATERIALIZED VIEW #{legacy_view_name} RENAME TO #{view_name}") rescue ex begin # While iterating through, we may have an email stored from a deleted account if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool) - logger.puts("CREATE #{view_name}") + logger.info("RefreshFeedsJob: CREATE #{view_name}") db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(email)}") db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email) end rescue ex - logger.puts("REFRESH #{email} : #{ex.message}") + logger.error("RefreshFeedJobs: REFRESH #{email} : #{ex.message}") end end end diff --git a/src/invidious/jobs/subscribe_to_feeds_job.cr b/src/invidious/jobs/subscribe_to_feeds_job.cr index 3d3b2218..2255730d 100644 --- a/src/invidious/jobs/subscribe_to_feeds_job.cr +++ b/src/invidious/jobs/subscribe_to_feeds_job.cr @@ -8,12 +8,12 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob end def begin - max_threads = 1 + max_fibers = 1 if config.use_pubsub_feeds.is_a?(Int32) - max_threads = config.use_pubsub_feeds.as(Int32) + max_fibers = config.use_pubsub_feeds.as(Int32) end - active_threads = 0 + active_fibers = 0 active_channel = Channel(Bool).new loop do @@ -21,23 +21,23 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob rs.each do ucid = rs.read(String) - if active_threads >= max_threads.as(Int32) + if active_fibers >= max_fibers.as(Int32) if active_channel.receive - active_threads -= 1 + active_fibers -= 1 end end - active_threads += 1 + active_fibers += 1 spawn do begin response = subscribe_pubsub(ucid, hmac_key, config) if response.status_code >= 400 - logger.puts("#{ucid} : #{response.body}") + logger.error("SubscribeToFeedsJob: #{ucid} : #{response.body}") end rescue ex - logger.puts("#{ucid} : #{ex.message}") + logger.error("SubscribeToFeedsJob: #{ucid} : #{ex.message}") end active_channel.send(true) diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr index c69eb0c4..55b01174 100644 --- a/src/invidious/mixes.cr +++ b/src/invidious/mixes.cr @@ -30,7 +30,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil) initial_data = extract_initial_data(response.body) if !initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]? - raise translate(locale, "Could not create mix.") + raise InfoException.new("Could not create mix.") end playlist = initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]["playlist"] diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index c984a12a..d5b41caa 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -220,6 +220,11 @@ struct InvidiousPlaylist json.field "videos" do json.array do + if !offset || offset == 0 + index = PG_DB.query_one?("SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1", self.id, continuation, as: Int64) + offset = self.index.index(index) || 0 + end + videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation) videos.each_with_index do |video, index| video.to_json(locale, json, offset + index) @@ -338,7 +343,7 @@ def get_playlist(db, plid, locale, refresh = true, force_refresh = false) if playlist = db.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) return playlist else - raise "Playlist does not exist." + raise InfoException.new("Playlist does not exist.") end else return fetch_playlist(plid, locale) @@ -353,16 +358,16 @@ def fetch_playlist(plid, locale) response = YT_POOL.client &.get("/playlist?list=#{plid}&hl=en") if response.status_code != 200 if response.headers["location"]?.try &.includes? "/sorry/index" - raise "Could not extract playlist info. Instance is likely blocked." + raise InfoException.new("Could not extract playlist info. Instance is likely blocked.") else - raise translate(locale, "Not a playlist.") + raise InfoException.new("Not a playlist.") end end initial_data = extract_initial_data(response.body) playlist_info = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]?.try &.[0]["playlistSidebarPrimaryInfoRenderer"]? - raise "Could not extract playlist info" if !playlist_info + raise InfoException.new("Could not extract playlist info") if !playlist_info title = playlist_info["title"]?.try &.["runs"][0]?.try &.["text"]?.try &.as_s || "" desc_item = playlist_info["description"]? @@ -390,7 +395,7 @@ def fetch_playlist(plid, locale) author_info = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]?.try &.[1]["playlistSidebarSecondaryInfoRenderer"]? .try &.["videoOwner"]["videoOwnerRenderer"]? - raise "Could not extract author info" if !author_info + raise InfoException.new("Could not extract author info") if !author_info author_thumbnail = author_info["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s || "" author = author_info["title"]["runs"][0]["text"]?.try &.as_s || "" @@ -412,11 +417,6 @@ end def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil) if playlist.is_a? InvidiousPlaylist - if !offset - index = PG_DB.query_one?("SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1", playlist.id, continuation, as: Int64) - offset = playlist.index.index(index) || 0 - end - db.query_all("SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3", playlist.id, playlist.index, offset, as: PlaylistVideo) else fetch_playlist_videos(playlist.id, playlist.video_count, offset, locale, continuation) diff --git a/src/invidious/routes/base_route.cr b/src/invidious/routes/base_route.cr index c6e6667e..2852cb04 100644 --- a/src/invidious/routes/base_route.cr +++ b/src/invidious/routes/base_route.cr @@ -4,6 +4,4 @@ abstract class Invidious::Routes::BaseRoute def initialize(@config, @logger) end - - abstract def handle(env) end diff --git a/src/invidious/routes/embed/index.cr b/src/invidious/routes/embed/index.cr index 79c91d86..32a4966b 100644 --- a/src/invidious/routes/embed/index.cr +++ b/src/invidious/routes/embed/index.cr @@ -8,9 +8,7 @@ class Invidious::Routes::Embed::Index < Invidious::Routes::BaseRoute offset = env.params.query["index"]?.try &.to_i? || 0 videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale) rescue ex - error_message = ex.message - env.response.status_code = 500 - return templated "error" + return error_template(500, ex) end url = "/embed/#{videos[0].id}?#{env.params.query}" diff --git a/src/invidious/routes/embed/show.cr b/src/invidious/routes/embed/show.cr index 23c2b86f..8a655556 100644 --- a/src/invidious/routes/embed/show.cr +++ b/src/invidious/routes/embed/show.cr @@ -38,9 +38,7 @@ class Invidious::Routes::Embed::Show < Invidious::Routes::BaseRoute offset = env.params.query["index"]?.try &.to_i? || 0 videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale) rescue ex - error_message = ex.message - env.response.status_code = 500 - return templated "error" + return error_template(500, ex) end url = "/embed/#{videos[0].id}" @@ -63,8 +61,7 @@ class Invidious::Routes::Embed::Show < Invidious::Routes::BaseRoute env.params.query.delete_all("channel") if !video_id || video_id == "live_stream" - error_message = "Video is unavailable." - return templated "error" + return error_template(500, "Video is unavailable.") end url = "/embed/#{video_id}" @@ -100,9 +97,7 @@ class Invidious::Routes::Embed::Show < Invidious::Routes::BaseRoute rescue ex : VideoRedirect return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex - error_message = ex.message - env.response.status_code = 500 - return templated "error" + return error_template(500, ex) end if preferences.annotations_subscribed && diff --git a/src/invidious/routes/home.cr b/src/invidious/routes/home.cr index 9b1bf61b..486a7344 100644 --- a/src/invidious/routes/home.cr +++ b/src/invidious/routes/home.cr @@ -5,30 +5,24 @@ class Invidious::Routes::Home < Invidious::Routes::BaseRoute user = env.get? "user" case preferences.default_home - when "" - templated "empty" when "Popular" - templated "popular" + env.redirect "/feed/popular" when "Trending" env.redirect "/feed/trending" when "Subscriptions" if user env.redirect "/feed/subscriptions" else - templated "popular" + env.redirect "/feed/popular" end when "Playlists" if user env.redirect "/view_all_playlists" else - templated "popular" + env.redirect "/feed/popular" end else templated "empty" end end - - private def popular_videos - Jobs::PullPopularVideosJob::POPULAR_VIDEOS.get - end end diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr new file mode 100644 index 00000000..42fb4676 --- /dev/null +++ b/src/invidious/routes/login.cr @@ -0,0 +1,508 @@ +class Invidious::Routes::Login < Invidious::Routes::BaseRoute + def login_page(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + + return env.redirect "/feed/subscriptions" if user + + if !config.login_enabled + return error_template(400, "Login has been disabled by administrator.") + end + + referer = get_referer(env, "/feed/subscriptions") + + email = nil + password = nil + captcha = nil + + account_type = env.params.query["type"]? + account_type ||= "invidious" + + captcha_type = env.params.query["captcha"]? + captcha_type ||= "image" + + tfa = env.params.query["tfa"]? + prompt = nil + + templated "login" + end + + def login(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + referer = get_referer(env, "/feed/subscriptions") + + if !config.login_enabled + return error_template(403, "Login has been disabled by administrator.") + end + + # https://stackoverflow.com/a/574698 + email = env.params.body["email"]?.try &.downcase.byte_slice(0, 254) + password = env.params.body["password"]? + + account_type = env.params.query["type"]? + account_type ||= "invidious" + + case account_type + when "google" + tfa_code = env.params.body["tfa"]?.try &.lchop("G-") + traceback = IO::Memory.new + + # See https://github.com/ytdl-org/youtube-dl/blob/2019.04.07/youtube_dl/extractor/youtube.py#L82 + begin + client = QUIC::Client.new(LOGIN_URL) + headers = HTTP::Headers.new + + login_page = client.get("/ServiceLogin") + headers = login_page.cookies.add_request_headers(headers) + + lookup_req = { + email, nil, [] of String, nil, "US", nil, nil, 2, false, true, + {nil, nil, + {2, 1, nil, 1, + "https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn", + nil, [] of String, 4}, + 1, + {nil, nil, [] of String}, + nil, nil, nil, true, + }, + email, + }.to_json + + traceback << "Getting lookup..." + + headers["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8" + headers["Google-Accounts-XSRF"] = "1" + + response = client.post("/_/signin/sl/lookup", headers, login_req(lookup_req)) + lookup_results = JSON.parse(response.body[5..-1]) + + traceback << "done, returned #{response.status_code}.<br/>" + + user_hash = lookup_results[0][2] + + if token = env.params.body["token"]? + answer = env.params.body["answer"]? + captcha = {token, answer} + else + captcha = nil + end + + challenge_req = { + user_hash, nil, 1, nil, + {1, nil, nil, nil, + {password, captcha, true}, + }, + {nil, nil, + {2, 1, nil, 1, + "https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn", + nil, [] of String, 4}, + 1, + {nil, nil, [] of String}, + nil, nil, nil, true, + }, + }.to_json + + traceback << "Getting challenge..." + + response = client.post("/_/signin/sl/challenge", headers, login_req(challenge_req)) + headers = response.cookies.add_request_headers(headers) + challenge_results = JSON.parse(response.body[5..-1]) + + traceback << "done, returned #{response.status_code}.<br/>" + + headers["Cookie"] = URI.decode_www_form(headers["Cookie"]) + + if challenge_results[0][3]?.try &.== 7 + return error_template(423, "Account has temporarily been disabled") + end + + if token = challenge_results[0][-1]?.try &.[-1]?.try &.as_h?.try &.["5001"]?.try &.[-1].as_a?.try &.[-1].as_s + account_type = "google" + captcha_type = "image" + prompt = nil + tfa = tfa_code + captcha = {tokens: [token], question: ""} + + return templated "login" + end + + if challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED" + return error_template(401, "Incorrect password") + end + + prompt_type = challenge_results[0][-1]?.try &.[0].as_a?.try &.[0][2]? + if {"TWO_STEP_VERIFICATION", "LOGIN_CHALLENGE"}.includes? prompt_type + traceback << "Handling prompt #{prompt_type}.<br/>" + case prompt_type + when "TWO_STEP_VERIFICATION" + prompt_type = 2 + else # "LOGIN_CHALLENGE" + prompt_type = 4 + end + + # Prefer Authenticator app and SMS over unsupported protocols + if !{6, 9, 12, 15}.includes?(challenge_results[0][-1][0][0][8].as_i) && prompt_type == 2 + tfa = challenge_results[0][-1][0].as_a.select { |auth_type| {6, 9, 12, 15}.includes? auth_type[8] }[0] + + traceback << "Selecting challenge #{tfa[8]}..." + select_challenge = {prompt_type, nil, nil, nil, {tfa[8]}}.to_json + + tl = challenge_results[1][2] + + tfa = client.post("/_/signin/selectchallenge?TL=#{tl}", headers, login_req(select_challenge)).body + tfa = tfa[5..-1] + tfa = JSON.parse(tfa)[0][-1] + + traceback << "done.<br/>" + else + traceback << "Using challenge #{challenge_results[0][-1][0][0][8]}.<br/>" + tfa = challenge_results[0][-1][0][0] + end + + if tfa[5] == "QUOTA_EXCEEDED" + return error_template(423, "Quota exceeded, try again in a few hours") + end + + if !tfa_code + account_type = "google" + captcha_type = "image" + + case tfa[8] + when 6, 9 + prompt = "Google verification code" + when 12 + prompt = "Login verification, recovery email: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}" + when 15 + prompt = "Login verification, security question: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}" + else + prompt = "Google verification code" + end + + tfa = nil + captcha = nil + return templated "login" + end + + tl = challenge_results[1][2] + + request_type = tfa[8] + case request_type + when 6 # Authenticator app + tfa_req = { + user_hash, nil, 2, nil, + {6, nil, nil, nil, nil, + {tfa_code, false}, + }, + }.to_json + when 9 # Voice or text message + tfa_req = { + user_hash, nil, 2, nil, + {9, nil, nil, nil, nil, nil, nil, nil, + {nil, tfa_code, false, 2}, + }, + }.to_json + when 12 # Recovery email + tfa_req = { + user_hash, nil, 4, nil, + {12, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + {tfa_code}, + }, + }.to_json + when 15 # Security question + tfa_req = { + user_hash, nil, 5, nil, + {15, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + {tfa_code}, + }, + }.to_json + else + return error_template(500, "Unable to log in, make sure two-factor authentication (Authenticator or SMS) is turned on.") + end + + traceback << "Submitting challenge..." + + response = client.post("/_/signin/challenge?hl=en&TL=#{tl}", headers, login_req(tfa_req)) + headers = response.cookies.add_request_headers(headers) + challenge_results = JSON.parse(response.body[5..-1]) + + if (challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED") || + (challenge_results[0][-1]?.try &.[5] == "INVALID_INPUT") + return error_template(401, "Invalid TFA code") + end + + traceback << "done.<br/>" + end + + traceback << "Logging in..." + + location = URI.parse(challenge_results[0][-1][2].to_s) + cookies = HTTP::Cookies.from_headers(headers) + + headers.delete("Content-Type") + headers.delete("Google-Accounts-XSRF") + + loop do + if !location || location.path == "/ManageAccount" + break + end + + # Occasionally there will be a second page after login confirming + # the user's phone number ("/b/0/SmsAuthInterstitial"), which we currently don't handle. + + if location.path.starts_with? "/b/0/SmsAuthInterstitial" + traceback << "Unhandled dialog /b/0/SmsAuthInterstitial." + end + + login = client.get(location.full_path, headers) + + headers = login.cookies.add_request_headers(headers) + location = login.headers["Location"]?.try { |u| URI.parse(u) } + end + + cookies = HTTP::Cookies.from_headers(headers) + sid = cookies["SID"]?.try &.value + if !sid + raise "Couldn't get SID." + end + + user, sid = get_user(sid, headers, PG_DB, logger) + + # We are now logged in + traceback << "done.<br/>" + + host = URI.parse(env.request.headers["Host"]).host + + if Kemal.config.ssl || config.https_only + secure = true + else + secure = false + end + + cookies.each do |cookie| + if Kemal.config.ssl || config.https_only + cookie.secure = secure + else + cookie.secure = secure + end + + if cookie.extension + cookie.extension = cookie.extension.not_nil!.gsub(".youtube.com", host) + cookie.extension = cookie.extension.not_nil!.gsub("Secure; ", "") + end + env.response.cookies << cookie + end + + if env.request.cookies["PREFS"]? + preferences = env.get("preferences").as(Preferences) + PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) + + cookie = env.request.cookies["PREFS"] + cookie.expires = Time.utc(1990, 1, 1) + env.response.cookies << cookie + end + + env.redirect referer + rescue ex + traceback.rewind + # error_message = translate(locale, "Login failed. This may be because two-factor authentication is not turned on for your account.") + error_message = %(#{ex.message}<br/>Traceback:<br/><div style="padding-left:2em" id="traceback">#{traceback.gets_to_end}</div>) + return error_template(500, error_message) + end + when "invidious" + if !email + return error_template(401, "User ID is a required field") + end + + if !password + return error_template(401, "Password is a required field") + end + + user = PG_DB.query_one?("SELECT * FROM users WHERE email = $1", email, as: User) + + if user + if !user.password + return error_template(400, "Please sign in using 'Log in with Google'") + end + + if Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55)) + sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) + PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) + + if Kemal.config.ssl || config.https_only + secure = true + else + secure = false + end + + if config.domain + env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{config.domain}", value: sid, expires: Time.utc + 2.years, + secure: secure, http_only: true) + else + env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.utc + 2.years, + secure: secure, http_only: true) + end + else + return error_template(401, "Wrong username or password") + end + + # Since this user has already registered, we don't want to overwrite their preferences + if env.request.cookies["PREFS"]? + cookie = env.request.cookies["PREFS"] + cookie.expires = Time.utc(1990, 1, 1) + env.response.cookies << cookie + end + else + if !config.registration_enabled + return error_template(400, "Registration has been disabled by administrator.") + end + + if password.empty? + return error_template(401, "Password cannot be empty") + end + + # See https://security.stackexchange.com/a/39851 + if password.bytesize > 55 + return error_template(400, "Password cannot be longer than 55 characters") + end + + password = password.byte_slice(0, 55) + + if config.captcha_enabled + captcha_type = env.params.body["captcha_type"]? + answer = env.params.body["answer"]? + change_type = env.params.body["change_type"]? + + if !captcha_type || change_type + if change_type + captcha_type = change_type + end + captcha_type ||= "image" + + account_type = "invidious" + tfa = false + prompt = "" + + if captcha_type == "image" + captcha = generate_captcha(HMAC_KEY, PG_DB) + else + captcha = generate_text_captcha(HMAC_KEY, PG_DB) + end + + return templated "login" + end + + tokens = env.params.body.select { |k, v| k.match(/^token\[\d+\]$/) }.map { |k, v| v } + + answer ||= "" + captcha_type ||= "image" + + case captcha_type + when "image" + answer = answer.lstrip('0') + answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer) + + begin + validate_request(tokens[0], answer, env.request, HMAC_KEY, PG_DB, locale) + rescue ex + return error_template(400, ex) + end + else # "text" + answer = Digest::MD5.hexdigest(answer.downcase.strip) + + if tokens.empty? + return error_template(500, "Erroneous CAPTCHA") + end + + found_valid_captcha = false + error_exception = Exception.new + tokens.each_with_index do |token, i| + begin + validate_request(token, answer, env.request, HMAC_KEY, PG_DB, locale) + found_valid_captcha = true + rescue ex + error_exception = ex + end + end + + if !found_valid_captcha + return error_template(500, error_exception) + end + end + end + + sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) + user, sid = create_user(sid, email, password) + user_array = user.to_a + user_array[4] = user_array[4].to_json # User preferences + + args = arg_array(user_array) + + PG_DB.exec("INSERT INTO users VALUES (#{args})", args: user_array) + PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) + + view_name = "subscriptions_#{sha256(user.email)}" + PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") + + if Kemal.config.ssl || config.https_only + secure = true + else + secure = false + end + + if config.domain + env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{config.domain}", value: sid, expires: Time.utc + 2.years, + secure: secure, http_only: true) + else + env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.utc + 2.years, + secure: secure, http_only: true) + end + + if env.request.cookies["PREFS"]? + preferences = env.get("preferences").as(Preferences) + PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) + + cookie = env.request.cookies["PREFS"] + cookie.expires = Time.utc(1990, 1, 1) + env.response.cookies << cookie + end + end + + env.redirect referer + else + env.redirect referer + end + end + + def signout(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + begin + validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + rescue ex + return error_template(400, ex) + end + + PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", sid) + + env.request.cookies.each do |cookie| + cookie.expires = Time.utc(1990, 1, 1) + env.response.cookies << cookie + end + + env.redirect referer + end +end diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr new file mode 100644 index 00000000..6c899054 --- /dev/null +++ b/src/invidious/routes/playlists.cr @@ -0,0 +1,472 @@ +class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute + def index(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + referer = get_referer(env) + + return env.redirect "/" if user.nil? + + user = user.as(User) + + items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + items_created.map! do |item| + item.author = "" + item + end + + items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + items_saved.map! do |item| + item.author = "" + item + end + + templated "view_all_playlists" + end + + def new(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + return env.redirect "/" if user.nil? + + user = user.as(User) + sid = sid.as(String) + csrf_token = generate_response(sid, {":create_playlist"}, HMAC_KEY, PG_DB) + + templated "create_playlist" + end + + def create(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + return env.redirect "/" if user.nil? + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + begin + validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + rescue ex + return error_template(400, ex) + end + + title = env.params.body["title"]?.try &.as(String) + if !title || title.empty? + return error_template(400, "Title cannot be empty.") + end + + privacy = PlaylistPrivacy.parse?(env.params.body["privacy"]?.try &.as(String) || "") + if !privacy + return error_template(400, "Invalid privacy setting.") + end + + if PG_DB.query_one("SELECT count(*) FROM playlists WHERE author = $1", user.email, as: Int64) >= 100 + return error_template(400, "User cannot have more than 100 playlists.") + end + + playlist = create_playlist(PG_DB, title, privacy, user) + + env.redirect "/playlist?list=#{playlist.id}" + end + + def subscribe(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + referer = get_referer(env) + + return env.redirect "/" if user.nil? + + user = user.as(User) + + playlist_id = env.params.query["list"] + playlist = get_playlist(PG_DB, playlist_id, locale) + subscribe_playlist(PG_DB, user, playlist) + + env.redirect "/playlist?list=#{playlist.id}" + end + + def delete_page(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + return env.redirect "/" if user.nil? + + user = user.as(User) + sid = sid.as(String) + + plid = env.params.query["list"]? + playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if !playlist || playlist.author != user.email + return env.redirect referer + end + + csrf_token = generate_response(sid, {":delete_playlist"}, HMAC_KEY, PG_DB) + + templated "delete_playlist" + end + + def delete(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + return env.redirect "/" if user.nil? + + plid = env.params.query["list"]? + return env.redirect referer if plid.nil? + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + begin + validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + rescue ex + return error_template(400, ex) + end + + playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if !playlist || playlist.author != user.email + return env.redirect referer + end + + PG_DB.exec("DELETE FROM playlist_videos * WHERE plid = $1", plid) + PG_DB.exec("DELETE FROM playlists * WHERE id = $1", plid) + + env.redirect "/view_all_playlists" + end + + def edit(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + return env.redirect "/" if user.nil? + + user = user.as(User) + sid = sid.as(String) + + plid = env.params.query["list"]? + if !plid || !plid.starts_with?("IV") + return env.redirect referer + end + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + begin + playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if !playlist || playlist.author != user.email + return env.redirect referer + end + rescue ex + return env.redirect referer + end + + begin + videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale) + rescue ex + videos = [] of PlaylistVideo + end + + csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY, PG_DB) + + templated "edit_playlist" + end + + def update(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + return env.redirect "/" if user.nil? + + plid = env.params.query["list"]? + return env.redirect referer if plid.nil? + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + begin + validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + rescue ex + return error_template(400, ex) + end + + playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if !playlist || playlist.author != user.email + return env.redirect referer + end + + title = env.params.body["title"]?.try &.delete("<>") || "" + privacy = PlaylistPrivacy.parse(env.params.body["privacy"]? || "Public") + description = env.params.body["description"]?.try &.delete("\r") || "" + + if title != playlist.title || + privacy != playlist.privacy || + description != playlist.description + updated = Time.utc + else + updated = playlist.updated + end + + PG_DB.exec("UPDATE playlists SET title = $1, privacy = $2, description = $3, updated = $4 WHERE id = $5", title, privacy, description, updated, plid) + + env.redirect "/playlist?list=#{plid}" + end + + def add_playlist_items_page(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + return env.redirect "/" if user.nil? + + user = user.as(User) + sid = sid.as(String) + + plid = env.params.query["list"]? + if !plid || !plid.starts_with?("IV") + return env.redirect referer + end + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + begin + playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if !playlist || playlist.author != user.email + return env.redirect referer + end + rescue ex + return env.redirect referer + end + + query = env.params.query["q"]? + if query + begin + search_query, count, items = process_search_query(query, page, user, region: nil) + videos = items.select { |item| item.is_a? SearchVideo }.map { |item| item.as(SearchVideo) } + rescue ex + videos = [] of SearchVideo + count = 0 + end + else + videos = [] of SearchVideo + count = 0 + end + + env.set "add_playlist_items", plid + templated "add_playlist_items" + end + + def playlist_ajax(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env, "/") + + redirect = env.params.query["redirect"]? + redirect ||= "true" + redirect = redirect == "true" + + if !user + if redirect + return env.redirect referer + else + return error_json(403, "No such user") + end + end + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + begin + validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + rescue ex + if redirect + return error_template(400, ex) + else + return error_json(400, ex) + end + end + + if env.params.query["action_create_playlist"]? + action = "action_create_playlist" + elsif env.params.query["action_delete_playlist"]? + action = "action_delete_playlist" + elsif env.params.query["action_edit_playlist"]? + action = "action_edit_playlist" + elsif env.params.query["action_add_video"]? + action = "action_add_video" + video_id = env.params.query["video_id"] + elsif env.params.query["action_remove_video"]? + action = "action_remove_video" + elsif env.params.query["action_move_video_before"]? + action = "action_move_video_before" + else + return env.redirect referer + end + + begin + playlist_id = env.params.query["playlist_id"] + playlist = get_playlist(PG_DB, playlist_id, locale).as(InvidiousPlaylist) + raise "Invalid user" if playlist.author != user.email + rescue ex + if redirect + return error_template(400, ex) + else + return error_json(400, ex) + end + end + + if !user.password + # TODO: Playlist stub, sync with YouTube for Google accounts + # playlist_ajax(playlist_id, action, env.request.headers) + end + email = user.email + + case action + when "action_edit_playlist" + # TODO: Playlist stub + when "action_add_video" + if playlist.index.size >= 500 + if redirect + return error_template(400, "Playlist cannot have more than 500 videos") + else + return error_json(400, "Playlist cannot have more than 500 videos") + end + end + + video_id = env.params.query["video_id"] + + begin + video = get_video(video_id, PG_DB) + rescue ex + if redirect + return error_template(500, ex) + else + return error_json(500, ex) + end + end + + playlist_video = PlaylistVideo.new({ + title: video.title, + id: video.id, + author: video.author, + ucid: video.ucid, + length_seconds: video.length_seconds, + published: video.published, + plid: playlist_id, + live_now: video.live_now, + index: Random::Secure.rand(0_i64..Int64::MAX), + }) + + video_array = playlist_video.to_a + args = arg_array(video_array) + + PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array) + PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, playlist_id) + when "action_remove_video" + index = env.params.query["set_video_id"] + PG_DB.exec("DELETE FROM playlist_videos * WHERE index = $1", index) + PG_DB.exec("UPDATE playlists SET index = array_remove(index, $1), video_count = cardinality(index) - 1, updated = $2 WHERE id = $3", index, Time.utc, playlist_id) + when "action_move_video_before" + # TODO: Playlist stub + else + return error_json(400, "Unsupported action #{action}") + end + + if redirect + env.redirect referer + else + env.response.content_type = "application/json" + "{}" + end + end + + def show(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get?("user").try &.as(User) + referer = get_referer(env) + + plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") + if !plid + return env.redirect "/" + end + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + if plid.starts_with? "RD" + return env.redirect "/mix?list=#{plid}" + end + + begin + playlist = get_playlist(PG_DB, plid, locale) + rescue ex + return error_template(500, ex) + end + + if playlist.privacy == PlaylistPrivacy::Private && playlist.author != user.try &.email + return error_template(403, "This playlist is private.") + end + + begin + videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale) + rescue ex + videos = [] of PlaylistVideo + end + + if playlist.author == user.try &.email + env.set "remove_playlist_items", plid + end + + templated "playlist" + end + + def mix(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + rdid = env.params.query["list"]? + if !rdid + return env.redirect "/" + end + + continuation = env.params.query["continuation"]? + continuation ||= rdid.lchop("RD") + + begin + mix = fetch_mix(rdid, continuation, locale: locale) + rescue ex + return error_template(500, ex) + end + + templated "mix" + end +end diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr new file mode 100644 index 00000000..48446161 --- /dev/null +++ b/src/invidious/routes/search.cr @@ -0,0 +1,59 @@ +class Invidious::Routes::Search < Invidious::Routes::BaseRoute + def opensearch(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + env.response.content_type = "application/opensearchdescription+xml" + + XML.build(indent: " ", encoding: "UTF-8") do |xml| + xml.element("OpenSearchDescription", xmlns: "http://a9.com/-/spec/opensearch/1.1/") do + xml.element("ShortName") { xml.text "Invidious" } + xml.element("LongName") { xml.text "Invidious Search" } + xml.element("Description") { xml.text "Search for videos, channels, and playlists on Invidious" } + xml.element("InputEncoding") { xml.text "UTF-8" } + xml.element("Image", width: 48, height: 48, type: "image/x-icon") { xml.text "#{HOST_URL}/favicon.ico" } + xml.element("Url", type: "text/html", method: "get", template: "#{HOST_URL}/search?q={searchTerms}") + end + end + end + + def results(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + query = env.params.query["search_query"]? + query ||= env.params.query["q"]? + query ||= "" + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + if query + env.redirect "/search?q=#{URI.encode_www_form(query)}&page=#{page}" + else + env.redirect "/" + end + end + + def search(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + region = env.params.query["region"]? + + query = env.params.query["search_query"]? + query ||= env.params.query["q"]? + query ||= "" + + return env.redirect "/" if query.empty? + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + user = env.get? "user" + + begin + search_query, count, videos = process_search_query(query, page, user, region: nil) + rescue ex + return error_template(500, ex) + end + + env.set "search", query + templated "search" + end +end diff --git a/src/invidious/routes/user_preferences.cr b/src/invidious/routes/user_preferences.cr new file mode 100644 index 00000000..7f334115 --- /dev/null +++ b/src/invidious/routes/user_preferences.cr @@ -0,0 +1,259 @@ +class Invidious::Routes::UserPreferences < Invidious::Routes::BaseRoute + def show(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + referer = get_referer(env) + + preferences = env.get("preferences").as(Preferences) + + templated "preferences" + end + + def update(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + referer = get_referer(env) + + video_loop = env.params.body["video_loop"]?.try &.as(String) + video_loop ||= "off" + video_loop = video_loop == "on" + + annotations = env.params.body["annotations"]?.try &.as(String) + annotations ||= "off" + annotations = annotations == "on" + + annotations_subscribed = env.params.body["annotations_subscribed"]?.try &.as(String) + annotations_subscribed ||= "off" + annotations_subscribed = annotations_subscribed == "on" + + autoplay = env.params.body["autoplay"]?.try &.as(String) + autoplay ||= "off" + autoplay = autoplay == "on" + + continue = env.params.body["continue"]?.try &.as(String) + continue ||= "off" + continue = continue == "on" + + continue_autoplay = env.params.body["continue_autoplay"]?.try &.as(String) + continue_autoplay ||= "off" + continue_autoplay = continue_autoplay == "on" + + listen = env.params.body["listen"]?.try &.as(String) + listen ||= "off" + listen = listen == "on" + + local = env.params.body["local"]?.try &.as(String) + local ||= "off" + local = local == "on" + + speed = env.params.body["speed"]?.try &.as(String).to_f32? + speed ||= CONFIG.default_user_preferences.speed + + player_style = env.params.body["player_style"]?.try &.as(String) + player_style ||= CONFIG.default_user_preferences.player_style + + quality = env.params.body["quality"]?.try &.as(String) + quality ||= CONFIG.default_user_preferences.quality + + quality_dash = env.params.body["quality_dash"]?.try &.as(String) + quality_dash ||= CONFIG.default_user_preferences.quality_dash + + volume = env.params.body["volume"]?.try &.as(String).to_i? + volume ||= CONFIG.default_user_preferences.volume + + comments = [] of String + 2.times do |i| + comments << (env.params.body["comments[#{i}]"]?.try &.as(String) || CONFIG.default_user_preferences.comments[i]) + end + + captions = [] of String + 3.times do |i| + captions << (env.params.body["captions[#{i}]"]?.try &.as(String) || CONFIG.default_user_preferences.captions[i]) + end + + related_videos = env.params.body["related_videos"]?.try &.as(String) + related_videos ||= "off" + related_videos = related_videos == "on" + + default_home = env.params.body["default_home"]?.try &.as(String) || CONFIG.default_user_preferences.default_home + + feed_menu = [] of String + 4.times do |index| + option = env.params.body["feed_menu[#{index}]"]?.try &.as(String) || "" + if !option.empty? + feed_menu << option + end + end + + locale = env.params.body["locale"]?.try &.as(String) + locale ||= CONFIG.default_user_preferences.locale + + dark_mode = env.params.body["dark_mode"]?.try &.as(String) + dark_mode ||= CONFIG.default_user_preferences.dark_mode + + thin_mode = env.params.body["thin_mode"]?.try &.as(String) + thin_mode ||= "off" + thin_mode = thin_mode == "on" + + max_results = env.params.body["max_results"]?.try &.as(String).to_i? + max_results ||= CONFIG.default_user_preferences.max_results + + sort = env.params.body["sort"]?.try &.as(String) + sort ||= CONFIG.default_user_preferences.sort + + latest_only = env.params.body["latest_only"]?.try &.as(String) + latest_only ||= "off" + latest_only = latest_only == "on" + + unseen_only = env.params.body["unseen_only"]?.try &.as(String) + unseen_only ||= "off" + unseen_only = unseen_only == "on" + + notifications_only = env.params.body["notifications_only"]?.try &.as(String) + notifications_only ||= "off" + notifications_only = notifications_only == "on" + + # Convert to JSON and back again to take advantage of converters used for compatability + preferences = Preferences.from_json({ + annotations: annotations, + annotations_subscribed: annotations_subscribed, + autoplay: autoplay, + captions: captions, + comments: comments, + continue: continue, + continue_autoplay: continue_autoplay, + dark_mode: dark_mode, + latest_only: latest_only, + listen: listen, + local: local, + locale: locale, + max_results: max_results, + notifications_only: notifications_only, + player_style: player_style, + quality: quality, + quality_dash: quality_dash, + default_home: default_home, + feed_menu: feed_menu, + related_videos: related_videos, + sort: sort, + speed: speed, + thin_mode: thin_mode, + unseen_only: unseen_only, + video_loop: video_loop, + volume: volume, + }.to_json).to_json + + if user = env.get? "user" + user = user.as(User) + PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences, user.email) + + if config.admins.includes? user.email + config.default_user_preferences.default_home = env.params.body["admin_default_home"]?.try &.as(String) || config.default_user_preferences.default_home + + admin_feed_menu = [] of String + 4.times do |index| + option = env.params.body["admin_feed_menu[#{index}]"]?.try &.as(String) || "" + if !option.empty? + admin_feed_menu << option + end + end + config.default_user_preferences.feed_menu = admin_feed_menu + + popular_enabled = env.params.body["popular_enabled"]?.try &.as(String) + popular_enabled ||= "off" + config.popular_enabled = popular_enabled == "on" + + captcha_enabled = env.params.body["captcha_enabled"]?.try &.as(String) + captcha_enabled ||= "off" + config.captcha_enabled = captcha_enabled == "on" + + login_enabled = env.params.body["login_enabled"]?.try &.as(String) + login_enabled ||= "off" + config.login_enabled = login_enabled == "on" + + registration_enabled = env.params.body["registration_enabled"]?.try &.as(String) + registration_enabled ||= "off" + config.registration_enabled = registration_enabled == "on" + + statistics_enabled = env.params.body["statistics_enabled"]?.try &.as(String) + statistics_enabled ||= "off" + config.statistics_enabled = statistics_enabled == "on" + + CONFIG.default_user_preferences = config.default_user_preferences + File.write("config/config.yml", config.to_yaml) + end + else + if Kemal.config.ssl || config.https_only + secure = true + else + secure = false + end + + if config.domain + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{config.domain}", value: preferences, expires: Time.utc + 2.years, + secure: secure, http_only: true) + else + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: preferences, expires: Time.utc + 2.years, + secure: secure, http_only: true) + end + end + + env.redirect referer + end + + def toggle_theme(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + referer = get_referer(env, unroll: false) + + redirect = env.params.query["redirect"]? + redirect ||= "true" + redirect = redirect == "true" + + if user = env.get? "user" + user = user.as(User) + preferences = user.preferences + + case preferences.dark_mode + when "dark" + preferences.dark_mode = "light" + else + preferences.dark_mode = "dark" + end + + preferences = preferences.to_json + + PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences, user.email) + else + preferences = env.get("preferences").as(Preferences) + + case preferences.dark_mode + when "dark" + preferences.dark_mode = "light" + else + preferences.dark_mode = "dark" + end + + preferences = preferences.to_json + + if Kemal.config.ssl || config.https_only + secure = true + else + secure = false + end + + if config.domain + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{config.domain}", value: preferences, expires: Time.utc + 2.years, + secure: secure, http_only: true) + else + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: preferences, expires: Time.utc + 2.years, + secure: secure, http_only: true) + end + end + + if redirect + env.redirect referer + else + env.response.content_type = "application/json" + "{}" + end + end +end diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 4eee7793..a5c05c00 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -12,9 +12,7 @@ class Invidious::Routes::Watch < Invidious::Routes::BaseRoute id = env.params.query["v"] if env.params.query["v"].empty? - error_message = "Invalid parameters." - env.response.status_code = 400 - return templated "error" + return error_template(400, "Invalid parameters.") end if id.size > 11 @@ -30,6 +28,14 @@ class Invidious::Routes::Watch < Invidious::Routes::BaseRoute return env.redirect "/" end + embed_link = "/embed/#{id}" + if env.params.query.size > 1 + embed_params = HTTP::Params.parse(env.params.query.to_s) + embed_params.delete_all("v") + embed_link += "?" + embed_link += embed_params.to_s + end + plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") continuation = process_continuation(PG_DB, env.params.query, plid, id) @@ -56,10 +62,8 @@ class Invidious::Routes::Watch < Invidious::Routes::BaseRoute rescue ex : VideoRedirect return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex - error_message = ex.message - env.response.status_code = 500 - logger.puts("#{id} : #{ex.message}") - return templated "error" + logger.error("get_video: #{id} : #{ex.message}") + return error_template(500, ex) end if preferences.annotations_subscribed && diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index c09dda38..602e6ae5 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -1,8 +1,15 @@ module Invidious::Routing - macro get(path, controller) + macro get(path, controller, method = :handle) get {{ path }} do |env| controller_instance = {{ controller }}.new(config, logger) - controller_instance.handle(env) + controller_instance.{{ method.id }}(env) + end + end + + macro post(path, controller, method = :handle) + post {{ path }} do |env| + controller_instance = {{ controller }}.new(config, logger) + controller_instance.{{ method.id }}(env) end end end diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 46bf8865..5dc16edd 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -66,6 +66,8 @@ struct Preferences @[JSON::Field(converter: Preferences::ProcessString)] property quality : String = CONFIG.default_user_preferences.quality + @[JSON::Field(converter: Preferences::ProcessString)] + property quality_dash : String = CONFIG.default_user_preferences.quality_dash property default_home : String = CONFIG.default_user_preferences.default_home property feed_menu : Array(String) = CONFIG.default_user_preferences.feed_menu property related_videos : Bool = CONFIG.default_user_preferences.related_videos @@ -267,12 +269,12 @@ struct Preferences end end -def get_user(sid, headers, db, refresh = true) +def get_user(sid, headers, db, logger, refresh = true) if email = db.query_one?("SELECT email FROM session_ids WHERE id = $1", sid, as: String) user = db.query_one("SELECT * FROM users WHERE email = $1", email, as: User) if refresh && Time.utc - user.updated > 1.minute - user, sid = fetch_user(sid, headers, db) + user, sid = fetch_user(sid, headers, db, logger) user_array = user.to_a user_array[4] = user_array[4].to_json # User preferences args = arg_array(user_array) @@ -290,7 +292,7 @@ def get_user(sid, headers, db, refresh = true) end end else - user, sid = fetch_user(sid, headers, db) + user, sid = fetch_user(sid, headers, db, logger) user_array = user.to_a user_array[4] = user_array[4].to_json # User preferences args = arg_array(user.to_a) @@ -311,7 +313,7 @@ def get_user(sid, headers, db, refresh = true) return user, sid end -def fetch_user(sid, headers, db) +def fetch_user(sid, headers, db, logger) feed = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers) feed = XML.parse_html(feed.body) @@ -324,7 +326,7 @@ def fetch_user(sid, headers, db) end end - channels = get_batch_channels(channels, db, false, false) + channels = get_batch_channels(channels, db, logger, false, false) email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"])) if email @@ -425,7 +427,7 @@ def generate_captcha(key, db) end def generate_text_captcha(key, db) - response = make_client(TEXTCAPTCHA_URL).get("/omarroth@protonmail.com.json").body + response = make_client(TEXTCAPTCHA_URL, &.get("/omarroth@protonmail.com.json").body) response = JSON.parse(response) tokens = response["a"].as_a.map do |answer| diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 8e314fe0..4a831110 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -235,6 +235,7 @@ struct VideoPreferences property preferred_captions : Array(String) property player_style : String property quality : String + property quality_dash : String property raw : Bool property region : String? property related_videos : Bool @@ -816,7 +817,7 @@ end def extract_polymer_config(body) params = {} of String => JSON::Any - player_response = body.match(/window\["ytInitialPlayerResponse"\]\s*=\s*(?<info>.*?);\n/) + player_response = body.match(/(window\["ytInitialPlayerResponse"\]|var\sytInitialPlayerResponse)\s*=\s*(?<info>{.*?});/m) .try { |r| JSON.parse(r["info"]).as_h } if body.includes?("To continue with your YouTube experience, please fill out the form below.") || @@ -830,7 +831,8 @@ def extract_polymer_config(body) params["reason"] = JSON::Any.new(reason) end - params["sessionToken"] = JSON::Any.new(body.match(/"XSRF_TOKEN":"(?<session_token>[^"]+)"/).try &.["session_token"]?) + session_token_json_encoded = body.match(/"XSRF_TOKEN":"(?<session_token>[^"]+)"/).try &.["session_token"]? || "" + params["sessionToken"] = JSON.parse(%({"key": "#{session_token_json_encoded}"}))["key"] params["shortDescription"] = JSON::Any.new(body.match(/"og:description" content="(?<description>[^"]+)"/).try &.["description"]?) return params if !player_response @@ -839,8 +841,7 @@ def extract_polymer_config(body) params[f] = player_response[f] if player_response[f]? end - yt_initial_data = body.match(/(window\["ytInitialData"\]|var\s+ytInitialData)\s*=\s*(?<info>.*?);\s*\n/) - .try { |r| JSON.parse(r["info"]).as_h } + yt_initial_data = extract_initial_data(body) params["relatedVideos"] = yt_initial_data.try &.["playerOverlays"]?.try &.["playerOverlayRenderer"]? .try &.["endScreen"]?.try &.["watchNextEndScreenRenderer"]?.try &.["results"]?.try &.as_a.compact_map { |r| @@ -915,10 +916,14 @@ def extract_polymer_config(body) .try { |r| JSON.parse(r["info"]) }.try &.["args"]["player_response"]? .try &.as_s?.try &.try { |r| JSON.parse(r).as_h } - return params if !initial_data - - {"playabilityStatus", "streamingData"}.each do |f| - params[f] = initial_data[f] if initial_data[f]? + if initial_data + {"playabilityStatus", "streamingData"}.each do |f| + params[f] = initial_data[f] if initial_data[f]? + end + else + {"playabilityStatus", "streamingData"}.each do |f| + params[f] = player_response[f] if player_response[f]? + end end params @@ -999,7 +1004,7 @@ def fetch_video(id, region) }.try { |a| JSON::Any.new(a) } || JSON::Any.new([] of JSON::Any) end - raise info["reason"]?.try &.as_s || "" if !info["videoDetails"]? + raise InfoException.new(info["reason"]?.try &.as_s || "") if !info["videoDetails"]? video = Video.new({ id: id, @@ -1039,6 +1044,7 @@ def process_video_params(query, preferences) player_style = query["player_style"]? preferred_captions = query["subtitles"]?.try &.split(",").map { |a| a.downcase } quality = query["quality"]? + quality_dash = query["quality_dash"]? region = query["region"]? related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe } speed = query["speed"]?.try &.rchop("x").to_f? @@ -1057,6 +1063,7 @@ def process_video_params(query, preferences) player_style ||= preferences.player_style preferred_captions ||= preferences.captions quality ||= preferences.quality + quality_dash ||= preferences.quality_dash related_videos ||= preferences.related_videos.to_unsafe speed ||= preferences.speed video_loop ||= preferences.video_loop.to_unsafe @@ -1073,6 +1080,7 @@ def process_video_params(query, preferences) player_style ||= CONFIG.default_user_preferences.player_style preferred_captions ||= CONFIG.default_user_preferences.captions quality ||= CONFIG.default_user_preferences.quality + quality_dash ||= CONFIG.default_user_preferences.quality_dash related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe speed ||= CONFIG.default_user_preferences.speed video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe @@ -1125,6 +1133,7 @@ def process_video_params(query, preferences) player_style: player_style, preferred_captions: preferred_captions, quality: quality, + quality_dash: quality_dash, raw: raw, region: region, related_videos: related_videos, diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index 0e6664fa..625c6fee 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -4,7 +4,7 @@ <% if params.video_loop %>loop<% end %> <% if params.controls %>controls<% end %>> <% if (hlsvp = video.hls_manifest_url) && !CONFIG.disabled?("livestreams") %> - <source src="<%= URI.parse(hlsvp).full_path %>?local=true" type="application/x-mpegURL" label="livestream"> + <source src="<%= URI.parse(hlsvp).full_path %><% if params.local %>?local=true<% end %>" type="application/x-mpegURL" label="livestream"> <% else %> <% if params.listen %> <% audio_streams.each_with_index do |fmt, i| %> diff --git a/src/invidious/views/components/player_sources.ecr b/src/invidious/views/components/player_sources.ecr index d02f82d2..8162546e 100644 --- a/src/invidious/views/components/player_sources.ecr +++ b/src/invidious/views/components/player_sources.ecr @@ -3,7 +3,6 @@ <link rel="stylesheet" href="/css/videojs.markers.min.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/videojs-share.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/videojs-vtt-thumbnails.css?v=<%= ASSET_COMMIT %>"> -<link rel="stylesheet" href="/css/videojs-vtt-thumbnails-fix.css?v=<%= ASSET_COMMIT %>"> <script src="/js/global.js?v=<%= ASSET_COMMIT %>"></script> <script src="/js/video.min.js?v=<%= ASSET_COMMIT %>"></script> <script src="/js/videojs-contrib-quality-levels.min.js?v=<%= ASSET_COMMIT %>"></script> diff --git a/src/invidious/views/data_control.ecr b/src/invidious/views/data_control.ecr index e3edb9ea..74ccc06c 100644 --- a/src/invidious/views/data_control.ecr +++ b/src/invidious/views/data_control.ecr @@ -14,7 +14,7 @@ <div class="pure-control-group"> <label for="import_youtube"> - <a rel="noopener" target="_blank" href="https://support.google.com/youtube/answer/6224202?hl=en"> + <a rel="noopener" target="_blank" href="https://github.com/iv-org/documentation/blob/master/Export-YouTube-subscriptions.md"> <%= translate(locale, "Import YouTube subscriptions") %> </a> </label> diff --git a/src/invidious/views/embed.ecr b/src/invidious/views/embed.ecr index 48dbc55f..dbb86009 100644 --- a/src/invidious/views/embed.ecr +++ b/src/invidious/views/embed.ecr @@ -9,12 +9,11 @@ <link rel="stylesheet" href="/css/videojs-overlay.css?v=<%= ASSET_COMMIT %>"> <script src="/js/videojs-overlay.min.js?v=<%= ASSET_COMMIT %>"></script> <link rel="stylesheet" href="/css/default.css?v=<%= ASSET_COMMIT %>"> - <link rel="stylesheet" href="/css/darktheme.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/embed.css?v=<%= ASSET_COMMIT %>"> <title><%= HTML.escape(video.title) %> - Invidious</title> </head> -<body> +<body class="dark-theme"> <script id="video_data" type="application/json"> <%= { diff --git a/src/invidious/views/message.ecr b/src/invidious/views/message.ecr new file mode 100644 index 00000000..8c7bf611 --- /dev/null +++ b/src/invidious/views/message.ecr @@ -0,0 +1,12 @@ +<% content_for "header" do %> +<meta name="description" content="<%= translate(locale, "An alternative front-end to YouTube") %>"> +<title> + Invidious +</title> +<% end %> + +<%= rendered "components/feed_menu" %> + +<p> + <%= message %> +</p> diff --git a/src/invidious/views/playlists.ecr b/src/invidious/views/playlists.ecr index 0c48be96..a77d106d 100644 --- a/src/invidious/views/playlists.ecr +++ b/src/invidious/views/playlists.ecr @@ -27,7 +27,7 @@ </div> <div class="h-box"> - <p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %></span></p> + <p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content if !channel.description_html.empty? %></span></p> </div> <div class="h-box"> diff --git a/src/invidious/views/preferences.ecr b/src/invidious/views/preferences.ecr index fb5bd44b..1ef080be 100644 --- a/src/invidious/views/preferences.ecr +++ b/src/invidious/views/preferences.ecr @@ -57,6 +57,17 @@ </select> </div> + <% if !CONFIG.disabled?("dash") %> + <div class="pure-control-group"> + <label for="quality_dash"><%= translate(locale, "Preferred dash video quality: ") %></label> + <select name="quality_dash" id="quality_dash"> + <% {"auto", "best", "4320p", "2160p", "1440p", "1080p", "720p", "480p", "360p", "240p", "144p", "worst"}.each do |option| %> + <option value="<%= option %>" <% if preferences.quality_dash == option %> selected <% end %>><%= translate(locale, option) %></option> + <% end %> + </select> + </div> + <% end %> + <div class="pure-control-group"> <label for="volume"><%= translate(locale, "Player volume: ") %></label> <input name="volume" id="volume" data-onrange="update_volume_value" type="range" min="0" max="100" step="5" value="<%= preferences.volume %>"> @@ -68,7 +79,7 @@ <% preferences.comments.each_with_index do |comments, index| %> <select name="comments[<%= index %>]" id="comments[<%= index %>]"> <% {"", "youtube", "reddit"}.each do |option| %> - <option value="<%= option %>" <% if preferences.comments[index] == option %> selected <% end %>><%= translate(locale, option) %></option> + <option value="<%= option %>" <% if preferences.comments[index] == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option> <% end %> </select> <% end %> @@ -79,7 +90,7 @@ <% preferences.captions.each_with_index do |caption, index| %> <select class="pure-u-1-6" name="captions[<%= index %>]" id="captions[<%= index %>]"> <% CAPTION_LANGUAGES.each do |option| %> - <option value="<%= option %>" <% if preferences.captions[index] == option %> selected <% end %>><%= translate(locale, option) %></option> + <option value="<%= option %>" <% if preferences.captions[index] == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option> <% end %> </select> <% end %> @@ -119,7 +130,7 @@ <label for="dark_mode"><%= translate(locale, "Theme: ") %></label> <select name="dark_mode" id="dark_mode"> <% {"", "light", "dark"}.each do |option| %> - <option value="<%= option %>" <% if preferences.dark_mode == option %> selected <% end %>><%= translate(locale, option) %></option> + <option value="<%= option %>" <% if preferences.dark_mode == option %> selected <% end %>><%= translate(locale, option.blank? ? "auto" : option) %></option> <% end %> </select> </div> @@ -130,16 +141,16 @@ </div> <% if env.get?("user") %> - <% feed_options = {"", "Popular", "Top", "Trending", "Subscriptions", "Playlists"} %> + <% feed_options = {"", "Popular", "Trending", "Subscriptions", "Playlists"} %> <% else %> - <% feed_options = {"", "Popular", "Top", "Trending"} %> + <% feed_options = {"", "Popular", "Trending"} %> <% end %> <div class="pure-control-group"> <label for="default_home"><%= translate(locale, "Default homepage: ") %></label> <select name="default_home" id="default_home"> <% feed_options.each do |option| %> - <option value="<%= option %>" <% if preferences.default_home == option %> selected <% end %>><%= translate(locale, option) %></option> + <option value="<%= option %>" <% if preferences.default_home == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option> <% end %> </select> </div> @@ -149,7 +160,7 @@ <% (feed_options.size - 1).times do |index| %> <select name="feed_menu[<%= index %>]" id="feed_menu[<%= index %>]"> <% feed_options.each do |option| %> - <option value="<%= option %>" <% if preferences.feed_menu[index]? == option %> selected <% end %>><%= translate(locale, option) %></option> + <option value="<%= option %>" <% if preferences.feed_menu[index]? == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option> <% end %> </select> <% end %> @@ -211,7 +222,7 @@ <label for="admin_default_home"><%= translate(locale, "Default homepage: ") %></label> <select name="admin_default_home" id="admin_default_home"> <% feed_options.each do |option| %> - <option value="<%= option %>" <% if CONFIG.default_user_preferences.default_home == option %> selected <% end %>><%= translate(locale, option) %></option> + <option value="<%= option %>" <% if CONFIG.default_user_preferences.default_home == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option> <% end %> </select> </div> @@ -221,13 +232,19 @@ <% (feed_options.size - 1).times do |index| %> <select name="admin_feed_menu[<%= index %>]" id="admin_feed_menu[<%= index %>]"> <% feed_options.each do |option| %> - <option value="<%= option %>" <% if CONFIG.default_user_preferences.feed_menu[index]? == option %> selected <% end %>><%= translate(locale, option) %></option> + <option value="<%= option %>" <% if CONFIG.default_user_preferences.feed_menu[index]? == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option> <% end %> </select> <% end %> </div> <div class="pure-control-group"> + <label for="popular_enabled"><%= translate(locale, "Popular enabled: ") %></label> + <input name="popular_enabled" id="popular_enabled" type="checkbox" <% if config.popular_enabled %>checked<% end %>> + </div> + + + <div class="pure-control-group"> <label for="captcha_enabled"><%= translate(locale, "CAPTCHA enabled: ") %></label> <input name="captcha_enabled" id="captcha_enabled" type="checkbox" <% if config.captcha_enabled %>checked<% end %>> </div> diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index 61cf5c3a..f6e5262d 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -4,7 +4,6 @@ <head> <meta charset="utf-8"> <meta name="viewport" content="width=device-width, initial-scale=1"> - <meta name="referrer" content="no-referrer"> <%= yield_content "header" %> <link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png?v=<%= ASSET_COMMIT %>"> <link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png?v=<%= ASSET_COMMIT %>"> @@ -18,13 +17,12 @@ <link rel="stylesheet" href="/css/grids-responsive-min.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/ionicons.min.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/default.css?v=<%= ASSET_COMMIT %>"> - <link rel="stylesheet" href="/css/darktheme.css?v=<%= ASSET_COMMIT %>" id="dark_theme" <% if env.get("preferences").as(Preferences).dark_mode != "dark" %>media="none"<% end %>> - <link rel="stylesheet" href="/css/lighttheme.css?v=<%= ASSET_COMMIT %>" id="light_theme" <% if env.get("preferences").as(Preferences).dark_mode == "dark" %>media="none"<% end %>> </head> <% locale = LOCALES[env.get("preferences").as(Preferences).locale]? %> +<% dark_mode = env.get("preferences").as(Preferences).dark_mode %> -<body> +<body class="<%= dark_mode.blank? ? "no" : dark_mode %>-theme"> <span style="display:none" id="dark_mode_pref"><%= env.get("preferences").as(Preferences).dark_mode %></span> <div class="pure-g"> <div class="pure-u-1 pure-u-md-2-24"></div> @@ -116,16 +114,15 @@ </a> </div> <div class="pure-u-1 pure-u-md-1-3"> - <i class="icon ion-logo-bitcoin"></i> - BTC: <a href="bitcoin:356DpZyMXu6rYd55Yqzjs29n79kGKWcYrY">356DpZyMXu6rYd55Yqzjs29n79kGKWcYrY</a> + <i class="icon ion-ios-wallet"></i> + BTC: <a href="bitcoin:bc1qfhe7rq3lqzuayzjxzyt9waz9ytrs09kla3tsgr">bc1qfhe7rq3lqzuayzjxzyt9waz9ytrs09kla3tsgr</a> </div> <div class="pure-u-1 pure-u-md-1-3"> - <i class="icon ion-logo-bitcoin"></i> - BCH: <a href="bitcoincash:qq4ptclkzej5eza6a50et5ggc58hxsq5aylqut2npk">qq4ptclkzej5eza6a50et5ggc58hxsq5aylqut2npk</a> + <i class="icon ion-ios-wallet"></i> + XMR: <a href="monero:41nMCtek197boJtiUvGnTFYMatrLEpnpkQDmUECqx5Es2uX3sTKKWVhSL76suXsG3LXqkEJBrCZBgPTwJrDp1FrZJfycGPR">Click here</a> </div> <div class="pure-u-1 pure-u-md-1-3"> - <i class="icon ion-logo-usd"></i> - <a href="https://liberapay.com/omarroth">Liberapay</a> + <a href="https://github.com/iv-org/documentation">Documentation</a> </div> <div class="pure-u-1 pure-u-md-1-3"> <i class="icon ion-logo-javascript"></i> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 9a1e6c32..786a88b6 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -88,7 +88,11 @@ <div class="h-box"> <span id="watch-on-youtube"> <a href="https://www.youtube.com/watch?v=<%= video.id %>"><%= translate(locale, "Watch on YouTube") %></a> + (<a href="https://www.youtube.com/embed/<%= video.id %>"><%= translate(locale, "Embed") %></a>) </span> + <p id="embed-link"> + <a href="<%= embed_link %>"><%= translate(locale, "Embed Link") %></a> + </p> <p id="annotations"> <% if params.annotations %> <a href="/watch?<%= env.params.query %>&iv_load_policy=3"> |
