diff options
33 files changed, 309 insertions, 432 deletions
@@ -20,6 +20,9 @@ Lint/ShadowingOuterLocalVar: Excluded: - src/invidious/helpers/tokens.cr +Lint/NotNil: + Enabled: false + # # Style @@ -31,6 +34,13 @@ Style/RedundantBegin: Style/RedundantReturn: Enabled: false +Style/ParenthesesAroundCondition: + Enabled: false + +# This requires a rewrite of most data structs (and their usage) in Invidious. +Style/QueryBoolMethods: + Enabled: false + # # Metrics @@ -39,50 +49,4 @@ Style/RedundantReturn: # Ignore function complexity (number of if/else & case/when branches) # For some functions that can hardly be simplified for now Metrics/CyclomaticComplexity: - Excluded: - # get_about_info(ucid, locale) => [17/10] - - src/invidious/channels/about.cr - - # fetch_channel_community(ucid, continuation, ...) => [34/10] - - src/invidious/channels/community.cr - - # create_notification_stream(env, topics, connection_channel) => [14/10] - - src/invidious/helpers/helpers.cr:84:5 - - # get_index(plural_form, count) => [25/10] - - src/invidious/helpers/i18next.cr - - # call(context) => [18/10] - - src/invidious/helpers/static_file_handler.cr - - # show(env) => [38/10] - - src/invidious/routes/embed.cr - - # get_video_playback(env) => [45/10] - - src/invidious/routes/video_playback.cr - - # handle(env) => [40/10] - - src/invidious/routes/watch.cr - - # playlist_ajax(env) => [24/10] - - src/invidious/routes/playlists.cr - - # fetch_youtube_comments(id, cursor, ....) => [40/10] - # template_youtube_comments(comments, locale, ...) => [16/10] - # content_to_comment_html(content) => [14/10] - - src/invidious/comments.cr - - # to_json(locale, json) => [21/10] - # extract_video_info(video_id, ...) => [44/10] - # process_video_params(query, preferences) => [20/10] - - src/invidious/videos.cr - - - -#src/invidious/playlists.cr:327:5 -#[C] Metrics/CyclomaticComplexity: Cyclomatic complexity too high [19/10] -# fetch_playlist(plid : String) - -#src/invidious/playlists.cr:436:5 -#[C] Metrics/CyclomaticComplexity: Cyclomatic complexity too high [11/10] -# extract_playlist_videos(initial_data : Hash(String, JSON::Any)) + Enabled: false diff --git a/.github/workflows/container-release.yml b/.github/workflows/build-nightly-container.yml index e44ac200..bee27600 100644 --- a/.github/workflows/container-release.yml +++ b/.github/workflows/build-nightly-container.yml @@ -1,4 +1,4 @@ -name: Build and release container +name: Build and release container directly from master on: push: @@ -24,9 +24,9 @@ jobs: uses: actions/checkout@v4 - name: Install Crystal - uses: crystal-lang/install-crystal@v1.8.0 + uses: crystal-lang/install-crystal@v1.8.2 with: - crystal: 1.9.2 + crystal: 1.12.2 - name: Run lint run: | @@ -58,7 +58,7 @@ jobs: images: quay.io/invidious/invidious tags: | type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} - type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + type=raw,value=master,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} labels: | quay.expires-after=12w @@ -83,7 +83,7 @@ jobs: suffix=-arm64 tags: | type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} - type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + type=raw,value=master,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} labels: | quay.expires-after=12w diff --git a/.github/workflows/build-stable-container.yml b/.github/workflows/build-stable-container.yml new file mode 100644 index 00000000..b5fbc705 --- /dev/null +++ b/.github/workflows/build-stable-container.yml @@ -0,0 +1,90 @@ +name: Build and release container + +on: + push: + tags: + - "v*" + +jobs: + release: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Crystal + uses: crystal-lang/install-crystal@v1.8.2 + with: + crystal: 1.12.2 + + - name: Run lint + run: | + if ! crystal tool format --check; then + crystal tool format + git diff + exit 1 + fi + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to registry + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSWORD }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: quay.io/invidious/invidious + tags: | + type=semver,pattern={{version}} + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + labels: | + quay.expires-after=12w + + - name: Build and push Docker AMD64 image for Push Event + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile + platforms: linux/amd64 + labels: ${{ steps.meta.outputs.labels }} + push: true + tags: ${{ steps.meta.outputs.tags }} + build-args: | + "release=1" + + - name: Docker meta + id: meta-arm64 + uses: docker/metadata-action@v5 + with: + images: quay.io/invidious/invidious + flavor: | + suffix=-arm64 + tags: | + type=semver,pattern={{version}} + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + labels: | + quay.expires-after=12w + + - name: Build and push Docker ARM64 image for Push Event + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile.arm64 + platforms: linux/arm64/v8 + labels: ${{ steps.meta-arm64.outputs.labels }} + push: true + tags: ${{ steps.meta-arm64.outputs.tags }} + build-args: | + "release=1" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 057e4d61..925a8fc7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,10 +38,10 @@ jobs: matrix: stable: [true] crystal: - - 1.7.3 - - 1.8.2 - 1.9.2 - 1.10.1 + - 1.11.2 + - 1.12.1 include: - crystal: nightly stable: false @@ -124,4 +124,28 @@ jobs: - name: Test Docker run: while curl -Isf http://localhost:3000; do sleep 1; done + ameba_lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Install Crystal + uses: crystal-lang/install-crystal@v1.8.0 + with: + crystal: latest + + - name: Cache Shards + uses: actions/cache@v3 + with: + path: | + ./lib + ./bin + key: shards-${{ hashFiles('shard.lock') }} + + - name: Install Shards + run: shards install + - name: Run Ameba linter + run: bin/ameba diff --git a/docker/Dockerfile b/docker/Dockerfile index ace096bf..3d9323fd 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM crystallang/crystal:1.8.2-alpine AS builder +FROM crystallang/crystal:1.12.1-alpine AS builder RUN apk add --no-cache sqlite-static yaml-static diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index 602f3ab2..f054b326 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -1,5 +1,5 @@ -FROM alpine:3.18 AS builder -RUN apk add --no-cache 'crystal=1.8.2-r0' shards sqlite-static yaml-static yaml-dev libxml2-static zlib-static openssl-libs-static openssl-dev musl-dev xz-static +FROM alpine:3.19 AS builder +RUN apk add --no-cache 'crystal=1.10.1-r0' shards sqlite-static yaml-static yaml-dev libxml2-static zlib-static openssl-libs-static openssl-dev musl-dev xz-static ARG release diff --git a/kubernetes/.gitignore b/kubernetes/.gitignore deleted file mode 100644 index 0ad51707..00000000 --- a/kubernetes/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/charts/*.tgz diff --git a/kubernetes/Chart.lock b/kubernetes/Chart.lock deleted file mode 100644 index ef12b0b6..00000000 --- a/kubernetes/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: postgresql - repository: https://charts.bitnami.com/bitnami/ - version: 12.11.1 -digest: sha256:3c10008175c4f5c1cec38782f5a7316154b89074c77ebbd9bcc4be4f5ff21122 -generated: "2023-09-14T22:40:43.171275362Z" diff --git a/kubernetes/Chart.yaml b/kubernetes/Chart.yaml deleted file mode 100644 index d22f6254..00000000 --- a/kubernetes/Chart.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: v2 -name: invidious -description: Invidious is an alternative front-end to YouTube -version: 1.1.1 -appVersion: 0.20.1 -keywords: -- youtube -- proxy -- video -- privacy -home: https://invidio.us/ -icon: https://raw.githubusercontent.com/iv-org/invidious/05988c1c49851b7d0094fca16aeaf6382a7f64ab/assets/favicon-32x32.png -sources: -- https://github.com/iv-org/invidious -maintainers: -- name: Leon Klingele - email: mail@leonklingele.de -dependencies: -- name: postgresql - version: ~12.11.0 - repository: "https://charts.bitnami.com/bitnami/" -engine: gotpl diff --git a/kubernetes/README.md b/kubernetes/README.md index 35478f99..e71f6a86 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -1,41 +1 @@ -# Invidious Helm chart - -Easily deploy Invidious to Kubernetes. - -## Installing Helm chart - -```sh -# Build Helm dependencies -$ helm dep build - -# Add PostgreSQL init scripts -$ kubectl create configmap invidious-postgresql-init \ - --from-file=../config/sql/channels.sql \ - --from-file=../config/sql/videos.sql \ - --from-file=../config/sql/channel_videos.sql \ - --from-file=../config/sql/users.sql \ - --from-file=../config/sql/session_ids.sql \ - --from-file=../config/sql/nonces.sql \ - --from-file=../config/sql/annotations.sql \ - --from-file=../config/sql/playlists.sql \ - --from-file=../config/sql/playlist_videos.sql - -# Install Helm app to your Kubernetes cluster -$ helm install invidious ./ -``` - -## Upgrading - -```sh -# Upgrading is easy, too! -$ helm upgrade invidious ./ -``` - -## Uninstall - -```sh -# Get rid of everything (except database) -$ helm delete invidious - -# To also delete the database, remove all invidious-postgresql PVCs -``` +The Helm chart has moved to a dedicated GitHub repository: https://github.com/iv-org/invidious-helm-chart/tree/master/invidious
\ No newline at end of file diff --git a/kubernetes/templates/_helpers.tpl b/kubernetes/templates/_helpers.tpl deleted file mode 100644 index 52158b78..00000000 --- a/kubernetes/templates/_helpers.tpl +++ /dev/null @@ -1,16 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "invidious.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "invidious.fullname" -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} diff --git a/kubernetes/templates/configmap.yaml b/kubernetes/templates/configmap.yaml deleted file mode 100644 index 58542a31..00000000 --- a/kubernetes/templates/configmap.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ template "invidious.fullname" . }} - labels: - app: {{ template "invidious.name" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: {{ .Release.Name }} -data: - INVIDIOUS_CONFIG: | -{{ toYaml .Values.config | indent 4 }} diff --git a/kubernetes/templates/deployment.yaml b/kubernetes/templates/deployment.yaml deleted file mode 100644 index bb0b832f..00000000 --- a/kubernetes/templates/deployment.yaml +++ /dev/null @@ -1,61 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "invidious.fullname" . }} - labels: - app: {{ template "invidious.name" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: {{ .Release.Name }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - app: {{ template "invidious.name" . }} - release: {{ .Release.Name }} - template: - metadata: - labels: - app: {{ template "invidious.name" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: {{ .Release.Name }} - spec: - securityContext: - runAsUser: {{ .Values.securityContext.runAsUser }} - runAsGroup: {{ .Values.securityContext.runAsGroup }} - fsGroup: {{ .Values.securityContext.fsGroup }} - initContainers: - - name: wait-for-postgresql - image: postgres - args: - - /bin/sh - - -c - - until pg_isready -h {{ .Values.config.db.host }} -p {{ .Values.config.db.port }} -U {{ .Values.config.db.user }}; do echo waiting for database; sleep 2; done; - containers: - - name: {{ .Chart.Name }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - containerPort: 3000 - env: - - name: INVIDIOUS_CONFIG - valueFrom: - configMapKeyRef: - key: INVIDIOUS_CONFIG - name: {{ template "invidious.fullname" . }} - securityContext: - allowPrivilegeEscalation: {{ .Values.securityContext.allowPrivilegeEscalation }} - capabilities: - drop: - - ALL - resources: -{{ toYaml .Values.resources | indent 10 }} - readinessProbe: - httpGet: - port: 3000 - path: / - livenessProbe: - httpGet: - port: 3000 - path: / - initialDelaySeconds: 15 - restartPolicy: Always diff --git a/kubernetes/templates/hpa.yaml b/kubernetes/templates/hpa.yaml deleted file mode 100644 index c6fbefe2..00000000 --- a/kubernetes/templates/hpa.yaml +++ /dev/null @@ -1,18 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ template "invidious.fullname" . }} - labels: - app: {{ template "invidious.name" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: {{ .Release.Name }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ template "invidious.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - targetCPUUtilizationPercentage: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} -{{- end }} diff --git a/kubernetes/templates/service.yaml b/kubernetes/templates/service.yaml deleted file mode 100644 index 01454d4e..00000000 --- a/kubernetes/templates/service.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ template "invidious.fullname" . }} - labels: - app: {{ template "invidious.name" . }} - chart: {{ .Chart.Name }} - release: {{ .Release.Name }} -spec: - type: {{ .Values.service.type }} - ports: - - name: http - port: {{ .Values.service.port }} - targetPort: 3000 - selector: - app: {{ template "invidious.name" . }} - release: {{ .Release.Name }} -{{- if .Values.service.loadBalancerIP }} - loadBalancerIP: {{ .Values.service.loadBalancerIP }} -{{- end }} diff --git a/kubernetes/values.yaml b/kubernetes/values.yaml deleted file mode 100644 index 5000c2b6..00000000 --- a/kubernetes/values.yaml +++ /dev/null @@ -1,61 +0,0 @@ -name: invidious - -image: - repository: quay.io/invidious/invidious - tag: latest - pullPolicy: Always - -replicaCount: 1 - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 16 - targetCPUUtilizationPercentage: 50 - -service: - type: ClusterIP - port: 3000 - #loadBalancerIP: - -resources: {} - #requests: - # cpu: 100m - # memory: 64Mi - #limits: - # cpu: 800m - # memory: 512Mi - -securityContext: - allowPrivilegeEscalation: false - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - -# See https://github.com/bitnami/charts/tree/master/bitnami/postgresql -postgresql: - image: - tag: 13 - auth: - username: kemal - password: kemal - database: invidious - primary: - initdb: - username: kemal - password: kemal - scriptsConfigMap: invidious-postgresql-init - -# Adapted from ../config/config.yml -config: - channel_threads: 1 - feed_threads: 1 - db: - user: kemal - password: kemal - host: invidious-postgresql - port: 5432 - dbname: invidious - full_refresh: false - https_only: false - domain: @@ -2,7 +2,7 @@ version: 2.0 shards: ameba: git: https://github.com/crystal-ameba/ameba.git - version: 1.5.0 + version: 1.6.1 athena-negotiation: git: https://github.com/athena-framework/negotiation.git @@ -35,7 +35,7 @@ development_dependencies: version: ~> 0.10.4 ameba: github: crystal-ameba/ameba - version: ~> 1.5.0 + version: ~> 1.6.1 crystal: ">= 1.0.0, < 2.0.0" diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 351790d7..6cc30142 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -1,4 +1,4 @@ -def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) +def produce_channel_content_continuation(ucid, content_type, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) object_inner_2 = { "2:0:embedded" => { "1:0:varint" => 0_i64, @@ -16,6 +16,13 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } + content_type_numerical = + case content_type + when "videos" then 15 + when "livestreams" then 14 + else 15 # Fallback to "videos" + end + sort_by_numerical = case sort_by when "newest" then 1_i64 @@ -27,7 +34,7 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so object_inner_1 = { "110:embedded" => { "3:embedded" => { - "15:embedded" => { + "#{content_type_numerical}:embedded" => { "1:embedded" => { "1:string" => object_inner_2_encoded, }, @@ -62,6 +69,10 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so return continuation end +def make_initial_content_ctoken(ucid, content_type, sort_by) : String + return produce_channel_content_continuation(ucid, content_type, sort_by: sort_by) +end + module Invidious::Channel::Tabs extend self @@ -69,10 +80,6 @@ module Invidious::Channel::Tabs # Regular videos # ------------------- - def make_initial_video_ctoken(ucid, sort_by) : String - return produce_channel_videos_continuation(ucid, sort_by: sort_by) - end - # Wrapper for AboutChannel, as we still need to call get_videos with # an author name and ucid directly (e.g in RSS feeds). # TODO: figure out how to get rid of that @@ -94,7 +101,7 @@ module Invidious::Channel::Tabs end def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest") - continuation ||= make_initial_video_ctoken(ucid, sort_by) + continuation ||= make_initial_content_ctoken(ucid, "videos", sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, author, ucid) @@ -138,21 +145,18 @@ module Invidious::Channel::Tabs # Livestreams # ------------------- - def get_livestreams(channel : AboutChannel, continuation : String? = nil) - if continuation.nil? - # EgdzdHJlYW1z8gYECgJ6AA%3D%3D is the protobuf object to load "streams" - initial_data = YoutubeAPI.browse(channel.ucid, params: "EgdzdHJlYW1z8gYECgJ6AA%3D%3D") - else - initial_data = YoutubeAPI.browse(continuation: continuation) - end + def get_livestreams(channel : AboutChannel, continuation : String? = nil, sort_by = "newest") + continuation ||= make_initial_content_ctoken(channel.ucid, "livestreams", sort_by) + + initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, channel.author, channel.ucid) end - def get_60_livestreams(channel : AboutChannel, continuation : String? = nil) + def get_60_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") if continuation.nil? - # Fetch the first "page" of streams - items, next_continuation = get_livestreams(channel) + # Fetch the first "page" of stream + items, next_continuation = get_livestreams(channel, sort_by: sort_by) else # Fetch a "page" of streams using the given continuation token items, next_continuation = get_livestreams(channel, continuation: continuation) diff --git a/src/invidious/http_server/utils.cr b/src/invidious/http_server/utils.cr index 222dfc4a..623a9177 100644 --- a/src/invidious/http_server/utils.cr +++ b/src/invidious/http_server/utils.cr @@ -11,11 +11,12 @@ module Invidious::HttpServer params = url.query_params params["host"] = url.host.not_nil! # Should never be nil, in theory params["region"] = region if !region.nil? + url.query_params = params if absolute - return "#{HOST_URL}#{url.request_target}?#{params}" + return "#{HOST_URL}#{url.request_target}" else - return "#{url.request_target}?#{params}" + return url.request_target end end diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index 0dced80b..59714828 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -114,25 +114,31 @@ module Invidious::JSONify::APIv1 json.field "projectionType", fmt["projectionType"] - if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) - fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 + height = fmt["height"]?.try &.as_i + width = fmt["width"]?.try &.as_i + + fps = fmt["fps"]?.try &.as_i + + if fps json.field "fps", fps - json.field "container", fmt_info["ext"] - json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] + end - if fmt_info["height"]? - json.field "resolution", "#{fmt_info["height"]}p" + if height && width + json.field "size", "#{width}x#{height}" + json.field "resolution", "#{height}p" - quality_label = "#{fmt_info["height"]}p" - if fps > 30 - quality_label += "60" - end - json.field "qualityLabel", quality_label + quality_label = "#{width > height ? height : width}p" - if fmt_info["width"]? - json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" - end + if fps && fps > 30 + quality_label += fps.to_s end + + json.field "qualityLabel", quality_label + end + + if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) + json.field "container", fmt_info["ext"] + json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] end # Livestream chunk infos @@ -163,26 +169,31 @@ module Invidious::JSONify::APIv1 json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]? - fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) - if fmt_info - fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 + height = fmt["height"]?.try &.as_i + width = fmt["width"]?.try &.as_i + + fps = fmt["fps"]?.try &.as_i + + if fps json.field "fps", fps - json.field "container", fmt_info["ext"] - json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] + end - if fmt_info["height"]? - json.field "resolution", "#{fmt_info["height"]}p" + if height && width + json.field "size", "#{width}x#{height}" + json.field "resolution", "#{height}p" - quality_label = "#{fmt_info["height"]}p" - if fps > 30 - quality_label += "60" - end - json.field "qualityLabel", quality_label + quality_label = "#{width > height ? height : width}p" - if fmt_info["width"]? - json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" - end + if fps && fps > 30 + quality_label += fps.to_s end + + json.field "qualityLabel", quality_label + end + + if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) + json.field "container", fmt_info["ext"] + json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] end end end diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 955e0855..a227f794 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -366,6 +366,8 @@ def fetch_playlist(plid : String) if text.includes? "video" video_count = text.gsub(/\D/, "").to_i? || 0 + elsif text.includes? "episode" + video_count = text.gsub(/\D/, "").to_i? || 0 elsif text.includes? "view" views = text.gsub(/\D/, "").to_i64? || 0_i64 else diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 7faf200a..43a5c35b 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -208,11 +208,12 @@ module Invidious::Routes::API::V1::Channels get_channel() # Retrieve continuation from URL parameters + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" continuation = env.params.query["continuation"]? begin videos, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation + channel, continuation: continuation, sort_by: sort_by ) rescue ex return error_json(500, ex) diff --git a/src/invidious/routes/api/v1/feeds.cr b/src/invidious/routes/api/v1/feeds.cr index 41865f34..fea2993c 100644 --- a/src/invidious/routes/api/v1/feeds.cr +++ b/src/invidious/routes/api/v1/feeds.cr @@ -31,7 +31,7 @@ module Invidious::Routes::API::V1::Feeds if !CONFIG.popular_enabled error_message = {"error" => "Administrator has disabled this endpoint."}.to_json - haltf env, 400, error_message + haltf env, 403, error_message end JSON.build do |json| diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 9281f4dd..5e269923 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -89,9 +89,14 @@ module Invidious::Routes::API::V1::Videos if CONFIG.use_innertube_for_captions params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated) - initial_data = YoutubeAPI.get_transcript(params) - webvtt = Invidious::Videos::Transcript.convert_transcripts_to_vtt(initial_data, caption.language_code) + transcript = Invidious::Videos::Transcript.from_raw( + YoutubeAPI.get_transcript(params), + caption.language_code, + caption.auto_generated + ) + + webvtt = transcript.to_vtt else # Timedtext API handling url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target @@ -136,7 +141,11 @@ module Invidious::Routes::API::V1::Videos end end else - webvtt = YT_POOL.client &.get("#{url}&fmt=vtt").body + uri = URI.parse(url) + query_params = uri.query_params + query_params["fmt"] = "vtt" + uri.query_params = query_params + webvtt = YT_POOL.client &.get(uri.request_target).body if webvtt.starts_with?("<?xml") webvtt = caption.timedtext_to_vtt(webvtt) diff --git a/src/invidious/routes/before_all.cr b/src/invidious/routes/before_all.cr index 396840a4..5695dee9 100644 --- a/src/invidious/routes/before_all.cr +++ b/src/invidious/routes/before_all.cr @@ -30,7 +30,7 @@ module Invidious::Routes::BeforeAll # Only allow the pages at /embed/* to be embedded if env.request.resource.starts_with?("/embed") - frame_ancestors = "'self' http: https:" + frame_ancestors = "'self' file: http: https:" else frame_ancestors = "'none'" end diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index fea49bbe..360af2cd 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -81,13 +81,12 @@ module Invidious::Routes::Channels return env.redirect "/channel/#{channel.ucid}" end - # TODO: support sort option for livestreams - sort_by = "" - sort_options = [] of String + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" + sort_options = {"newest", "oldest", "popular"} # Fetch items and continuation token items, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation + channel, continuation: continuation, sort_by: sort_by ) selected_tab = Frontend::ChannelPage::TabsAvailable::Streams diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index 112535bd..05bc2714 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -214,7 +214,7 @@ module Invidious::Routes::PreferencesRoute statistics_enabled ||= "off" CONFIG.statistics_enabled = statistics_enabled == "on" - CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.try &.as(String) + CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.presence File.write("config/config.yml", CONFIG.to_yaml) end diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr index dac00eea..9cd064c5 100644 --- a/src/invidious/videos/transcript.cr +++ b/src/invidious/videos/transcript.cr @@ -1,8 +1,26 @@ module Invidious::Videos - # Namespace for methods primarily relating to Transcripts - module Transcript - record TranscriptLine, start_ms : Time::Span, end_ms : Time::Span, line : String + # A `Transcripts` struct encapsulates a sequence of lines that together forms the whole transcript for a given YouTube video. + # These lines can be categorized into two types: section headings and regular lines representing content from the video. + struct Transcript + # Types + record HeadingLine, start_ms : Time::Span, end_ms : Time::Span, line : String + record RegularLine, start_ms : Time::Span, end_ms : Time::Span, line : String + alias TranscriptLine = HeadingLine | RegularLine + property lines : Array(TranscriptLine) + + property language_code : String + property auto_generated : Bool + + # User friendly label for the current transcript. + # Example: "English (auto-generated)" + property label : String + + # Initializes a new Transcript struct with the contents and associated metadata describing it + def initialize(@lines : Array(TranscriptLine), @language_code : String, @auto_generated : Bool, @label : String) + end + + # Generates a protobuf string to fetch the requested transcript from YouTube def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String kind = auto_generated ? "asr" : "" @@ -30,48 +48,79 @@ module Invidious::Videos return params end - def self.convert_transcripts_to_vtt(initial_data : Hash(String, JSON::Any), target_language : String) : String - # Convert into array of TranscriptLine - lines = self.parse(initial_data) + # Constructs a Transcripts struct from the initial YouTube response + def self.from_raw(initial_data : Hash(String, JSON::Any), language_code : String, auto_generated : Bool) + transcript_panel = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", + "content", "transcriptSearchPanelRenderer") - settings_field = { - "Kind" => "captions", - "Language" => target_language, - } + segment_list = transcript_panel.dig("body", "transcriptSegmentListRenderer") - # Taken from Invidious::Videos::Captions::Metadata.timedtext_to_vtt() - vtt = WebVTT.build(settings_field) do |vtt| - lines.each do |line| - vtt.cue(line.start_ms, line.end_ms, line.line) - end + if !segment_list["initialSegments"]? + raise NotFoundException.new("Requested transcript does not exist") end - return vtt - end + # Extract user-friendly label for the current transcript + + footer_language_menu = transcript_panel.dig?( + "footer", "transcriptFooterRenderer", "languageMenu", "sortFilterSubMenuRenderer", "subMenuItems" + ) - private def self.parse(initial_data : Hash(String, JSON::Any)) - body = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", - "content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer", - "initialSegments").as_a + if footer_language_menu + label = footer_language_menu.as_a.select(&.["selected"].as_bool)[0]["title"].as_s + else + label = language_code + end + + # Extract transcript lines + + initial_segments = segment_list["initialSegments"].as_a lines = [] of TranscriptLine - body.each do |line| - # Transcript section headers. They are not apart of the captions and as such we can safely skip them. - if line.as_h.has_key?("transcriptSectionHeaderRenderer") - next + + initial_segments.each do |line| + if unpacked_line = line["transcriptSectionHeaderRenderer"]? + line_type = HeadingLine + else + unpacked_line = line["transcriptSegmentRenderer"] + line_type = RegularLine end - line = line["transcriptSegmentRenderer"] + start_ms = unpacked_line["startMs"].as_s.to_i.millisecond + end_ms = unpacked_line["endMs"].as_s.to_i.millisecond + text = extract_text(unpacked_line["snippet"]) || "" + + lines << line_type.new(start_ms, end_ms, text) + end + + return Transcript.new( + lines: lines, + language_code: language_code, + auto_generated: auto_generated, + label: label + ) + end - start_ms = line["startMs"].as_s.to_i.millisecond - end_ms = line["endMs"].as_s.to_i.millisecond + # Converts transcript lines to a WebVTT file + # + # This is used within Invidious to replace subtitles + # as to workaround YouTube's rate-limited timedtext endpoint. + def to_vtt + settings_field = { + "Kind" => "captions", + "Language" => @language_code, + } - text = extract_text(line["snippet"]) || "" + vtt = WebVTT.build(settings_field) do |vtt| + @lines.each do |line| + # Section headers are excluded from the VTT conversion as to + # match the regular captions returned from YouTube as much as possible + next if line.is_a? HeadingLine - lines << TranscriptLine.new(start_ms, end_ms, text) + vtt.cue(line.start_ms, line.end_ms, line.line) + end end - return lines + return vtt end end end diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr index 09df106d..a84e44bc 100644 --- a/src/invidious/views/channel.ecr +++ b/src/invidious/views/channel.ecr @@ -30,13 +30,13 @@ <meta property="og:site_name" content="Invidious"> <meta property="og:url" content="<%= HOST_URL %>/channel/<%= ucid %>"> <meta property="og:title" content="<%= author %>"> -<meta property="og:image" content="/ggpht<%= channel_profile_pic %>"> +<meta property="og:image" content="<%= HOST_URL %>/ggpht<%= channel_profile_pic %>"> <meta property="og:description" content="<%= channel.description %>"> <meta name="twitter:card" content="summary"> <meta name="twitter:url" content="<%= HOST_URL %>/channel/<%= ucid %>"> <meta name="twitter:title" content="<%= author %>"> <meta name="twitter:description" content="<%= channel.description %>"> -<meta name="twitter:image" content="/ggpht<%= channel_profile_pic %>"> +<meta name="twitter:image" content="<%= HOST_URL %>/ggpht<%= channel_profile_pic %>"> <link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/channel/<%= ucid %>" /> <%- end -%> diff --git a/src/invidious/views/user/preferences.ecr b/src/invidious/views/user/preferences.ecr index 55349c5a..b89c73ca 100644 --- a/src/invidious/views/user/preferences.ecr +++ b/src/invidious/views/user/preferences.ecr @@ -310,7 +310,7 @@ <div class="pure-control-group"> <label for="modified_source_code_url"><%= translate(locale, "adminprefs_modified_source_code_url_label") %></label> - <input name="modified_source_code_url" id="modified_source_code_url" type="input" <% if CONFIG.modified_source_code_url %>checked<% end %>> + <input name="modified_source_code_url" id="modified_source_code_url" type="url" value="<%= CONFIG.modified_source_code_url %>"> </div> <% end %> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 7a1cf2c3..9e7467dd 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -10,7 +10,7 @@ <meta property="og:site_name" content="<%= author %> | Invidious"> <meta property="og:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>"> <meta property="og:title" content="<%= title %>"> -<meta property="og:image" content="/vi/<%= video.id %>/maxres.jpg"> +<meta property="og:image" content="<%= HOST_URL %>/vi/<%= video.id %>/maxres.jpg"> <meta property="og:description" content="<%= HTML.escape(video.short_description) %>"> <meta property="og:type" content="video.other"> <meta property="og:video:url" content="<%= HOST_URL %>/embed/<%= video.id %>"> diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 727ce9a3..c8b037c8 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -5,9 +5,6 @@ module YoutubeAPI extend self - private DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" - private ANDROID_API_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w" - # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history private ANDROID_APP_VERSION = "19.14.42" private ANDROID_USER_AGENT = "com.google.android.youtube/19.14.42 (Linux; U; Android 12; US) gzip" @@ -52,7 +49,6 @@ module YoutubeAPI name: "WEB", name_proto: "1", version: "2.20240304.00.00", - api_key: DEFAULT_API_KEY, screen: "WATCH_FULL_SCREEN", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -62,7 +58,6 @@ module YoutubeAPI name: "WEB_EMBEDDED_PLAYER", name_proto: "56", version: "1.20240303.00.00", - api_key: DEFAULT_API_KEY, screen: "EMBED", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -72,7 +67,6 @@ module YoutubeAPI name: "MWEB", name_proto: "2", version: "2.20240304.08.00", - api_key: DEFAULT_API_KEY, os_name: "Android", os_version: ANDROID_VERSION, platform: "MOBILE", @@ -81,7 +75,6 @@ module YoutubeAPI name: "WEB", name_proto: "1", version: "2.20240304.00.00", - api_key: DEFAULT_API_KEY, screen: "EMBED", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -94,7 +87,6 @@ module YoutubeAPI name: "ANDROID", name_proto: "3", version: ANDROID_APP_VERSION, - api_key: ANDROID_API_KEY, android_sdk_version: ANDROID_SDK_VERSION, user_agent: ANDROID_USER_AGENT, os_name: "Android", @@ -105,13 +97,11 @@ module YoutubeAPI name: "ANDROID_EMBEDDED_PLAYER", name_proto: "55", version: ANDROID_APP_VERSION, - api_key: "AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw", }, ClientType::AndroidScreenEmbed => { name: "ANDROID", name_proto: "3", version: ANDROID_APP_VERSION, - api_key: DEFAULT_API_KEY, screen: "EMBED", android_sdk_version: ANDROID_SDK_VERSION, user_agent: ANDROID_USER_AGENT, @@ -123,7 +113,6 @@ module YoutubeAPI name: "ANDROID_TESTSUITE", name_proto: "30", version: ANDROID_TS_APP_VERSION, - api_key: ANDROID_API_KEY, android_sdk_version: ANDROID_SDK_VERSION, user_agent: ANDROID_TS_USER_AGENT, os_name: "Android", @@ -137,7 +126,6 @@ module YoutubeAPI name: "IOS", name_proto: "5", version: IOS_APP_VERSION, - api_key: "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc", user_agent: IOS_USER_AGENT, device_make: "Apple", device_model: "iPhone14,5", @@ -149,7 +137,6 @@ module YoutubeAPI name: "IOS_MESSAGES_EXTENSION", name_proto: "66", version: IOS_APP_VERSION, - api_key: DEFAULT_API_KEY, user_agent: IOS_USER_AGENT, device_make: "Apple", device_model: "iPhone14,5", @@ -161,7 +148,6 @@ module YoutubeAPI name: "IOS_MUSIC", name_proto: "26", version: "6.42", - api_key: "AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s", user_agent: "com.google.ios.youtubemusic/6.42 (iPhone14,5; U; CPU iOS 17_4 like Mac OS X;)", device_make: "Apple", device_model: "iPhone14,5", @@ -176,13 +162,11 @@ module YoutubeAPI name: "TVHTML5", name_proto: "7", version: "7.20240304.10.00", - api_key: DEFAULT_API_KEY, }, ClientType::TvHtml5ScreenEmbed => { name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER", name_proto: "85", version: "2.0", - api_key: DEFAULT_API_KEY, screen: "EMBED", }, } @@ -238,11 +222,6 @@ module YoutubeAPI end # :ditto: - def api_key : String - HARDCODED_CLIENTS[@client_type][:api_key] - end - - # :ditto: def screen : String HARDCODED_CLIENTS[@client_type][:screen]? || "" end @@ -606,7 +585,7 @@ module YoutubeAPI client_config ||= DEFAULT_CLIENT_CONFIG # Query parameters - url = "#{endpoint}?key=#{client_config.api_key}&prettyPrint=false" + url = "#{endpoint}?prettyPrint=false" headers = HTTP::Headers{ "Content-Type" => "application/json; charset=UTF-8", |