diff options
| -rw-r--r-- | .ameba.yml | 58 | ||||
| -rw-r--r-- | .github/workflows/build-nightly-container.yml (renamed from .github/workflows/container-release.yml) | 10 | ||||
| -rw-r--r-- | .github/workflows/build-stable-container.yml | 90 | ||||
| -rw-r--r-- | .github/workflows/ci.yml | 24 | ||||
| -rw-r--r-- | kubernetes/.gitignore | 1 | ||||
| -rw-r--r-- | kubernetes/Chart.lock | 6 | ||||
| -rw-r--r-- | kubernetes/Chart.yaml | 22 | ||||
| -rw-r--r-- | kubernetes/README.md | 42 | ||||
| -rw-r--r-- | kubernetes/templates/_helpers.tpl | 16 | ||||
| -rw-r--r-- | kubernetes/templates/configmap.yaml | 11 | ||||
| -rw-r--r-- | kubernetes/templates/deployment.yaml | 61 | ||||
| -rw-r--r-- | kubernetes/templates/hpa.yaml | 18 | ||||
| -rw-r--r-- | kubernetes/templates/service.yaml | 20 | ||||
| -rw-r--r-- | kubernetes/values.yaml | 61 | ||||
| -rw-r--r-- | shard.lock | 2 | ||||
| -rw-r--r-- | shard.yml | 2 | ||||
| -rw-r--r-- | src/invidious/routes/api/v1/videos.cr | 9 | ||||
| -rw-r--r-- | src/invidious/videos/transcript.cr | 111 |
18 files changed, 220 insertions, 344 deletions
@@ -20,6 +20,9 @@ Lint/ShadowingOuterLocalVar: Excluded: - src/invidious/helpers/tokens.cr +Lint/NotNil: + Enabled: false + # # Style @@ -31,6 +34,13 @@ Style/RedundantBegin: Style/RedundantReturn: Enabled: false +Style/ParenthesesAroundCondition: + Enabled: false + +# This requires a rewrite of most data structs (and their usage) in Invidious. +Style/QueryBoolMethods: + Enabled: false + # # Metrics @@ -39,50 +49,4 @@ Style/RedundantReturn: # Ignore function complexity (number of if/else & case/when branches) # For some functions that can hardly be simplified for now Metrics/CyclomaticComplexity: - Excluded: - # get_about_info(ucid, locale) => [17/10] - - src/invidious/channels/about.cr - - # fetch_channel_community(ucid, continuation, ...) => [34/10] - - src/invidious/channels/community.cr - - # create_notification_stream(env, topics, connection_channel) => [14/10] - - src/invidious/helpers/helpers.cr:84:5 - - # get_index(plural_form, count) => [25/10] - - src/invidious/helpers/i18next.cr - - # call(context) => [18/10] - - src/invidious/helpers/static_file_handler.cr - - # show(env) => [38/10] - - src/invidious/routes/embed.cr - - # get_video_playback(env) => [45/10] - - src/invidious/routes/video_playback.cr - - # handle(env) => [40/10] - - src/invidious/routes/watch.cr - - # playlist_ajax(env) => [24/10] - - src/invidious/routes/playlists.cr - - # fetch_youtube_comments(id, cursor, ....) => [40/10] - # template_youtube_comments(comments, locale, ...) => [16/10] - # content_to_comment_html(content) => [14/10] - - src/invidious/comments.cr - - # to_json(locale, json) => [21/10] - # extract_video_info(video_id, ...) => [44/10] - # process_video_params(query, preferences) => [20/10] - - src/invidious/videos.cr - - - -#src/invidious/playlists.cr:327:5 -#[C] Metrics/CyclomaticComplexity: Cyclomatic complexity too high [19/10] -# fetch_playlist(plid : String) - -#src/invidious/playlists.cr:436:5 -#[C] Metrics/CyclomaticComplexity: Cyclomatic complexity too high [11/10] -# extract_playlist_videos(initial_data : Hash(String, JSON::Any)) + Enabled: false diff --git a/.github/workflows/container-release.yml b/.github/workflows/build-nightly-container.yml index e44ac200..bee27600 100644 --- a/.github/workflows/container-release.yml +++ b/.github/workflows/build-nightly-container.yml @@ -1,4 +1,4 @@ -name: Build and release container +name: Build and release container directly from master on: push: @@ -24,9 +24,9 @@ jobs: uses: actions/checkout@v4 - name: Install Crystal - uses: crystal-lang/install-crystal@v1.8.0 + uses: crystal-lang/install-crystal@v1.8.2 with: - crystal: 1.9.2 + crystal: 1.12.2 - name: Run lint run: | @@ -58,7 +58,7 @@ jobs: images: quay.io/invidious/invidious tags: | type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} - type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + type=raw,value=master,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} labels: | quay.expires-after=12w @@ -83,7 +83,7 @@ jobs: suffix=-arm64 tags: | type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} - type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + type=raw,value=master,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} labels: | quay.expires-after=12w diff --git a/.github/workflows/build-stable-container.yml b/.github/workflows/build-stable-container.yml new file mode 100644 index 00000000..b5fbc705 --- /dev/null +++ b/.github/workflows/build-stable-container.yml @@ -0,0 +1,90 @@ +name: Build and release container + +on: + push: + tags: + - "v*" + +jobs: + release: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Crystal + uses: crystal-lang/install-crystal@v1.8.2 + with: + crystal: 1.12.2 + + - name: Run lint + run: | + if ! crystal tool format --check; then + crystal tool format + git diff + exit 1 + fi + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to registry + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSWORD }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: quay.io/invidious/invidious + tags: | + type=semver,pattern={{version}} + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + labels: | + quay.expires-after=12w + + - name: Build and push Docker AMD64 image for Push Event + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile + platforms: linux/amd64 + labels: ${{ steps.meta.outputs.labels }} + push: true + tags: ${{ steps.meta.outputs.tags }} + build-args: | + "release=1" + + - name: Docker meta + id: meta-arm64 + uses: docker/metadata-action@v5 + with: + images: quay.io/invidious/invidious + flavor: | + suffix=-arm64 + tags: | + type=semver,pattern={{version}} + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + labels: | + quay.expires-after=12w + + - name: Build and push Docker ARM64 image for Push Event + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile.arm64 + platforms: linux/arm64/v8 + labels: ${{ steps.meta-arm64.outputs.labels }} + push: true + tags: ${{ steps.meta-arm64.outputs.tags }} + build-args: | + "release=1" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 057e4d61..eb18f639 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -124,4 +124,28 @@ jobs: - name: Test Docker run: while curl -Isf http://localhost:3000; do sleep 1; done + ameba_lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Install Crystal + uses: crystal-lang/install-crystal@v1.8.0 + with: + crystal: latest + + - name: Cache Shards + uses: actions/cache@v3 + with: + path: | + ./lib + ./bin + key: shards-${{ hashFiles('shard.lock') }} + + - name: Install Shards + run: shards install + - name: Run Ameba linter + run: bin/ameba diff --git a/kubernetes/.gitignore b/kubernetes/.gitignore deleted file mode 100644 index 0ad51707..00000000 --- a/kubernetes/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/charts/*.tgz diff --git a/kubernetes/Chart.lock b/kubernetes/Chart.lock deleted file mode 100644 index ef12b0b6..00000000 --- a/kubernetes/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: postgresql - repository: https://charts.bitnami.com/bitnami/ - version: 12.11.1 -digest: sha256:3c10008175c4f5c1cec38782f5a7316154b89074c77ebbd9bcc4be4f5ff21122 -generated: "2023-09-14T22:40:43.171275362Z" diff --git a/kubernetes/Chart.yaml b/kubernetes/Chart.yaml deleted file mode 100644 index d22f6254..00000000 --- a/kubernetes/Chart.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: v2 -name: invidious -description: Invidious is an alternative front-end to YouTube -version: 1.1.1 -appVersion: 0.20.1 -keywords: -- youtube -- proxy -- video -- privacy -home: https://invidio.us/ -icon: https://raw.githubusercontent.com/iv-org/invidious/05988c1c49851b7d0094fca16aeaf6382a7f64ab/assets/favicon-32x32.png -sources: -- https://github.com/iv-org/invidious -maintainers: -- name: Leon Klingele - email: mail@leonklingele.de -dependencies: -- name: postgresql - version: ~12.11.0 - repository: "https://charts.bitnami.com/bitnami/" -engine: gotpl diff --git a/kubernetes/README.md b/kubernetes/README.md index 35478f99..e71f6a86 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -1,41 +1 @@ -# Invidious Helm chart - -Easily deploy Invidious to Kubernetes. - -## Installing Helm chart - -```sh -# Build Helm dependencies -$ helm dep build - -# Add PostgreSQL init scripts -$ kubectl create configmap invidious-postgresql-init \ - --from-file=../config/sql/channels.sql \ - --from-file=../config/sql/videos.sql \ - --from-file=../config/sql/channel_videos.sql \ - --from-file=../config/sql/users.sql \ - --from-file=../config/sql/session_ids.sql \ - --from-file=../config/sql/nonces.sql \ - --from-file=../config/sql/annotations.sql \ - --from-file=../config/sql/playlists.sql \ - --from-file=../config/sql/playlist_videos.sql - -# Install Helm app to your Kubernetes cluster -$ helm install invidious ./ -``` - -## Upgrading - -```sh -# Upgrading is easy, too! -$ helm upgrade invidious ./ -``` - -## Uninstall - -```sh -# Get rid of everything (except database) -$ helm delete invidious - -# To also delete the database, remove all invidious-postgresql PVCs -``` +The Helm chart has moved to a dedicated GitHub repository: https://github.com/iv-org/invidious-helm-chart/tree/master/invidious
\ No newline at end of file diff --git a/kubernetes/templates/_helpers.tpl b/kubernetes/templates/_helpers.tpl deleted file mode 100644 index 52158b78..00000000 --- a/kubernetes/templates/_helpers.tpl +++ /dev/null @@ -1,16 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "invidious.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "invidious.fullname" -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} diff --git a/kubernetes/templates/configmap.yaml b/kubernetes/templates/configmap.yaml deleted file mode 100644 index 58542a31..00000000 --- a/kubernetes/templates/configmap.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ template "invidious.fullname" . }} - labels: - app: {{ template "invidious.name" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: {{ .Release.Name }} -data: - INVIDIOUS_CONFIG: | -{{ toYaml .Values.config | indent 4 }} diff --git a/kubernetes/templates/deployment.yaml b/kubernetes/templates/deployment.yaml deleted file mode 100644 index bb0b832f..00000000 --- a/kubernetes/templates/deployment.yaml +++ /dev/null @@ -1,61 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "invidious.fullname" . }} - labels: - app: {{ template "invidious.name" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: {{ .Release.Name }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - app: {{ template "invidious.name" . }} - release: {{ .Release.Name }} - template: - metadata: - labels: - app: {{ template "invidious.name" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: {{ .Release.Name }} - spec: - securityContext: - runAsUser: {{ .Values.securityContext.runAsUser }} - runAsGroup: {{ .Values.securityContext.runAsGroup }} - fsGroup: {{ .Values.securityContext.fsGroup }} - initContainers: - - name: wait-for-postgresql - image: postgres - args: - - /bin/sh - - -c - - until pg_isready -h {{ .Values.config.db.host }} -p {{ .Values.config.db.port }} -U {{ .Values.config.db.user }}; do echo waiting for database; sleep 2; done; - containers: - - name: {{ .Chart.Name }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - containerPort: 3000 - env: - - name: INVIDIOUS_CONFIG - valueFrom: - configMapKeyRef: - key: INVIDIOUS_CONFIG - name: {{ template "invidious.fullname" . }} - securityContext: - allowPrivilegeEscalation: {{ .Values.securityContext.allowPrivilegeEscalation }} - capabilities: - drop: - - ALL - resources: -{{ toYaml .Values.resources | indent 10 }} - readinessProbe: - httpGet: - port: 3000 - path: / - livenessProbe: - httpGet: - port: 3000 - path: / - initialDelaySeconds: 15 - restartPolicy: Always diff --git a/kubernetes/templates/hpa.yaml b/kubernetes/templates/hpa.yaml deleted file mode 100644 index c6fbefe2..00000000 --- a/kubernetes/templates/hpa.yaml +++ /dev/null @@ -1,18 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ template "invidious.fullname" . }} - labels: - app: {{ template "invidious.name" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: {{ .Release.Name }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ template "invidious.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - targetCPUUtilizationPercentage: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} -{{- end }} diff --git a/kubernetes/templates/service.yaml b/kubernetes/templates/service.yaml deleted file mode 100644 index 01454d4e..00000000 --- a/kubernetes/templates/service.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ template "invidious.fullname" . }} - labels: - app: {{ template "invidious.name" . }} - chart: {{ .Chart.Name }} - release: {{ .Release.Name }} -spec: - type: {{ .Values.service.type }} - ports: - - name: http - port: {{ .Values.service.port }} - targetPort: 3000 - selector: - app: {{ template "invidious.name" . }} - release: {{ .Release.Name }} -{{- if .Values.service.loadBalancerIP }} - loadBalancerIP: {{ .Values.service.loadBalancerIP }} -{{- end }} diff --git a/kubernetes/values.yaml b/kubernetes/values.yaml deleted file mode 100644 index 5000c2b6..00000000 --- a/kubernetes/values.yaml +++ /dev/null @@ -1,61 +0,0 @@ -name: invidious - -image: - repository: quay.io/invidious/invidious - tag: latest - pullPolicy: Always - -replicaCount: 1 - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 16 - targetCPUUtilizationPercentage: 50 - -service: - type: ClusterIP - port: 3000 - #loadBalancerIP: - -resources: {} - #requests: - # cpu: 100m - # memory: 64Mi - #limits: - # cpu: 800m - # memory: 512Mi - -securityContext: - allowPrivilegeEscalation: false - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - -# See https://github.com/bitnami/charts/tree/master/bitnami/postgresql -postgresql: - image: - tag: 13 - auth: - username: kemal - password: kemal - database: invidious - primary: - initdb: - username: kemal - password: kemal - scriptsConfigMap: invidious-postgresql-init - -# Adapted from ../config/config.yml -config: - channel_threads: 1 - feed_threads: 1 - db: - user: kemal - password: kemal - host: invidious-postgresql - port: 5432 - dbname: invidious - full_refresh: false - https_only: false - domain: @@ -2,7 +2,7 @@ version: 2.0 shards: ameba: git: https://github.com/crystal-ameba/ameba.git - version: 1.5.0 + version: 1.6.1 athena-negotiation: git: https://github.com/athena-framework/negotiation.git @@ -35,7 +35,7 @@ development_dependencies: version: ~> 0.10.4 ameba: github: crystal-ameba/ameba - version: ~> 1.5.0 + version: ~> 1.6.1 crystal: ">= 1.0.0, < 2.0.0" diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 9281f4dd..faff2f59 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -89,9 +89,14 @@ module Invidious::Routes::API::V1::Videos if CONFIG.use_innertube_for_captions params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated) - initial_data = YoutubeAPI.get_transcript(params) - webvtt = Invidious::Videos::Transcript.convert_transcripts_to_vtt(initial_data, caption.language_code) + transcript = Invidious::Videos::Transcript.from_raw( + YoutubeAPI.get_transcript(params), + caption.language_code, + caption.auto_generated + ) + + webvtt = transcript.to_vtt else # Timedtext API handling url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr index dac00eea..9cd064c5 100644 --- a/src/invidious/videos/transcript.cr +++ b/src/invidious/videos/transcript.cr @@ -1,8 +1,26 @@ module Invidious::Videos - # Namespace for methods primarily relating to Transcripts - module Transcript - record TranscriptLine, start_ms : Time::Span, end_ms : Time::Span, line : String + # A `Transcripts` struct encapsulates a sequence of lines that together forms the whole transcript for a given YouTube video. + # These lines can be categorized into two types: section headings and regular lines representing content from the video. + struct Transcript + # Types + record HeadingLine, start_ms : Time::Span, end_ms : Time::Span, line : String + record RegularLine, start_ms : Time::Span, end_ms : Time::Span, line : String + alias TranscriptLine = HeadingLine | RegularLine + property lines : Array(TranscriptLine) + + property language_code : String + property auto_generated : Bool + + # User friendly label for the current transcript. + # Example: "English (auto-generated)" + property label : String + + # Initializes a new Transcript struct with the contents and associated metadata describing it + def initialize(@lines : Array(TranscriptLine), @language_code : String, @auto_generated : Bool, @label : String) + end + + # Generates a protobuf string to fetch the requested transcript from YouTube def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String kind = auto_generated ? "asr" : "" @@ -30,48 +48,79 @@ module Invidious::Videos return params end - def self.convert_transcripts_to_vtt(initial_data : Hash(String, JSON::Any), target_language : String) : String - # Convert into array of TranscriptLine - lines = self.parse(initial_data) + # Constructs a Transcripts struct from the initial YouTube response + def self.from_raw(initial_data : Hash(String, JSON::Any), language_code : String, auto_generated : Bool) + transcript_panel = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", + "content", "transcriptSearchPanelRenderer") - settings_field = { - "Kind" => "captions", - "Language" => target_language, - } + segment_list = transcript_panel.dig("body", "transcriptSegmentListRenderer") - # Taken from Invidious::Videos::Captions::Metadata.timedtext_to_vtt() - vtt = WebVTT.build(settings_field) do |vtt| - lines.each do |line| - vtt.cue(line.start_ms, line.end_ms, line.line) - end + if !segment_list["initialSegments"]? + raise NotFoundException.new("Requested transcript does not exist") end - return vtt - end + # Extract user-friendly label for the current transcript + + footer_language_menu = transcript_panel.dig?( + "footer", "transcriptFooterRenderer", "languageMenu", "sortFilterSubMenuRenderer", "subMenuItems" + ) - private def self.parse(initial_data : Hash(String, JSON::Any)) - body = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", - "content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer", - "initialSegments").as_a + if footer_language_menu + label = footer_language_menu.as_a.select(&.["selected"].as_bool)[0]["title"].as_s + else + label = language_code + end + + # Extract transcript lines + + initial_segments = segment_list["initialSegments"].as_a lines = [] of TranscriptLine - body.each do |line| - # Transcript section headers. They are not apart of the captions and as such we can safely skip them. - if line.as_h.has_key?("transcriptSectionHeaderRenderer") - next + + initial_segments.each do |line| + if unpacked_line = line["transcriptSectionHeaderRenderer"]? + line_type = HeadingLine + else + unpacked_line = line["transcriptSegmentRenderer"] + line_type = RegularLine end - line = line["transcriptSegmentRenderer"] + start_ms = unpacked_line["startMs"].as_s.to_i.millisecond + end_ms = unpacked_line["endMs"].as_s.to_i.millisecond + text = extract_text(unpacked_line["snippet"]) || "" + + lines << line_type.new(start_ms, end_ms, text) + end + + return Transcript.new( + lines: lines, + language_code: language_code, + auto_generated: auto_generated, + label: label + ) + end - start_ms = line["startMs"].as_s.to_i.millisecond - end_ms = line["endMs"].as_s.to_i.millisecond + # Converts transcript lines to a WebVTT file + # + # This is used within Invidious to replace subtitles + # as to workaround YouTube's rate-limited timedtext endpoint. + def to_vtt + settings_field = { + "Kind" => "captions", + "Language" => @language_code, + } - text = extract_text(line["snippet"]) || "" + vtt = WebVTT.build(settings_field) do |vtt| + @lines.each do |line| + # Section headers are excluded from the VTT conversion as to + # match the regular captions returned from YouTube as much as possible + next if line.is_a? HeadingLine - lines << TranscriptLine.new(start_ms, end_ms, text) + vtt.cue(line.start_ms, line.end_ms, line.line) + end end - return lines + return vtt end end end |