diff options
| -rw-r--r-- | config/sql/videos.sql | 2 | ||||
| -rw-r--r-- | docker/Dockerfile.arm64 | 4 | ||||
| -rw-r--r-- | src/invidious/channels/about.cr | 139 | ||||
| -rw-r--r-- | src/invidious/jobs/refresh_channels_job.cr | 4 | ||||
| -rw-r--r-- | src/invidious/routes/api/v1/channels.cr | 2 |
5 files changed, 67 insertions, 84 deletions
diff --git a/config/sql/videos.sql b/config/sql/videos.sql index 7040703c..55da3967 100644 --- a/config/sql/videos.sql +++ b/config/sql/videos.sql @@ -2,7 +2,7 @@ -- DROP TABLE public.videos; -CREATE TABLE IF NOT EXISTS public.videos +CREATE UNLOGGED TABLE IF NOT EXISTS public.videos ( id text NOT NULL, info text, diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index 4d5d46bf..d2955c7c 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -1,4 +1,4 @@ -FROM alpine:edge AS builder +FROM alpine:3.15 AS builder RUN apk add --no-cache 'crystal=1.2.2-r0' shards sqlite-static yaml-static yaml-dev libxml2-dev zlib-static openssl-libs-static openssl-dev musl-dev ARG release @@ -29,7 +29,7 @@ RUN if [ ${release} == 1 ] ; then \ --link-flags "-lxml2 -llzma"; \ fi -FROM alpine:edge +FROM alpine:3.15 RUN apk add --no-cache librsvg ttf-opensans WORKDIR /invidious RUN addgroup -g 1000 -S invidious && \ diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index d93ee681..8cae7ae2 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -1,33 +1,26 @@ # TODO: Refactor into either SearchChannel or InvidiousChannel -struct AboutChannel - include DB::Serializable - - property ucid : String - property author : String - property auto_generated : Bool - property author_url : String - property author_thumbnail : String - property banner : String? - property description_html : String - property total_views : Int64 - property sub_count : Int32 - property joined : Time - property is_family_friendly : Bool - property allowed_regions : Array(String) - property related_channels : Array(AboutRelatedChannel) - property tabs : Array(String) -end - -struct AboutRelatedChannel - include DB::Serializable - - property ucid : String - property author : String - property author_url : String - property author_thumbnail : String -end - -def get_about_info(ucid, locale) +record AboutChannel, + ucid : String, + author : String, + auto_generated : Bool, + author_url : String, + author_thumbnail : String, + banner : String?, + description_html : String, + total_views : Int64, + sub_count : Int32, + joined : Time, + is_family_friendly : Bool, + allowed_regions : Array(String), + tabs : Array(String) + +record AboutRelatedChannel, + ucid : String, + author : String, + author_url : String, + author_thumbnail : String + +def get_about_info(ucid, locale) : AboutChannel begin # "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"} initdata = YoutubeAPI.browse(browse_id: ucid, params: "EgVhYm91dA==") @@ -63,8 +56,6 @@ def get_about_info(ucid, locale) is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map(&.as_s) - - related_channels = [] of AboutRelatedChannel else author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s @@ -85,38 +76,6 @@ def get_about_info(ucid, locale) is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map(&.as_s) - - related_channels = initdata["contents"]["twoColumnBrowseResultsRenderer"] - .["secondaryContents"]?.try &.["browseSecondaryContentsRenderer"]["contents"][0]? - .try &.["verticalChannelSectionRenderer"]?.try &.["items"]?.try &.as_a.map do |node| - renderer = node["miniChannelRenderer"]? - related_id = renderer.try &.["channelId"]?.try &.as_s? - related_id ||= "" - - related_title = renderer.try &.["title"]?.try &.["simpleText"]?.try &.as_s? - related_title ||= "" - - related_author_url = renderer.try &.["navigationEndpoint"]?.try &.["commandMetadata"]?.try &.["webCommandMetadata"]? - .try &.["url"]?.try &.as_s? - related_author_url ||= "" - - related_author_thumbnails = renderer.try &.["thumbnail"]?.try &.["thumbnails"]?.try &.as_a? - related_author_thumbnails ||= [] of JSON::Any - - related_author_thumbnail = "" - if related_author_thumbnails.size > 0 - related_author_thumbnail = related_author_thumbnails[-1]["url"]?.try &.as_s? - related_author_thumbnail ||= "" - end - - AboutRelatedChannel.new({ - ucid: related_id, - author: related_title, - author_url: related_author_url, - author_thumbnail: related_author_thumbnail, - }) - end - related_channels ||= [] of AboutRelatedChannel end total_views = 0_i64 @@ -155,20 +114,44 @@ def get_about_info(ucid, locale) sub_count = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s? .try { |text| short_text_to_number(text.split(" ")[0]) } || 0 - AboutChannel.new({ - ucid: ucid, - author: author, - auto_generated: auto_generated, - author_url: author_url, - author_thumbnail: author_thumbnail, - banner: banner, - description_html: description_html, - total_views: total_views, - sub_count: sub_count, - joined: joined, + AboutChannel.new( + ucid: ucid, + author: author, + auto_generated: auto_generated, + author_url: author_url, + author_thumbnail: author_thumbnail, + banner: banner, + description_html: description_html, + total_views: total_views, + sub_count: sub_count, + joined: joined, is_family_friendly: is_family_friendly, - allowed_regions: allowed_regions, - related_channels: related_channels, - tabs: tabs, - }) + allowed_regions: allowed_regions, + tabs: tabs, + ) +end + +def fetch_related_channels(about_channel : AboutChannel) : Array(AboutRelatedChannel) + # params is {"2:string":"channels"} encoded + channels = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D") + + tabs = channels.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs").try(&.as_a?) || [] of JSON::Any + tab = tabs.find { |tab| tab.dig?("tabRenderer", "title").try(&.as_s?) == "Channels" } + return [] of AboutRelatedChannel if tab.nil? + + items = tab.dig?("tabRenderer", "content", "sectionListRenderer", "contents", 0, "itemSectionRenderer", "contents", 0, "gridRenderer", "items").try(&.as_a?) || [] of JSON::Any + + items.map do |item| + related_id = item.dig("gridChannelRenderer", "channelId").as_s + related_title = item.dig("gridChannelRenderer", "title", "simpleText").as_s + related_author_url = item.dig("gridChannelRenderer", "navigationEndpoint", "browseEndpoint", "canonicalBaseUrl").as_s + related_author_thumbnail = item.dig("gridChannelRenderer", "thumbnail", "thumbnails", -1, "url").as_s + + AboutRelatedChannel.new( + ucid: related_id, + author: related_title, + author_url: related_author_url, + author_thumbnail: related_author_thumbnail, + ) + end end diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr index c224c745..941089c1 100644 --- a/src/invidious/jobs/refresh_channels_job.cr +++ b/src/invidious/jobs/refresh_channels_job.cr @@ -13,7 +13,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob loop do LOGGER.debug("RefreshChannelsJob: Refreshing all channels") - db.query("SELECT id FROM channels ORDER BY updated") do |rs| + PG_DB.query("SELECT id FROM channels ORDER BY updated") do |rs| rs.each do id = rs.read(String) @@ -30,7 +30,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob spawn do begin LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel") - channel = fetch_channel(id, db, CONFIG.full_refresh) + channel = fetch_channel(id, CONFIG.full_refresh) lim_fibers = max_fibers diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 8b6df3fd..322ac42e 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -96,7 +96,7 @@ module Invidious::Routes::API::V1::Channels json.field "relatedChannels" do json.array do - channel.related_channels.each do |related_channel| + fetch_related_channels(channel).each do |related_channel| json.object do json.field "author", related_channel.author json.field "authorId", related_channel.ucid |