diff options
Diffstat (limited to 'src')
39 files changed, 964 insertions, 724 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index 2874cc71..d4f8e0fb 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -34,6 +34,7 @@ require "protodec/utils" require "./invidious/database/*" require "./invidious/database/migrations/*" +require "./invidious/http_server/*" require "./invidious/helpers/*" require "./invidious/yt_backend/*" require "./invidious/frontend/*" @@ -48,6 +49,13 @@ require "./invidious/search/*" require "./invidious/routes/**" require "./invidious/jobs/**" +# Declare the base namespace for invidious +module Invidious +end + +# Simple alias to make code easier to read +alias IV = Invidious + CONFIG = Config.load HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32) @@ -172,7 +180,7 @@ if CONFIG.popular_enabled Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB) end -CONNECTION_CHANNEL = Channel({Bool, Channel(PQ::Notification)}).new(32) +CONNECTION_CHANNEL = ::Channel({Bool, ::Channel(PQ::Notification)}).new(32) Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL, CONFIG.database_url) Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index 4c442959..0054f8f2 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -16,12 +16,6 @@ record AboutChannel, tabs : Array(String), verified : Bool -record AboutRelatedChannel, - ucid : String, - author : String, - author_url : String, - author_thumbnail : String - def get_about_info(ucid, locale) : AboutChannel begin # "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"} @@ -100,34 +94,46 @@ def get_about_info(ucid, locale) : AboutChannel total_views = 0_i64 joined = Time.unix(0) - tabs = [] of String - - tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?.try &.as_a? - if !tabs_json.nil? - # Retrieve information from the tabs array. The index we are looking for varies between channels. - tabs_json.each do |node| - # Try to find the about section which is located in only one of the tabs. - channel_about_meta = node["tabRenderer"]?.try &.["content"]?.try &.["sectionListRenderer"]? - .try &.["contents"]?.try &.[0]?.try &.["itemSectionRenderer"]?.try &.["contents"]? - .try &.[0]?.try &.["channelAboutFullMetadataRenderer"]? - - if !channel_about_meta.nil? - total_views = channel_about_meta["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 - - # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. - joined = channel_about_meta["joinedDateText"]?.try &.["runs"]?.try &.as_a.reduce("") { |acc, nd| acc + nd["text"].as_s } - .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) - - # Normal Auto-generated channels - # https://support.google.com/youtube/answer/2579942 - # For auto-generated channels, channel_about_meta only has ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] - if (channel_about_meta["primaryLinks"]?.try &.size || 0) == 1 && (channel_about_meta["primaryLinks"][0]?) && - (channel_about_meta["primaryLinks"][0]["title"]?.try &.["simpleText"]?.try &.as_s? || "") == "Auto-generated by YouTube" - auto_generated = true - end - end + tab_names = [] of String + + if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]? + # Get the name of the tabs available on this channel + tab_names = tabs_json.as_a.compact_map do |entry| + name = entry.dig?("tabRenderer", "title").try &.as_s.downcase + + # This is a small fix to not add extra code on the HTML side + # I.e, the URL for the "live" tab is .../streams, so use "streams" + # everywhere for the sake of simplicity + (name == "live") ? "streams" : name + end + + # Get the currently active tab ("About") + about_tab = extract_selected_tab(tabs_json) + + # Try to find the about metadata section + channel_about_meta = about_tab.dig?( + "content", + "sectionListRenderer", "contents", 0, + "itemSectionRenderer", "contents", 0, + "channelAboutFullMetadataRenderer" + ) + + if !channel_about_meta.nil? + total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 + + # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. + joined = extract_text(channel_about_meta["joinedDateText"]?) + .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) + + # Normal Auto-generated channels + # https://support.google.com/youtube/answer/2579942 + # For auto-generated channels, channel_about_meta only has + # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] + auto_generated = ( + (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \ + extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" + ) end - tabs = tabs_json.reject { |node| node["tabRenderer"]?.nil? }.map(&.["tabRenderer"]["title"].as_s.downcase) end sub_count = initdata @@ -148,46 +154,20 @@ def get_about_info(ucid, locale) : AboutChannel joined: joined, is_family_friendly: is_family_friendly, allowed_regions: allowed_regions, - tabs: tabs, + tabs: tab_names, verified: author_verified || false, ) end -def fetch_related_channels(about_channel : AboutChannel) : Array(AboutRelatedChannel) - # params is {"2:string":"channels"} encoded - channels = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D") - - tabs = channels.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs").try(&.as_a?) || [] of JSON::Any - tab = tabs.find(&.dig?("tabRenderer", "title").try(&.as_s?).try(&.== "Channels")) - - return [] of AboutRelatedChannel if tab.nil? - - items = tab.dig?( - "tabRenderer", "content", - "sectionListRenderer", "contents", 0, - "itemSectionRenderer", "contents", 0, - "gridRenderer", "items" - ).try &.as_a? - - related = [] of AboutRelatedChannel - return related if (items.nil? || items.empty?) - - items.each do |item| - renderer = item["gridChannelRenderer"]? - next if !renderer - - related_id = renderer.dig("channelId").as_s - related_title = renderer.dig("title", "simpleText").as_s - related_author_url = renderer.dig("navigationEndpoint", "browseEndpoint", "canonicalBaseUrl").as_s - related_author_thumbnail = HelperExtractors.get_thumbnails(renderer) - - related << AboutRelatedChannel.new( - ucid: related_id, - author: related_title, - author_url: related_author_url, - author_thumbnail: related_author_thumbnail, - ) +def fetch_related_channels(about_channel : AboutChannel, continuation : String? = nil) : {Array(SearchChannel), String?} + if continuation.nil? + # params is {"2:string":"channels"} encoded + initial_data = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D") + else + initial_data = YoutubeAPI.browse(continuation) end - return related + items, continuation = extract_items(initial_data) + + return items.select(SearchChannel), continuation end diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index e3d3d9ee..63dd2194 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -180,11 +180,16 @@ def fetch_channel(ucid, pull_all_videos : Bool) LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}") - page = 1 + channel = InvidiousChannel.new({ + id: ucid, + author: author, + updated: Time.utc, + deleted: false, + subscribed: nil, + }) LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page") - initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated) - videos = extract_videos(initial_data, author, ucid) + videos, continuation = IV::Channel::Tabs.get_videos(channel) LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed") rss.xpath_nodes("//feed/entry").each do |entry| @@ -197,7 +202,9 @@ def fetch_channel(ucid, pull_all_videos : Bool) views = entry.xpath_node("group/community/statistics").try &.["views"]?.try &.to_i64? views ||= 0_i64 - channel_video = videos.select { |video| video.id == video_id }[0]? + channel_video = videos + .select(SearchVideo) + .select(&.id.== video_id)[0]? length_seconds = channel_video.try &.length_seconds length_seconds ||= 0 @@ -228,58 +235,56 @@ def fetch_channel(ucid, pull_all_videos : Bool) if was_insert LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions") - Invidious::Database::Users.add_notification(video) + if CONFIG.enable_user_notifications + Invidious::Database::Users.add_notification(video) + else + Invidious::Database::Users.feed_needs_update(video) + end else LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated") end end if pull_all_videos - page += 1 - - ids = [] of String - loop do - initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated) - videos = extract_videos(initial_data, author, ucid) - - count = videos.size - videos = videos.map { |video| ChannelVideo.new({ - id: video.id, - title: video.title, - published: video.published, - updated: Time.utc, - ucid: video.ucid, - author: video.author, - length_seconds: video.length_seconds, - live_now: video.live_now, - premiere_timestamp: video.premiere_timestamp, - views: video.views, - }) } - - videos.each do |video| - ids << video.id + # Keep fetching videos using the continuation token retrieved earlier + videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation) + + count = 0 + videos.select(SearchVideo).each do |video| + count += 1 + video = ChannelVideo.new({ + id: video.id, + title: video.title, + published: video.published, + updated: Time.utc, + ucid: video.ucid, + author: video.author, + length_seconds: video.length_seconds, + live_now: video.live_now, + premiere_timestamp: video.premiere_timestamp, + views: video.views, + }) # We are notified of Red videos elsewhere (PubSub), which includes a correct published date, # so since they don't provide a published date here we can safely ignore them. if Time.utc - video.published > 1.minute was_insert = Invidious::Database::ChannelVideos.insert(video) - Invidious::Database::Users.add_notification(video) if was_insert + if was_insert + if CONFIG.enable_user_notifications + Invidious::Database::Users.add_notification(video) + else + Invidious::Database::Users.feed_needs_update(video) + end + end end end break if count < 25 - page += 1 + sleep 500.milliseconds end end - channel = InvidiousChannel.new({ - id: ucid, - author: author, - updated: Time.utc, - deleted: false, - subscribed: nil, - }) - + channel.updated = Time.utc return channel end diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr index d5628f6a..8dc824b2 100644 --- a/src/invidious/channels/playlists.cr +++ b/src/invidious/channels/playlists.cr @@ -1,93 +1,28 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by) if continuation - response_json = YoutubeAPI.browse(continuation) - continuation_items = response_json["onResponseReceivedActions"]? - .try &.[0]["appendContinuationItemsAction"]["continuationItems"] - - return [] of SearchItem, nil if !continuation_items - - items = [] of SearchItem - continuation_items.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item| - extract_item(item, author, ucid).try { |t| items << t } - } - - continuation = continuation_items.as_a.last["continuationItemRenderer"]? - .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s - else - url = "/channel/#{ucid}/playlists?flow=list&view=1" - - case sort_by - when "last", "last_added" - # - when "oldest", "oldest_created" - url += "&sort=da" - when "newest", "newest_created" - url += "&sort=dd" - else nil # Ignore - end - - response = YT_POOL.client &.get(url) - initial_data = extract_initial_data(response.body) - return [] of SearchItem, nil if !initial_data - - items = extract_items(initial_data, author, ucid) - continuation = response.body.match(/"token":"(?<continuation>[^"]+)"/).try &.["continuation"]? - end - - return items, continuation -end - -# ## NOTE: DEPRECATED -# Reason -> Unstable -# The Protobuf object must be provided with an id of the last playlist from the current "page" -# in order to fetch the next one accurately -# (if the id isn't included, entries shift around erratically between pages, -# leading to repetitions and skip overs) -# -# Since it's impossible to produce the appropriate Protobuf without an id being provided by the user, -# it's better to stick to continuation tokens provided by the first request and onward -def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated = false) - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:base64" => { - "2:string" => "playlists", - "6:varint" => 2_i64, - "7:varint" => 1_i64, - "12:varint" => 1_i64, - "13:string" => "", - "23:varint" => 0_i64, - }, - }, - } - - if cursor - cursor = Base64.urlsafe_encode(cursor, false) if !auto_generated - object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = cursor - end - - if auto_generated - object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x32_i64 + initial_data = YoutubeAPI.browse(continuation) else - object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 1_i64 - case sort - when "oldest", "oldest_created" - object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 2_i64 - when "newest", "newest_created" - object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 3_i64 - when "last", "last_added" - object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 4_i64 - else nil # Ignore - end + params = + case sort_by + when "last", "last_added" + # Equivalent to "&sort=lad" + # {"2:string": "playlists", "3:varint": 4, "4:varint": 1, "6:varint": 1} + "EglwbGF5bGlzdHMYBCABMAE%3D" + when "oldest", "oldest_created" + # formerly "&sort=da" + # Not available anymore :c or maybe ?? + # {"2:string": "playlists", "3:varint": 2, "4:varint": 1, "6:varint": 1} + "EglwbGF5bGlzdHMYAiABMAE%3D" + # {"2:string": "playlists", "3:varint": 1, "4:varint": 1, "6:varint": 1} + # "EglwbGF5bGlzdHMYASABMAE%3D" + when "newest", "newest_created" + # Formerly "&sort=dd" + # {"2:string": "playlists", "3:varint": 3, "4:varint": 1, "6:varint": 1} + "EglwbGF5bGlzdHMYAyABMAE%3D" + end + + initial_data = YoutubeAPI.browse(ucid, params: params || "") end - object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"]))) - object["80226972:embedded"].delete("3:base64") - - continuation = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" + return extract_items(initial_data, author, ucid) end diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index b495e597..befec03d 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -16,6 +16,14 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } + sort_by_numerical = + case sort_by + when "newest" then 1_i64 + when "popular" then 2_i64 + when "oldest" then 3_i64 # Broken as of 10/2022 :c + else 1_i64 # Fallback to "newest" + end + object_inner_1 = { "110:embedded" => { "3:embedded" => { @@ -24,7 +32,7 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so "1:string" => object_inner_2_encoded, "2:string" => "00000000-0000-0000-0000-000000000000", }, - "3:varint" => 1_i64, + "3:varint" => sort_by_numerical, }, }, }, @@ -52,34 +60,138 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so return continuation end -def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest") - continuation = produce_channel_videos_continuation(ucid, page, - auto_generated: auto_generated, sort_by: sort_by, v2: true) - - return YoutubeAPI.browse(continuation) +# Used in bypass_captcha_job.cr +def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) + continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2) + return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" end -def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") - videos = [] of SearchVideo +module Invidious::Channel::Tabs + extend self - # 2.times do |i| - # initial_data = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by) - initial_data = get_channel_videos_response(ucid, 1, auto_generated: auto_generated, sort_by: sort_by) - videos = extract_videos(initial_data, author, ucid) - # end + # ------------------- + # Regular videos + # ------------------- - return videos.size, videos -end + def make_initial_video_ctoken(ucid, sort_by) : String + return produce_channel_videos_continuation(ucid, sort_by: sort_by) + end -def get_latest_videos(ucid) - initial_data = get_channel_videos_response(ucid) - author = initial_data["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s + # Wrapper for AboutChannel, as we still need to call get_videos with + # an author name and ucid directly (e.g in RSS feeds). + # TODO: figure out how to get rid of that + def get_videos(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") + return get_videos( + channel.author, channel.ucid, + continuation: continuation, sort_by: sort_by + ) + end - return extract_videos(initial_data, author, ucid) -end + # Wrapper for InvidiousChannel, as we still need to call get_videos with + # an author name and ucid directly (e.g in RSS feeds). + # TODO: figure out how to get rid of that + def get_videos(channel : InvidiousChannel, *, continuation : String? = nil, sort_by = "newest") + return get_videos( + channel.author, channel.id, + continuation: continuation, sort_by: sort_by + ) + end -# Used in bypass_captcha_job.cr -def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2) - return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" + def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest") + continuation ||= make_initial_video_ctoken(ucid, sort_by) + initial_data = YoutubeAPI.browse(continuation: continuation) + + return extract_items(initial_data, author, ucid) + end + + def get_60_videos(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") + if continuation.nil? + # Fetch the first "page" of video + items, next_continuation = get_videos(channel, sort_by: sort_by) + else + # Fetch a "page" of videos using the given continuation token + items, next_continuation = get_videos(channel, continuation: continuation) + end + + # If there is more to load, then load a second "page" + # and replace the previous continuation token + if !next_continuation.nil? + items_2, next_continuation = get_videos(channel, continuation: next_continuation) + items.concat items_2 + end + + return items, next_continuation + end + + # ------------------- + # Shorts + # ------------------- + + private def fetch_shorts_data(ucid : String, continuation : String? = nil) + if continuation.nil? + # EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts" + # TODO: try to extract the continuation tokens that allows other sorting options + return YoutubeAPI.browse(ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D") + else + return YoutubeAPI.browse(continuation: continuation) + end + end + + def get_shorts(channel : AboutChannel, continuation : String? = nil) + initial_data = self.fetch_shorts_data(channel.ucid, continuation) + + begin + # Try to parse the initial data fetched above + return extract_items(initial_data, channel.author, channel.ucid) + rescue ex : RetryOnceException + # Sometimes, for a completely unknown reason, the "reelItemRenderer" + # object is missing some critical information (it happens once in about + # 20 subsequent requests). Refreshing the page is required to properly + # show the "shorts" tab. + # + # In order to make the experience smoother for the user, we simulate + # said page refresh by fetching again the JSON. If that still doesn't + # work, we raise a BrokenTubeException, as something is really broken. + begin + initial_data = self.fetch_shorts_data(channel.ucid, continuation) + return extract_items(initial_data, channel.author, channel.ucid) + rescue ex : RetryOnceException + raise BrokenTubeException.new "reelPlayerHeaderSupportedRenderers" + end + end + end + + # ------------------- + # Livestreams + # ------------------- + + def get_livestreams(channel : AboutChannel, continuation : String? = nil) + if continuation.nil? + # EgdzdHJlYW1z8gYECgJ6AA%3D%3D is the protobuf object to load "streams" + initial_data = YoutubeAPI.browse(channel.ucid, params: "EgdzdHJlYW1z8gYECgJ6AA%3D%3D") + else + initial_data = YoutubeAPI.browse(continuation: continuation) + end + + return extract_items(initial_data, channel.author, channel.ucid) + end + + def get_60_livestreams(channel : AboutChannel, continuation : String? = nil) + if continuation.nil? + # Fetch the first "page" of streams + items, next_continuation = get_livestreams(channel) + else + # Fetch a "page" of streams using the given continuation token + items, next_continuation = get_livestreams(channel, continuation: continuation) + end + + # If there is more to load, then load a second "page" + # and replace the previous continuation token + if !next_continuation.nil? + items_2, next_continuation = get_livestreams(channel, continuation: next_continuation) + items.concat items_2 + end + + return items, next_continuation + end end diff --git a/src/invidious/config.cr b/src/invidious/config.cr index c9bf43a4..9fc58409 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -110,6 +110,8 @@ class Config property hsts : Bool? = true # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local' property disable_proxy : Bool? | Array(String)? = false + # Enable the user notifications for all users + property enable_user_notifications : Bool = true # URL to the modified source code to be easily AGPL compliant # Will display in the footer, next to the main source code link diff --git a/src/invidious/database/users.cr b/src/invidious/database/users.cr index f62b43ea..0a4a4fd8 100644 --- a/src/invidious/database/users.cr +++ b/src/invidious/database/users.cr @@ -154,6 +154,16 @@ module Invidious::Database::Users # Update (misc) # ------------------- + def feed_needs_update(video : ChannelVideo) + request = <<-SQL + UPDATE users + SET feed_needs_update = true + WHERE $1 = ANY(subscriptions) + SQL + + PG_DB.exec(request, video.ucid) + end + def update_preferences(user : User) request = <<-SQL UPDATE users diff --git a/src/invidious/exceptions.cr b/src/invidious/exceptions.cr index 425c08da..690db907 100644 --- a/src/invidious/exceptions.cr +++ b/src/invidious/exceptions.cr @@ -33,3 +33,8 @@ end class VideoNotAvailableException < Exception end + +# Exception used to indicate that the JSON response from YT is missing +# some important informations, and that the query should be sent again. +class RetryOnceException < Exception +end diff --git a/src/invidious/frontend/channel_page.cr b/src/invidious/frontend/channel_page.cr new file mode 100644 index 00000000..53745dd5 --- /dev/null +++ b/src/invidious/frontend/channel_page.cr @@ -0,0 +1,44 @@ +module Invidious::Frontend::ChannelPage + extend self + + enum TabsAvailable + Videos + Shorts + Streams + Playlists + Community + Channels + end + + def generate_tabs_links(locale : String, channel : AboutChannel, selected_tab : TabsAvailable) + return String.build(1500) do |str| + base_url = "/channel/#{channel.ucid}" + + TabsAvailable.each do |tab| + # Ignore playlists, as it is not supported for auto-generated channels yet + next if (tab.playlists? && channel.auto_generated) + + tab_name = tab.to_s.downcase + + if channel.tabs.includes? tab_name + str << %(<div class="pure-u-1 pure-md-1-3">\n) + + if tab == selected_tab + str << "\t<b>" + str << translate(locale, "channel_tab_#{tab_name}_label") + str << "</b>\n" + else + # Video tab doesn't have the last path component + url = tab.videos? ? base_url : "#{base_url}/#{tab_name}" + + str << %(\t<a href=") << url << %(">) + str << translate(locale, "channel_tab_#{tab_name}_label") + str << "</a>\n" + end + + str << "</div>" + end + end + end + end +end diff --git a/src/invidious/hashtag.cr b/src/invidious/hashtag.cr index afe31a36..bc329205 100644 --- a/src/invidious/hashtag.cr +++ b/src/invidious/hashtag.cr @@ -8,7 +8,8 @@ module Invidious::Hashtag client_config = YoutubeAPI::ClientConfig.new(region: region) response = YoutubeAPI.browse(continuation: ctoken, client_config: client_config) - return extract_items(response) + items, _ = extract_items(response) + return items end def generate_continuation(hashtag : String, cursor : Int) diff --git a/src/invidious/helpers/json_filter.cr b/src/invidious/helpers/json_filter.cr index b8e8f96d..3f4080ba 100644 --- a/src/invidious/helpers/json_filter.cr +++ b/src/invidious/helpers/json_filter.cr @@ -20,7 +20,7 @@ module JSONFilter /^\(|\(\(|\/\(/ end - def self.parse_fields(fields_text : String) : Nil + def self.parse_fields(fields_text : String, &) : Nil if fields_text.empty? raise FieldsParser::ParseError.new "Fields is empty" end @@ -42,7 +42,7 @@ module JSONFilter parse_nest_groups(fields_text) { |nest_list| yield nest_list } end - def self.parse_single_nests(fields_text : String) : Nil + def self.parse_single_nests(fields_text : String, &) : Nil single_nests = remove_nest_groups(fields_text) if !single_nests.empty? @@ -60,7 +60,7 @@ module JSONFilter end end - def self.parse_nest_groups(fields_text : String) : Nil + def self.parse_nest_groups(fields_text : String, &) : Nil nest_stack = [] of NamedTuple(group_name: String, closing_bracket_index: Int64) bracket_pairs = get_bracket_pairs(fields_text, true) diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index c52e2a0d..635f0984 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -265,4 +265,11 @@ class Category end end +struct Continuation + getter token + + def initialize(@token : String) + end +end + alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category diff --git a/src/invidious/http_server/utils.cr b/src/invidious/http_server/utils.cr new file mode 100644 index 00000000..e3f1fa0f --- /dev/null +++ b/src/invidious/http_server/utils.cr @@ -0,0 +1,20 @@ +module Invidious::HttpServer + module Utils + extend self + + def proxy_video_url(raw_url : String, *, region : String? = nil, absolute : Bool = false) + url = URI.parse(raw_url) + + # Add some URL parameters + params = url.query_params + params["host"] = url.host.not_nil! # Should never be nil, in theory + params["region"] = region if !region.nil? + + if absolute + return "#{HOST_URL}#{url.request_target}?#{params}" + else + return "#{url.request_target}?#{params}" + end + end + end +end diff --git a/src/invidious/jobs/notification_job.cr b/src/invidious/jobs/notification_job.cr index 2f525e08..b445107b 100644 --- a/src/invidious/jobs/notification_job.cr +++ b/src/invidious/jobs/notification_job.cr @@ -1,12 +1,12 @@ class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob - private getter connection_channel : Channel({Bool, Channel(PQ::Notification)}) + private getter connection_channel : ::Channel({Bool, ::Channel(PQ::Notification)}) private getter pg_url : URI def initialize(@connection_channel, @pg_url) end def begin - connections = [] of Channel(PQ::Notification) + connections = [] of ::Channel(PQ::Notification) PG.connect_listen(pg_url, "notifications") { |event| connections.each(&.send(event)) } diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr index 92681408..80812a63 100644 --- a/src/invidious/jobs/refresh_channels_job.cr +++ b/src/invidious/jobs/refresh_channels_job.cr @@ -8,7 +8,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob max_fibers = CONFIG.channel_threads lim_fibers = max_fibers active_fibers = 0 - active_channel = Channel(Bool).new + active_channel = ::Channel(Bool).new backoff = 2.minutes loop do diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr index 4b52c959..4f8130df 100644 --- a/src/invidious/jobs/refresh_feeds_job.cr +++ b/src/invidious/jobs/refresh_feeds_job.cr @@ -7,7 +7,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob def begin max_fibers = CONFIG.feed_threads active_fibers = 0 - active_channel = Channel(Bool).new + active_channel = ::Channel(Bool).new loop do db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs| diff --git a/src/invidious/jobs/subscribe_to_feeds_job.cr b/src/invidious/jobs/subscribe_to_feeds_job.cr index a431a48a..8584fb9c 100644 --- a/src/invidious/jobs/subscribe_to_feeds_job.cr +++ b/src/invidious/jobs/subscribe_to_feeds_job.cr @@ -12,7 +12,7 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob end active_fibers = 0 - active_channel = Channel(Bool).new + active_channel = ::Channel(Bool).new loop do db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > interval '4 days' OR subscribed IS NULL") do |rs| diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index 642789aa..a2b1a35c 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -3,7 +3,7 @@ require "json" module Invidious::JSONify::APIv1 extend self - def video(video : Video, json : JSON::Builder, *, locale : String?) + def video(video : Video, json : JSON::Builder, *, locale : String?, proxy : Bool = false) json.object do json.field "type", video.video_type @@ -89,7 +89,14 @@ module Invidious::JSONify::APIv1 # Not available on MPEG-4 Timed Text (`text/mp4`) streams (livestreams only) json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]? - json.field "url", fmt["url"] + if proxy + json.field "url", Invidious::HttpServer::Utils.proxy_video_url( + fmt["url"].to_s, absolute: true + ) + else + json.field "url", fmt["url"] + end + json.field "itag", fmt["itag"].as_i.to_s json.field "type", fmt["mimeType"] json.field "clen", fmt["contentLength"]? || "-1" diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index ae65f10d..662d1002 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -29,7 +29,7 @@ module Invidious::Routes::API::Manifest if local uri = URI.parse(url) - url = "#{uri.request_target}host/#{uri.host}/" + url = "#{HOST_URL}#{uri.request_target}host/#{uri.host}/" end "<BaseURL>#{url}</BaseURL>" @@ -42,7 +42,7 @@ module Invidious::Routes::API::Manifest if local adaptive_fmts.each do |fmt| - fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) + fmt["url"] = JSON::Any.new("#{HOST_URL}#{URI.parse(fmt["url"].as_s).request_target}") end end diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 6b81c546..ca2b2734 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -1,13 +1,7 @@ module Invidious::Routes::API::V1::Channels - def self.home(env) - locale = env.get("preferences").as(Preferences).locale - - env.response.content_type = "application/json" - - ucid = env.params.url["ucid"] - sort_by = env.params.query["sort_by"]?.try &.downcase - sort_by ||= "newest" - + # Macro to avoid duplicating some code below + # This sets the `channel` variable, or handles Exceptions. + private macro get_channel begin channel = get_about_info(ucid, locale) rescue ex : ChannelRedirect @@ -18,17 +12,25 @@ module Invidious::Routes::API::V1::Channels rescue ex return error_json(500, ex) end + end - page = 1 - if channel.auto_generated - videos = [] of SearchVideo - count = 0 - else - begin - count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) - rescue ex - return error_json(500, ex) - end + def self.home(env) + locale = env.get("preferences").as(Preferences).locale + ucid = env.params.url["ucid"] + + env.response.content_type = "application/json" + + # Use the private macro defined above. + channel = nil # Make the compiler happy + get_channel() + + # Retrieve "sort by" setting from URL parameters + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" + + begin + videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by) + rescue ex + return error_json(500, ex) end JSON.build do |json| @@ -100,31 +102,13 @@ module Invidious::Routes::API::V1::Channels json.array do # Fetch related channels begin - related_channels = fetch_related_channels(channel) + related_channels, _ = fetch_related_channels(channel) rescue ex - related_channels = [] of AboutRelatedChannel + related_channels = [] of SearchChannel end related_channels.each do |related_channel| - json.object do - json.field "author", related_channel.author - json.field "authorId", related_channel.ucid - json.field "authorUrl", related_channel.author_url - - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - - qualities.each do |quality| - json.object do - json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", quality - end - end - end - end - end + related_channel.to_json(locale, json) end end end # relatedChannels @@ -134,61 +118,112 @@ module Invidious::Routes::API::V1::Channels end def self.latest(env) + # Remove parameters that could affect this endpoint's behavior + env.params.query.delete("sort_by") if env.params.query.has_key?("sort_by") + env.params.query.delete("continuation") if env.params.query.has_key?("continuation") + + return self.videos(env) + end + + def self.videos(env) locale = env.get("preferences").as(Preferences).locale + ucid = env.params.url["ucid"] env.response.content_type = "application/json" - ucid = env.params.url["ucid"] + # Use the private macro defined above. + channel = nil # Make the compiler happy + get_channel() + + # Retrieve some URL parameters + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" + continuation = env.params.query["continuation"]? begin - videos = get_latest_videos(ucid) + videos, next_continuation = Channel::Tabs.get_60_videos( + channel, continuation: continuation, sort_by: sort_by + ) rescue ex return error_json(500, ex) end - JSON.build do |json| - json.array do - videos.each do |video| - video.to_json(locale, json) + return JSON.build do |json| + json.object do + json.field "videos" do + json.array do + videos.each &.to_json(locale, json) + end end + + json.field "continuation", next_continuation if next_continuation end end end - def self.videos(env) + def self.shorts(env) locale = env.get("preferences").as(Preferences).locale + ucid = env.params.url["ucid"] env.response.content_type = "application/json" - ucid = env.params.url["ucid"] - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - sort_by = env.params.query["sort"]?.try &.downcase - sort_by ||= env.params.query["sort_by"]?.try &.downcase - sort_by ||= "newest" + # Use the private macro defined above. + channel = nil # Make the compiler happy + get_channel() + + # Retrieve continuation from URL parameters + continuation = env.params.query["continuation"]? begin - channel = get_about_info(ucid, locale) - rescue ex : ChannelRedirect - env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) - return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) - rescue ex : NotFoundException - return error_json(404, ex) + videos, next_continuation = Channel::Tabs.get_shorts( + channel, continuation: continuation + ) rescue ex return error_json(500, ex) end + return JSON.build do |json| + json.object do + json.field "videos" do + json.array do + videos.each &.to_json(locale, json) + end + end + + json.field "continuation", next_continuation if next_continuation + end + end + end + + def self.streams(env) + locale = env.get("preferences").as(Preferences).locale + ucid = env.params.url["ucid"] + + env.response.content_type = "application/json" + + # Use the private macro defined above. + channel = nil # Make the compiler happy + get_channel() + + # Retrieve continuation from URL parameters + continuation = env.params.query["continuation"]? + begin - count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) + videos, next_continuation = Channel::Tabs.get_60_livestreams( + channel, continuation: continuation + ) rescue ex return error_json(500, ex) end - JSON.build do |json| - json.array do - videos.each do |video| - video.to_json(locale, json) + return JSON.build do |json| + json.object do + json.field "videos" do + json.array do + videos.each &.to_json(locale, json) + end end + + json.field "continuation", next_continuation if next_continuation end end end @@ -204,16 +239,9 @@ module Invidious::Routes::API::V1::Channels env.params.query["sort_by"]?.try &.downcase || "last" - begin - channel = get_about_info(ucid, locale) - rescue ex : ChannelRedirect - env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) - return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) - rescue ex : NotFoundException - return error_json(404, ex) - rescue ex - return error_json(500, ex) - end + # Use the macro defined above + channel = nil # Make the compiler happy + get_channel() items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) @@ -255,6 +283,37 @@ module Invidious::Routes::API::V1::Channels end end + def self.channels(env) + locale = env.get("preferences").as(Preferences).locale + ucid = env.params.url["ucid"] + + env.response.content_type = "application/json" + + # Use the macro defined above + channel = nil # Make the compiler happy + get_channel() + + continuation = env.params.query["continuation"]? + + begin + items, next_continuation = fetch_related_channels(channel, continuation) + rescue ex + return error_json(500, ex) + end + + JSON.build do |json| + json.object do + json.field "relatedChannels" do + json.array do + items.each &.to_json(locale, json) + end + end + + json.field "continuation", next_continuation if next_continuation + end + end + end + def self.search(env) locale = env.get("preferences").as(Preferences).locale region = env.params.query["region"]? diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index a6b2eb4e..f312211e 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -6,6 +6,7 @@ module Invidious::Routes::API::V1::Videos id = env.params.url["id"] region = env.params.query["region"]? + proxy = {"1", "true"}.any? &.== env.params.query["local"]? begin video = get_video(id, region: region) @@ -15,7 +16,9 @@ module Invidious::Routes::API::V1::Videos return error_json(500, ex) end - video.to_json(locale, nil) + return JSON.build do |json| + Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy) + end end def self.captions(env) @@ -90,45 +93,50 @@ module Invidious::Routes::API::V1::Videos # as well as some other markup that makes it cumbersome, so we try to fix that here if caption.name.includes? "auto-generated" caption_xml = YT_POOL.client &.get(url).body - caption_xml = XML.parse(caption_xml) - webvtt = String.build do |str| - str << <<-END_VTT - WEBVTT - Kind: captions - Language: #{tlang || caption.language_code} + if caption_xml.starts_with?("<?xml") + webvtt = caption.timedtext_to_vtt(caption_xml, tlang) + else + caption_xml = XML.parse(caption_xml) + webvtt = String.build do |str| + str << <<-END_VTT + WEBVTT + Kind: captions + Language: #{tlang || caption.language_code} - END_VTT - caption_nodes = caption_xml.xpath_nodes("//transcript/text") - caption_nodes.each_with_index do |node, i| - start_time = node["start"].to_f.seconds - duration = node["dur"]?.try &.to_f.seconds - duration ||= start_time + END_VTT - if caption_nodes.size > i + 1 - end_time = caption_nodes[i + 1]["start"].to_f.seconds - else - end_time = start_time + duration - end + caption_nodes = caption_xml.xpath_nodes("//transcript/text") + caption_nodes.each_with_index do |node, i| + start_time = node["start"].to_f.seconds + duration = node["dur"]?.try &.to_f.seconds + duration ||= start_time - start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}" - end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}" + if caption_nodes.size > i + 1 + end_time = caption_nodes[i + 1]["start"].to_f.seconds + else + end_time = start_time + duration + end - text = HTML.unescape(node.content) - text = text.gsub(/<font color="#[a-fA-F0-9]{6}">/, "") - text = text.gsub(/<\/font>/, "") - if md = text.match(/(?<name>.*) : (?<text>.*)/) - text = "<v #{md["name"]}>#{md["text"]}</v>" - end + start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}" + end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}" + + text = HTML.unescape(node.content) + text = text.gsub(/<font color="#[a-fA-F0-9]{6}">/, "") + text = text.gsub(/<\/font>/, "") + if md = text.match(/(?<name>.*) : (?<text>.*)/) + text = "<v #{md["name"]}>#{md["text"]}</v>" + end - str << <<-END_CUE - #{start_time} --> #{end_time} - #{text} + str << <<-END_CUE + #{start_time} --> #{end_time} + #{text} - END_CUE + END_CUE + end end end else @@ -138,7 +146,12 @@ module Invidious::Routes::API::V1::Videos # # See: https://github.com/iv-org/invidious/issues/2391 webvtt = YT_POOL.client &.get("#{url}&format=vtt").body - .gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1") + if webvtt.starts_with?("<?xml") + webvtt = caption.timedtext_to_vtt(webvtt) + else + webvtt = YT_POOL.client &.get("#{url}&format=vtt").body + .gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1") + end end if title = env.params.query["title"]? diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index c6e02cbd..d3969d29 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -7,21 +7,19 @@ module Invidious::Routes::Channels def self.videos(env) data = self.fetch_basic_information(env) - if !data.is_a?(Tuple) - return data - end - locale, user, subscriptions, continuation, ucid, channel = data + return data if !data.is_a?(Tuple) - page = env.params.query["page"]?.try &.to_i? - page ||= 1 + locale, user, subscriptions, continuation, ucid, channel = data sort_by = env.params.query["sort_by"]?.try &.downcase if channel.auto_generated sort_options = {"last", "oldest", "newest"} - sort_by ||= "last" - items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) + items, next_continuation = fetch_channel_playlists( + channel.ucid, channel.author, continuation, (sort_by || "last") + ) + items.uniq! do |item| if item.responds_to?(:title) item.title @@ -33,34 +31,85 @@ module Invidious::Routes::Channels items.each(&.author = "") else sort_options = {"newest", "oldest", "popular"} - sort_by ||= "newest" - count, items = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) + # Fetch items and continuation token + items, next_continuation = Channel::Tabs.get_videos( + channel, continuation: continuation, sort_by: (sort_by || "newest") + ) end + selected_tab = Frontend::ChannelPage::TabsAvailable::Videos templated "channel" end - def self.playlists(env) + def self.shorts(env) data = self.fetch_basic_information(env) - if !data.is_a?(Tuple) - return data + return data if !data.is_a?(Tuple) + + locale, user, subscriptions, continuation, ucid, channel = data + + if !channel.tabs.includes? "shorts" + return env.redirect "/channel/#{channel.ucid}" + end + + # TODO: support sort option for shorts + sort_by = "" + sort_options = [] of String + + # Fetch items and continuation token + items, next_continuation = Channel::Tabs.get_shorts( + channel, continuation: continuation + ) + + selected_tab = Frontend::ChannelPage::TabsAvailable::Shorts + templated "channel" + end + + def self.streams(env) + data = self.fetch_basic_information(env) + return data if !data.is_a?(Tuple) + + locale, user, subscriptions, continuation, ucid, channel = data + + if !channel.tabs.includes? "streams" + return env.redirect "/channel/#{channel.ucid}" end + + # TODO: support sort option for livestreams + sort_by = "" + sort_options = [] of String + + # Fetch items and continuation token + items, next_continuation = Channel::Tabs.get_60_livestreams( + channel, continuation: continuation + ) + + selected_tab = Frontend::ChannelPage::TabsAvailable::Streams + templated "channel" + end + + def self.playlists(env) + data = self.fetch_basic_information(env) + return data if !data.is_a?(Tuple) + locale, user, subscriptions, continuation, ucid, channel = data sort_options = {"last", "oldest", "newest"} sort_by = env.params.query["sort_by"]?.try &.downcase - sort_by ||= "last" if channel.auto_generated return env.redirect "/channel/#{channel.ucid}" end - items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) + items, next_continuation = fetch_channel_playlists( + channel.ucid, channel.author, continuation, (sort_by || "last") + ) + items = items.select(SearchPlaylist).map(&.as(SearchPlaylist)) items.each(&.author = "") - templated "playlists" + selected_tab = Frontend::ChannelPage::TabsAvailable::Playlists + templated "channel" end def self.community(env) @@ -74,12 +123,15 @@ module Invidious::Routes::Channels thin_mode = thin_mode == "true" continuation = env.params.query["continuation"]? - # sort_by = env.params.query["sort_by"]?.try &.downcase if !channel.tabs.includes? "community" return env.redirect "/channel/#{channel.ucid}" end + # TODO: support sort options for community posts + sort_by = "" + sort_options = [] of String + begin items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode)) rescue ex : InfoException @@ -95,6 +147,26 @@ module Invidious::Routes::Channels templated "community" end + def self.channels(env) + data = self.fetch_basic_information(env) + return data if !data.is_a?(Tuple) + + locale, user, subscriptions, continuation, ucid, channel = data + + if channel.auto_generated + return env.redirect "/channel/#{channel.ucid}" + end + + items, next_continuation = fetch_related_channels(channel, continuation) + + # Featured/related channels can't be sorted + sort_options = [] of String + sort_by = nil + + selected_tab = Frontend::ChannelPage::TabsAvailable::Channels + templated "channel" + end + def self.about(env) data = self.fetch_basic_information(env) if !data.is_a?(Tuple) @@ -125,7 +197,7 @@ module Invidious::Routes::Channels end selected_tab = env.request.path.split("/")[-1] - if ["home", "videos", "playlists", "community", "channels", "about"].includes? selected_tab + if {"home", "videos", "shorts", "streams", "playlists", "community", "channels", "about"}.includes? selected_tab url = "/channel/#{ucid}/#{selected_tab}" else url = "/channel/#{ucid}" diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 289d87c9..266f7ba4 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -147,7 +147,7 @@ module Invidious::Routes::Embed # PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.as(User).email) # end - if notifications && notifications.includes? id + if CONFIG.enable_user_notifications && notifications && notifications.includes? id Invidious::Database::Users.remove_notification(user.as(User), id) env.get("user").as(User).notifications.delete(id) notifications.delete(id) diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index b601db94..fb482e33 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -96,12 +96,14 @@ module Invidious::Routes::Feeds videos, notifications = get_subscription_feed(user, max_results, page) - # "updated" here is used for delivering new notifications, so if - # we know a user has looked at their feed e.g. in the past 10 minutes, - # they've already seen a video posted 20 minutes ago, and don't need - # to be notified. - Invidious::Database::Users.clear_notifications(user) - user.notifications = [] of String + if CONFIG.enable_user_notifications + # "updated" here is used for delivering new notifications, so if + # we know a user has looked at their feed e.g. in the past 10 minutes, + # they've already seen a video posted 20 minutes ago, and don't need + # to be notified. + Invidious::Database::Users.clear_notifications(user) + user.notifications = [] of String + end env.set "user", user templated "feeds/subscriptions" @@ -404,13 +406,15 @@ module Invidious::Routes::Feeds video = get_video(id, force_refresh: true) - # Deliver notifications to `/api/v1/auth/notifications` - payload = { - "topic" => video.ucid, - "videoId" => video.id, - "published" => published.to_unix, - }.to_json - PG_DB.exec("NOTIFY notifications, E'#{payload}'") + if CONFIG.enable_user_notifications + # Deliver notifications to `/api/v1/auth/notifications` + payload = { + "topic" => video.ucid, + "videoId" => video.id, + "published" => published.to_unix, + }.to_json + PG_DB.exec("NOTIFY notifications, E'#{payload}'") + end video = ChannelVideo.new({ id: id, @@ -426,7 +430,13 @@ module Invidious::Routes::Feeds }) was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true) - Invidious::Database::Users.add_notification(video) if was_insert + if was_insert + if CONFIG.enable_user_notifications + Invidious::Database::Users.add_notification(video) + else + Invidious::Database::Users.feed_needs_update(video) + end + end end end diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 560f9c19..1e932d11 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -35,6 +35,13 @@ module Invidious::Routes::VideoPlayback end end + # See: https://github.com/iv-org/invidious/issues/3302 + range_header = env.request.headers["Range"]? + if range_header.nil? + range_for_head = query_params["range"]? || "0-640" + headers["Range"] = "bytes=#{range_for_head}" + end + client = make_client(URI.parse(host), region) response = HTTP::Client::Response.new(500) error = "" @@ -70,6 +77,9 @@ module Invidious::Routes::VideoPlayback end end + # Remove the Range header added previously. + headers.delete("Range") if range_header.nil? + if response.status_code >= 400 env.response.content_type = "text/plain" haltf env, response.status_code @@ -91,14 +101,8 @@ module Invidious::Routes::VideoPlayback env.response.headers["Access-Control-Allow-Origin"] = "*" if location = resp.headers["Location"]? - location = URI.parse(location) - location = "#{location.request_target}&host=#{location.host}" - - if region - location += "®ion=#{region}" - end - - return env.redirect location + url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region) + return env.redirect url end IO.copy(resp.body_io, env.response) diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 5f481557..5d3845c3 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -80,7 +80,7 @@ module Invidious::Routes::Watch Invidious::Database::Users.mark_watched(user.as(User), id) end - if notifications && notifications.includes? id + if CONFIG.enable_user_notifications && notifications && notifications.includes? id Invidious::Database::Users.remove_notification(user.as(User), id) env.get("user").as(User).notifications.delete(id) notifications.delete(id) diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index f409f13c..491022a5 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -37,7 +37,9 @@ module Invidious::Routing get "/feed/webhook/:token", Routes::Feeds, :push_notifications_get post "/feed/webhook/:token", Routes::Feeds, :push_notifications_post - get "/modify_notifications", Routes::Notifications, :modify + if CONFIG.enable_user_notifications + get "/modify_notifications", Routes::Notifications, :modify + end {% end %} self.register_image_routes @@ -115,14 +117,17 @@ module Invidious::Routing get "/channel/:ucid", Routes::Channels, :home get "/channel/:ucid/home", Routes::Channels, :home get "/channel/:ucid/videos", Routes::Channels, :videos + get "/channel/:ucid/shorts", Routes::Channels, :shorts + get "/channel/:ucid/streams", Routes::Channels, :streams get "/channel/:ucid/playlists", Routes::Channels, :playlists get "/channel/:ucid/community", Routes::Channels, :community + get "/channel/:ucid/channels", Routes::Channels, :channels get "/channel/:ucid/about", Routes::Channels, :about get "/channel/:ucid/live", Routes::Channels, :live get "/user/:user/live", Routes::Channels, :live get "/c/:user/live", Routes::Channels, :live - ["", "/videos", "/playlists", "/community", "/about"].each do |path| + {"", "/videos", "/shorts", "/streams", "/playlists", "/community", "/about"}.each do |path| # /c/LinusTechTips get "/c/:user#{path}", Routes::Channels, :brand_redirect # /user/linustechtips | Not always the same as /c/ @@ -220,6 +225,10 @@ module Invidious::Routing # Channels get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home + get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts + get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams + get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels + {% for route in {"videos", "latest", "playlists", "community", "search"} %} get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}} get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}} @@ -260,8 +269,10 @@ module Invidious::Routing post "/api/v1/auth/tokens/register", {{namespace}}::Authenticated, :register_token post "/api/v1/auth/tokens/unregister", {{namespace}}::Authenticated, :unregister_token - get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications - post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + if CONFIG.enable_user_notifications + get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + end # Misc get "/api/v1/stats", {{namespace}}::Misc, :stats diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr index d1409c06..7e909590 100644 --- a/src/invidious/search/processors.cr +++ b/src/invidious/search/processors.cr @@ -9,7 +9,8 @@ module Invidious::Search client_config = YoutubeAPI::ClientConfig.new(region: query.region) initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config) - return extract_items(initial_data) + items, _ = extract_items(initial_data) + return items end # Search a youtube channel @@ -30,16 +31,7 @@ module Invidious::Search continuation = produce_channel_search_continuation(ucid, query.text, query.page) response_json = YoutubeAPI.browse(continuation) - continuation_items = response_json["onResponseReceivedActions"]? - .try &.[0]["appendContinuationItemsAction"]["continuationItems"] - - return [] of SearchItem if !continuation_items - - items = [] of SearchItem - continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item| - extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t } - end - + items, _ = extract_items(response_json, "", ucid) return items end diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr index 4642c1a7..13f81a31 100644 --- a/src/invidious/videos/caption.cr +++ b/src/invidious/videos/caption.cr @@ -31,6 +31,72 @@ module Invidious::Videos return captions_list end + def timedtext_to_vtt(timedtext : String, tlang = nil) : String + # In the future, we could just directly work with the url. This is more of a POC + cues = [] of XML::Node + tree = XML.parse(timedtext) + tree = tree.children.first + + tree.children.each do |item| + if item.name == "body" + item.children.each do |cue| + if cue.name == "p" && !(cue.children.size == 1 && cue.children[0].content == "\n") + cues << cue + end + end + break + end + end + result = String.build do |result| + result << <<-END_VTT + WEBVTT + Kind: captions + Language: #{tlang || @language_code} + + + END_VTT + + result << "\n\n" + + cues.each_with_index do |node, i| + start_time = node["t"].to_f.milliseconds + + duration = node["d"]?.try &.to_f.milliseconds + + duration ||= start_time + + if cues.size > i + 1 + end_time = cues[i + 1]["t"].to_f.milliseconds + else + end_time = start_time + duration + end + + # start_time + result << start_time.hours.to_s.rjust(2, '0') + result << ':' << start_time.minutes.to_s.rjust(2, '0') + result << ':' << start_time.seconds.to_s.rjust(2, '0') + result << '.' << start_time.milliseconds.to_s.rjust(3, '0') + + result << " --> " + + # end_time + result << end_time.hours.to_s.rjust(2, '0') + result << ':' << end_time.minutes.to_s.rjust(2, '0') + result << ':' << end_time.seconds.to_s.rjust(2, '0') + result << '.' << end_time.milliseconds.to_s.rjust(3, '0') + + result << "\n" + + node.children.each do |s| + result << s.content + end + result << "\n" + result << "\n" + end + end + return result + end + # List of all caption languages available on Youtube. LANGUAGES = { "", diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 5df49286..5c323975 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -66,8 +66,10 @@ def extract_video_info(video_id : String, proxy_region : String? = nil) reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("") reason ||= player_response.dig("playabilityStatus", "reason").as_s - # Stop here if video is not a scheduled livestream - if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) + # Stop here if video is not a scheduled livestream or + # for LOGIN_REQUIRED when videoDetails element is not found because retrying won't help + if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) || + playability_status == "LOGIN_REQUIRED" && !player_response.dig?("videoDetails") return { "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), "reason" => JSON::Any.new(reason), diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr index dea86abe..a29315ef 100644 --- a/src/invidious/views/channel.ecr +++ b/src/invidious/views/channel.ecr @@ -1,8 +1,24 @@ -<% ucid = channel.ucid %> -<% author = HTML.escape(channel.author) %> -<% channel_profile_pic = URI.parse(channel.author_thumbnail).request_target %> +<%- + ucid = channel.ucid + author = HTML.escape(channel.author) + channel_profile_pic = URI.parse(channel.author_thumbnail).request_target + + relative_url = + case selected_tab + when .shorts? then "/channel/#{ucid}/shorts" + when .streams? then "/channel/#{ucid}/streams" + when .playlists? then "/channel/#{ucid}/playlists" + when .channels? then "/channel/#{ucid}/channels" + else + "/channel/#{ucid}" + end + + youtube_url = "https://www.youtube.com#{relative_url}" + redirect_url = Invidious::Frontend::Misc.redirect_url(env) +-%> <% content_for "header" do %> +<%- if selected_tab.videos? -%> <meta name="description" content="<%= channel.description %>"> <meta property="og:site_name" content="Invidious"> <meta property="og:url" content="<%= HOST_URL %>/channel/<%= ucid %>"> @@ -14,91 +30,14 @@ <meta name="twitter:title" content="<%= author %>"> <meta name="twitter:description" content="<%= channel.description %>"> <meta name="twitter:image" content="/ggpht<%= channel_profile_pic %>"> -<link rel="alternate" href="https://www.youtube.com/channel/<%= ucid %>"> -<title><%= author %> - Invidious</title> <link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/channel/<%= ucid %>" /> -<% end %> - -<% if channel.banner %> - <div class="h-box"> - <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>"> - </div> +<%- end -%> - <div class="h-box"> - <hr> - </div> +<link rel="alternate" href="<%= youtube_url %>"> +<title><%= author %> - Invidious</title> <% end %> -<div class="pure-g h-box"> - <div class="pure-u-2-3"> - <div class="channel-profile"> - <img src="/ggpht<%= channel_profile_pic %>"> - <span><%= author %></span><% if !channel.verified.nil? && channel.verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end %> - </div> - </div> - <div class="pure-u-1-3"> - <h3 style="text-align:right"> - <a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a> - </h3> - </div> -</div> - -<div class="h-box"> - <div id="descriptionWrapper"> - <p><span style="white-space:pre-wrap"><%= channel.description_html %></span></p> - </div> -</div> - -<div class="h-box"> - <% sub_count_text = number_to_short_text(channel.sub_count) %> - <%= rendered "components/subscribe_widget" %> -</div> - -<div class="pure-g h-box"> - <div class="pure-u-1-3"> - <a href="https://www.youtube.com/channel/<%= ucid %>"><%= translate(locale, "View channel on YouTube") %></a> - <div class="pure-u-1 pure-md-1-3"> - <% if env.get("preferences").as(Preferences).automatic_instance_redirect%> - <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a> - <% else %> - <a href="https://redirect.invidious.io<%= env.request.path %>"><%= translate(locale, "Switch Invidious Instance") %></a> - <% end %> - </div> - <% if !channel.auto_generated %> - <div class="pure-u-1 pure-md-1-3"> - <b><%= translate(locale, "Videos") %></b> - </div> - <% end %> - <div class="pure-u-1 pure-md-1-3"> - <% if channel.auto_generated %> - <b><%= translate(locale, "Playlists") %></b> - <% else %> - <a href="/channel/<%= ucid %>/playlists"><%= translate(locale, "Playlists") %></a> - <% end %> - </div> - <div class="pure-u-1 pure-md-1-3"> - <% if channel.tabs.includes? "community" %> - <a href="/channel/<%= ucid %>/community"><%= translate(locale, "Community") %></a> - <% end %> - </div> - </div> - <div class="pure-u-1-3"></div> - <div class="pure-u-1-3"> - <div class="pure-g" style="text-align:right"> - <% sort_options.each do |sort| %> - <div class="pure-u-1 pure-md-1-3"> - <% if sort_by == sort %> - <b><%= translate(locale, sort) %></b> - <% else %> - <a href="/channel/<%= ucid %>?page=<%= page %>&sort_by=<%= sort %>"> - <%= translate(locale, sort) %> - </a> - <% end %> - </div> - <% end %> - </div> - </div> -</div> +<%= rendered "components/channel_info" %> <div class="h-box"> <hr> @@ -111,17 +50,10 @@ </div> <div class="pure-g h-box"> - <div class="pure-u-1 pure-u-lg-1-5"> - <% if page > 1 %> - <a href="/channel/<%= ucid %>?page=<%= page - 1 %><% if sort_by != "newest" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>"> - <%= translate(locale, "Previous page") %> - </a> - <% end %> - </div> - <div class="pure-u-1 pure-u-lg-3-5"></div> + <div class="pure-u-1 pure-u-md-4-5"></div> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> - <% if count == 60 %> - <a href="/channel/<%= ucid %>?page=<%= page + 1 %><% if sort_by != "newest" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>"> + <% if next_continuation %> + <a href="<%= relative_url %>?continuation=<%= next_continuation %><% if sort_options.any? sort_by %>&sort_by=<%= sort_by %><% end %>"> <%= translate(locale, "Next page") %> </a> <% end %> diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr index 3bc29e55..9e11d562 100644 --- a/src/invidious/views/community.ecr +++ b/src/invidious/views/community.ecr @@ -1,71 +1,21 @@ -<% ucid = channel.ucid %> -<% author = HTML.escape(channel.author) %> +<%- + ucid = channel.ucid + author = HTML.escape(channel.author) + channel_profile_pic = URI.parse(channel.author_thumbnail).request_target -<% content_for "header" do %> -<title><%= author %> - Invidious</title> -<% end %> + relative_url = "/channel/#{ucid}/community" + youtube_url = "https://www.youtube.com#{relative_url}" + redirect_url = Invidious::Frontend::Misc.redirect_url(env) -<% if channel.banner %> - <div class="h-box"> - <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>"> - </div> + selected_tab = Invidious::Frontend::ChannelPage::TabsAvailable::Community +-%> - <div class="h-box"> - <hr> - </div> +<% content_for "header" do %> +<link rel="alternate" href="<%= youtube_url %>"> +<title><%= author %> - Invidious</title> <% end %> -<div class="pure-g h-box"> - <div class="pure-u-2-3"> - <div class="channel-profile"> - <img src="/ggpht<%= URI.parse(channel.author_thumbnail).request_target %>"> - <span><%= author %></span><% if !channel.verified.nil? && channel.verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end %> - </div> - </div> - <div class="pure-u-1-3" style="text-align:right"> - <h3 style="text-align:right"> - <a href="/feed/channel/<%= channel.ucid %>"><i class="icon ion-logo-rss"></i></a> - </h3> - </div> -</div> - -<div class="h-box"> - <div id="descriptionWrapper"> - <p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %></span></p> - </div> -</div> - -<div class="h-box"> - <% sub_count_text = number_to_short_text(channel.sub_count) %> - <%= rendered "components/subscribe_widget" %> -</div> - -<div class="pure-g h-box"> - <div class="pure-u-1-3"> - <a href="https://www.youtube.com/channel/<%= channel.ucid %>/community"><%= translate(locale, "View channel on YouTube") %></a> - <div class="pure-u-1 pure-md-1-3"> - <% if env.get("preferences").as(Preferences).automatic_instance_redirect%> - <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a> - <% else %> - <a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a> - <% end %> - </div> - <% if !channel.auto_generated %> - <div class="pure-u-1 pure-md-1-3"> - <a href="/channel/<%= channel.ucid %>"><%= translate(locale, "Videos") %></a> - </div> - <% end %> - <div class="pure-u-1 pure-md-1-3"> - <a href="/channel/<%= channel.ucid %>/playlists"><%= translate(locale, "Playlists") %></a> - </div> - <div class="pure-u-1 pure-md-1-3"> - <% if channel.tabs.includes? "community" %> - <b><%= translate(locale, "Community") %></b> - <% end %> - </div> - </div> - <div class="pure-u-2-3"></div> -</div> +<%= rendered "components/channel_info" %> <div class="h-box"> <hr> diff --git a/src/invidious/views/components/channel_info.ecr b/src/invidious/views/components/channel_info.ecr new file mode 100644 index 00000000..f216359f --- /dev/null +++ b/src/invidious/views/components/channel_info.ecr @@ -0,0 +1,60 @@ +<% if channel.banner %> + <div class="h-box"> + <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>"> + </div> + + <div class="h-box"> + <hr> + </div> +<% end %> + +<div class="pure-g h-box"> + <div class="pure-u-2-3"> + <div class="channel-profile"> + <img src="/ggpht<%= channel_profile_pic %>"> + <span><%= author %></span><% if !channel.verified.nil? && channel.verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end %> + </div> + </div> + <div class="pure-u-1-3"> + <h3 style="text-align:right"> + <a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a> + </h3> + </div> +</div> + +<div class="h-box"> + <div id="descriptionWrapper"> + <p><span style="white-space:pre-wrap"><%= channel.description_html %></span></p> + </div> +</div> + +<div class="h-box"> + <% sub_count_text = number_to_short_text(channel.sub_count) %> + <%= rendered "components/subscribe_widget" %> +</div> + +<div class="pure-g h-box"> + <div class="pure-u-1-2"> + <div class="pure-u-1 pure-md-1-3"> + <a href="<%= youtube_url %>"><%= translate(locale, "View channel on YouTube") %></a> + </div> + <div class="pure-u-1 pure-md-1-3"> + <a href="<%= redirect_url %>"><%= translate(locale, "Switch Invidious Instance") %></a> + </div> + + <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, selected_tab) %> + </div> + <div class="pure-u-1-2"> + <div class="pure-g" style="text-align:end"> + <% sort_options.each do |sort| %> + <div class="pure-u-1 pure-md-1-3"> + <% if sort_by == sort %> + <b><%= translate(locale, sort) %></b> + <% else %> + <a href="<%= relative_url %>?sort_by=<%= sort %>"><%= translate(locale, sort) %></a> + <% end %> + </div> + <% end %> + </div> + </div> +</div> diff --git a/src/invidious/views/feeds/subscriptions.ecr b/src/invidious/views/feeds/subscriptions.ecr index 8d56ad14..76f2f2bd 100644 --- a/src/invidious/views/feeds/subscriptions.ecr +++ b/src/invidious/views/feeds/subscriptions.ecr @@ -23,6 +23,8 @@ </div> </div> +<% if CONFIG.enable_user_notifications %> + <center> <%= translate_count(locale, "subscriptions_unseen_notifs_count", notifications.size) %> </center> @@ -39,6 +41,8 @@ <% end %> </div> +<% end %> + <div class="h-box"> <hr> </div> diff --git a/src/invidious/views/playlists.ecr b/src/invidious/views/playlists.ecr deleted file mode 100644 index c8718e7b..00000000 --- a/src/invidious/views/playlists.ecr +++ /dev/null @@ -1,108 +0,0 @@ -<% ucid = channel.ucid %> -<% author = HTML.escape(channel.author) %> - -<% content_for "header" do %> -<title><%= author %> - Invidious</title> -<% end %> - -<% if channel.banner %> - <div class="h-box"> - <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>"> - </div> - - <div class="h-box"> - <hr> - </div> -<% end %> - -<div class="pure-g h-box"> - <div class="pure-u-2-3"> - <div class="channel-profile"> - <img src="/ggpht<%= URI.parse(channel.author_thumbnail).request_target %>"> - <span><%= author %></span><% if !channel.verified.nil? && channel.verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end %> - </div> - </div> - <div class="pure-u-1-3" style="text-align:right"> - <h3 style="text-align:right"> - <a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a> - </h3> - </div> -</div> - -<div class="h-box"> - <div id="descriptionWrapper"> - <p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content if !channel.description_html.empty? %></span></p> - </div> -</div> - -<div class="h-box"> - <% sub_count_text = number_to_short_text(channel.sub_count) %> - <%= rendered "components/subscribe_widget" %> -</div> - -<div class="pure-g h-box"> - <div class="pure-g pure-u-1-3"> - <div class="pure-u-1 pure-md-1-3"> - <a href="https://www.youtube.com/channel/<%= ucid %>/playlists"><%= translate(locale, "View channel on YouTube") %></a> - </div> - - <div class="pure-u-1 pure-md-1-3"> - <% if env.get("preferences").as(Preferences).automatic_instance_redirect%> - <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a> - <% else %> - <a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a> - <% end %> - </div> - - <div class="pure-u-1 pure-md-1-3"> - <a href="/channel/<%= ucid %>"><%= translate(locale, "Videos") %></a> - </div> - <div class="pure-u-1 pure-md-1-3"> - <% if !channel.auto_generated %> - <b><%= translate(locale, "Playlists") %></b> - <% end %> - </div> - <div class="pure-u-1 pure-md-1-3"> - <% if channel.tabs.includes? "community" %> - <a href="/channel/<%= ucid %>/community"><%= translate(locale, "Community") %></a> - <% end %> - </div> - </div> - <div class="pure-u-1-3"></div> - <div class="pure-u-1-3"> - <div class="pure-g" style="text-align:right"> - <% {"last", "oldest", "newest"}.each do |sort| %> - <div class="pure-u-1 pure-md-1-3"> - <% if sort_by == sort %> - <b><%= translate(locale, sort) %></b> - <% else %> - <a href="/channel/<%= ucid %>/playlists?sort_by=<%= sort %>"> - <%= translate(locale, sort) %> - </a> - <% end %> - </div> - <% end %> - </div> - </div> -</div> - -<div class="h-box"> - <hr> -</div> - -<div class="pure-g"> -<% items.each do |item| %> - <%= rendered "components/item" %> -<% end %> -</div> - -<div class="pure-g h-box"> - <div class="pure-u-1 pure-u-md-4-5"></div> - <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> - <% if continuation %> - <a href="/channel/<%= ucid %>/playlists?continuation=<%= continuation %><% if sort_by != "last" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>"> - <%= translate(locale, "Next page") %> - </a> - <% end %> - </div> -</div> diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index 98f72eba..77265679 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -54,7 +54,7 @@ <div class="pure-u-1-4"> <a id="notification_ticker" title="<%= translate(locale, "Subscriptions") %>" href="/feed/subscriptions" class="pure-menu-heading"> <% notification_count = env.get("user").as(Invidious::User).notifications.size %> - <% if notification_count > 0 %> + <% if CONFIG.enable_user_notifications && notification_count > 0 %> <span id="notification_count"><%= notification_count %></span> <i class="icon ion-ios-notifications"></i> <% else %> <i class="icon ion-ios-notifications-outline"></i> @@ -170,7 +170,9 @@ }.to_pretty_json %> </script> + <% if CONFIG.enable_user_notifications %> <script src="/js/notifications.js?v=<%= ASSET_COMMIT %>"></script> + <% end %> <% end %> </body> diff --git a/src/invidious/views/user/preferences.ecr b/src/invidious/views/user/preferences.ecr index d841982c..dfda1434 100644 --- a/src/invidious/views/user/preferences.ecr +++ b/src/invidious/views/user/preferences.ecr @@ -244,6 +244,7 @@ <input name="unseen_only" id="unseen_only" type="checkbox" <% if preferences.unseen_only %>checked<% end %>> </div> + <% if CONFIG.enable_user_notifications %> <div class="pure-control-group"> <label for="notifications_only"><%= translate(locale, "preferences_notifications_only_label") %></label> <input name="notifications_only" id="notifications_only" type="checkbox" <% if preferences.notifications_only %>checked<% end %>> @@ -255,6 +256,7 @@ <a href="#" data-onclick="notification_requestPermission"><%= translate(locale, "Enable web notifications") %></a> </div> <% end %> + <% end %> <% end %> <% if env.get?("user") && CONFIG.admins.includes? env.get?("user").as(Invidious::User).email %> diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 1f7726fb..b14ad7b9 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -7,7 +7,7 @@ require "../helpers/serialized_yt_data" private ITEM_CONTAINER_EXTRACTOR = { Extractors::YouTubeTabs, Extractors::SearchResults, - Extractors::Continuation, + Extractors::ContinuationContent, } private ITEM_PARSERS = { @@ -18,8 +18,11 @@ private ITEM_PARSERS = { Parsers::CategoryRendererParser, Parsers::RichItemRendererParser, Parsers::ReelItemRendererParser, + Parsers::ContinuationItemRendererParser, } +private alias InitialData = Hash(String, JSON::Any) + record AuthorFallback, name : String, id : String # Namespace for logic relating to parsing InnerTube data into various datastructs. @@ -355,14 +358,9 @@ private module Parsers content_container = item_contents["contents"] end - raw_contents = content_container["items"]?.try &.as_a - if !raw_contents.nil? - raw_contents.each do |item| - result = extract_item(item) - if !result.nil? - contents << result - end - end + content_container["items"]?.try &.as_a.each do |item| + result = parse_item(item, author_fallback.name, author_fallback.id) + contents << result if result.is_a?(SearchItem) end Category.new({ @@ -394,7 +392,9 @@ private module Parsers end private def self.parse(item_contents, author_fallback) - return VideoRendererParser.process(item_contents, author_fallback) + child = VideoRendererParser.process(item_contents, author_fallback) + child ||= ReelItemRendererParser.process(item_contents, author_fallback) + return child end def self.parser_name @@ -418,9 +418,19 @@ private module Parsers private def self.parse(item_contents, author_fallback) video_id = item_contents["videoId"].as_s - video_details_container = item_contents.dig( + reel_player_overlay = item_contents.dig( "navigationEndpoint", "reelWatchEndpoint", - "overlay", "reelPlayerOverlayRenderer", + "overlay", "reelPlayerOverlayRenderer" + ) + + # Sometimes, the "reelPlayerOverlayRenderer" object is missing the + # important part of the response. We use this exception to tell + # the calling function to fetch the content again. + if !reel_player_overlay.as_h.has_key?("reelPlayerHeaderSupportedRenderers") + raise RetryOnceException.new + end + + video_details_container = reel_player_overlay.dig( "reelPlayerHeaderSupportedRenderers", "reelPlayerHeaderRenderer" ) @@ -446,9 +456,9 @@ private module Parsers # View count - view_count_text = video_details_container.dig?("viewCountText", "simpleText") - view_count_text ||= video_details_container - .dig?("viewCountText", "accessibility", "accessibilityData", "label") + # View count used to be in the reelWatchEndpoint, but that changed? + view_count_text = item_contents.dig?("viewCountText", "simpleText") + view_count_text ||= video_details_container.dig?("viewCountText", "simpleText") view_count = view_count_text.try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64 @@ -460,8 +470,8 @@ private module Parsers regex_match = /- (?<min>\d+ minutes? )?(?<sec>\d+ seconds?)+ -/.match(a11y_data) - minutes = regex_match.try &.["min"].to_i(strict: false) || 0 - seconds = regex_match.try &.["sec"].to_i(strict: false) || 0 + minutes = regex_match.try &.["min"]?.try &.to_i(strict: false) || 0 + seconds = regex_match.try &.["sec"]?.try &.to_i(strict: false) || 0 duration = (minutes*60 + seconds) @@ -485,6 +495,35 @@ private module Parsers return {{@type.name}} end end + + # Parses an InnerTube continuationItemRenderer into a Continuation. + # Returns nil when the given object isn't a continuationItemRenderer. + # + # continuationItemRenderer contains various metadata ued to load more + # content (i.e when the user scrolls down). The interesting bit is the + # protobuf object known as the "continutation token". Previously, those + # were generated from sratch, but recent (as of 11/2022) Youtube changes + # are forcing us to extract them from replies. + # + module ContinuationItemRendererParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["continuationItemRenderer"]? + return self.parse(item_contents) + end + end + + private def self.parse(item_contents) + token = item_contents + .dig?("continuationEndpoint", "continuationCommand", "token") + .try &.as_s + + return Continuation.new(token) if token + end + + def self.parser_name + return {{@type.name}} + end + end end # The following are the extractors for extracting an array of items from @@ -520,7 +559,7 @@ private module Extractors # }] # module YouTubeTabs - def self.process(initial_data : Hash(String, JSON::Any)) + def self.process(initial_data : InitialData) if target = initial_data["twoColumnBrowseResultsRenderer"]? self.extract(target) end @@ -585,7 +624,7 @@ private module Extractors # } # module SearchResults - def self.process(initial_data : Hash(String, JSON::Any)) + def self.process(initial_data : InitialData) if target = initial_data["twoColumnSearchResultsRenderer"]? self.extract(target) end @@ -618,8 +657,8 @@ private module Extractors # The way they are structured is too varied to be accurately written down here. # However, they all eventually lead to an array of parsable items after traversing # through the JSON structure. - module Continuation - def self.process(initial_data : Hash(String, JSON::Any)) + module ContinuationContent + def self.process(initial_data : InitialData) if target = initial_data["continuationContents"]? self.extract(target) elsif target = initial_data["appendContinuationItemsAction"]? @@ -705,8 +744,7 @@ end # Parses an item from Youtube's JSON response into a more usable structure. # The end result can either be a SearchVideo, SearchPlaylist or SearchChannel. -def extract_item(item : JSON::Any, author_fallback : String? = "", - author_id_fallback : String? = "") +def parse_item(item : JSON::Any, author_fallback : String? = "", author_id_fallback : String? = "") # We "allow" nil values but secretly use empty strings instead. This is to save us the # hassle of modifying every author_fallback and author_id_fallback arg usage # which is more often than not nil. @@ -716,24 +754,23 @@ def extract_item(item : JSON::Any, author_fallback : String? = "", # Each parser automatically validates the data given to see if the data is # applicable to itself. If not nil is returned and the next parser is attempted. ITEM_PARSERS.each do |parser| - LOGGER.trace("extract_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)") + LOGGER.trace("parse_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)") if result = parser.process(item, author_fallback) - LOGGER.debug("extract_item: Successfully parsed via #{parser.parser_name}") - + LOGGER.debug("parse_item: Successfully parsed via #{parser.parser_name}") return result else - LOGGER.trace("extract_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...") + LOGGER.trace("parse_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...") end end end # Parses multiple items from YouTube's initial JSON response into a more usable structure. # The end result is an array of SearchItem. -def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, - author_id_fallback : String? = nil) : Array(SearchItem) - items = [] of SearchItem - +# +# This function yields the container so that items can be parsed separately. +# +def extract_items(initial_data : InitialData, &block) if unpackaged_data = initial_data["contents"]?.try &.as_h elsif unpackaged_data = initial_data["response"]?.try &.as_h elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 0).try &.as_h @@ -741,24 +778,37 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri unpackaged_data = initial_data end - # This is identical to the parser cycling of extract_item(). + # This is identical to the parser cycling of parse_item(). ITEM_CONTAINER_EXTRACTOR.each do |extractor| LOGGER.trace("extract_items: Attempting to extract item container using \"#{extractor.extractor_name}\" (cycling...)") if container = extractor.process(unpackaged_data) LOGGER.debug("extract_items: Successfully unpacked container with \"#{extractor.extractor_name}\"") # Extract items in container - container.each do |item| - if parsed_result = extract_item(item, author_fallback, author_id_fallback) - items << parsed_result - end - end - - break + container.each { |item| yield item } else LOGGER.trace("extract_items: Extractor \"#{extractor.extractor_name}\" does not apply. Cycling to the next one...") end end +end + +# Wrapper using the block function above +def extract_items( + initial_data : InitialData, + author_fallback : String? = nil, + author_id_fallback : String? = nil +) : {Array(SearchItem), String?} + items = [] of SearchItem + continuation = nil + + extract_items(initial_data) do |item| + parsed = parse_item(item, author_fallback, author_id_fallback) + + case parsed + when .is_a?(Continuation) then continuation = parsed.token + when .is_a?(SearchItem) then items << parsed + end + end - return items + return items, continuation end diff --git a/src/invidious/yt_backend/extractors_utils.cr b/src/invidious/yt_backend/extractors_utils.cr index f8245160..0cb3c079 100644 --- a/src/invidious/yt_backend/extractors_utils.cr +++ b/src/invidious/yt_backend/extractors_utils.cr @@ -68,10 +68,10 @@ rescue ex return false end -def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) - extracted = extract_items(initial_data, author_fallback, author_id_fallback) +def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) : Array(SearchVideo) + extracted, _ = extract_items(initial_data, author_fallback, author_id_fallback) - target = [] of SearchItem + target = [] of (SearchItem | Continuation) extracted.each do |i| if i.is_a?(Category) i.contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video } @@ -79,28 +79,11 @@ def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : Str target << i end end - return target.select(SearchVideo).map(&.as(SearchVideo)) + + return target.select(SearchVideo) end def extract_selected_tab(tabs) # Extract the selected tab from the array of tabs Youtube returns return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"]?.try &.as_bool)[0]["tabRenderer"] end - -def fetch_continuation_token(items : Array(JSON::Any)) - # Fetches the continuation token from an array of items - return items.last["continuationItemRenderer"]? - .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s -end - -def fetch_continuation_token(initial_data : Hash(String, JSON::Any)) - # Fetches the continuation token from initial data - if initial_data["onResponseReceivedActions"]? - continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"] - else - tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]) - continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"] - end - - return fetch_continuation_token(continuation_items.as_a) -end |