diff options
| author | Wes van der Vleuten <16665772+WesVleuten@users.noreply.github.com> | 2023-01-21 23:35:39 +0100 |
|---|---|---|
| committer | Wes van der Vleuten <16665772+WesVleuten@users.noreply.github.com> | 2023-01-21 23:35:39 +0100 |
| commit | 420e12bb8b4809f819ef4f6cb06c33338d42f125 (patch) | |
| tree | 4d85abadb39c4a02a04d669da467eb8a4e610790 /src | |
| parent | 7fd205179b5707a2774d83866f5a35b2bd8cfe16 (diff) | |
| parent | 4e3884cae749ffcc39b77f2dcb6bc336c255d2e5 (diff) | |
| download | invidious-420e12bb8b4809f819ef4f6cb06c33338d42f125.tar.gz invidious-420e12bb8b4809f819ef4f6cb06c33338d42f125.tar.bz2 invidious-420e12bb8b4809f819ef4f6cb06c33338d42f125.zip | |
Merge remote-tracking branch 'upstream/master'
Diffstat (limited to 'src')
51 files changed, 2091 insertions, 1768 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index 58adaa35..d4f8e0fb 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -34,9 +34,13 @@ require "protodec/utils" require "./invidious/database/*" require "./invidious/database/migrations/*" +require "./invidious/http_server/*" require "./invidious/helpers/*" require "./invidious/yt_backend/*" require "./invidious/frontend/*" +require "./invidious/videos/*" + +require "./invidious/jsonify/**" require "./invidious/*" require "./invidious/channels/*" @@ -45,6 +49,13 @@ require "./invidious/search/*" require "./invidious/routes/**" require "./invidious/jobs/**" +# Declare the base namespace for invidious +module Invidious +end + +# Simple alias to make code easier to read +alias IV = Invidious + CONFIG = Config.load HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32) @@ -169,7 +180,7 @@ if CONFIG.popular_enabled Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB) end -CONNECTION_CHANNEL = Channel({Bool, Channel(PQ::Notification)}).new(32) +CONNECTION_CHANNEL = ::Channel({Bool, ::Channel(PQ::Notification)}).new(32) Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL, CONFIG.database_url) Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index f60ee7af..0054f8f2 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -16,12 +16,6 @@ record AboutChannel, tabs : Array(String), verified : Bool -record AboutRelatedChannel, - ucid : String, - author : String, - author_url : String, - author_thumbnail : String - def get_about_info(ucid, locale) : AboutChannel begin # "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"} @@ -100,38 +94,51 @@ def get_about_info(ucid, locale) : AboutChannel total_views = 0_i64 joined = Time.unix(0) - tabs = [] of String - - tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?.try &.as_a? - if !tabs_json.nil? - # Retrieve information from the tabs array. The index we are looking for varies between channels. - tabs_json.each do |node| - # Try to find the about section which is located in only one of the tabs. - channel_about_meta = node["tabRenderer"]?.try &.["content"]?.try &.["sectionListRenderer"]? - .try &.["contents"]?.try &.[0]?.try &.["itemSectionRenderer"]?.try &.["contents"]? - .try &.[0]?.try &.["channelAboutFullMetadataRenderer"]? - - if !channel_about_meta.nil? - total_views = channel_about_meta["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 - - # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. - joined = channel_about_meta["joinedDateText"]?.try &.["runs"]?.try &.as_a.reduce("") { |acc, nd| acc + nd["text"].as_s } - .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) - - # Normal Auto-generated channels - # https://support.google.com/youtube/answer/2579942 - # For auto-generated channels, channel_about_meta only has ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] - if (channel_about_meta["primaryLinks"]?.try &.size || 0) == 1 && (channel_about_meta["primaryLinks"][0]?) && - (channel_about_meta["primaryLinks"][0]["title"]?.try &.["simpleText"]?.try &.as_s? || "") == "Auto-generated by YouTube" - auto_generated = true - end - end + tab_names = [] of String + + if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]? + # Get the name of the tabs available on this channel + tab_names = tabs_json.as_a.compact_map do |entry| + name = entry.dig?("tabRenderer", "title").try &.as_s.downcase + + # This is a small fix to not add extra code on the HTML side + # I.e, the URL for the "live" tab is .../streams, so use "streams" + # everywhere for the sake of simplicity + (name == "live") ? "streams" : name + end + + # Get the currently active tab ("About") + about_tab = extract_selected_tab(tabs_json) + + # Try to find the about metadata section + channel_about_meta = about_tab.dig?( + "content", + "sectionListRenderer", "contents", 0, + "itemSectionRenderer", "contents", 0, + "channelAboutFullMetadataRenderer" + ) + + if !channel_about_meta.nil? + total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 + + # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. + joined = extract_text(channel_about_meta["joinedDateText"]?) + .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) + + # Normal Auto-generated channels + # https://support.google.com/youtube/answer/2579942 + # For auto-generated channels, channel_about_meta only has + # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] + auto_generated = ( + (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \ + extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" + ) end - tabs = tabs_json.reject { |node| node["tabRenderer"]?.nil? }.map(&.["tabRenderer"]["title"].as_s.downcase) end - sub_count = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s? - .try { |text| short_text_to_number(text.split(" ")[0]) } || 0 + sub_count = initdata + .dig?("header", "c4TabbedHeaderRenderer", "subscriberCountText", "simpleText").try &.as_s? + .try { |text| short_text_to_number(text.split(" ")[0]).to_i32 } || 0 AboutChannel.new( ucid: ucid, @@ -147,46 +154,20 @@ def get_about_info(ucid, locale) : AboutChannel joined: joined, is_family_friendly: is_family_friendly, allowed_regions: allowed_regions, - tabs: tabs, + tabs: tab_names, verified: author_verified || false, ) end -def fetch_related_channels(about_channel : AboutChannel) : Array(AboutRelatedChannel) - # params is {"2:string":"channels"} encoded - channels = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D") - - tabs = channels.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs").try(&.as_a?) || [] of JSON::Any - tab = tabs.find(&.dig?("tabRenderer", "title").try(&.as_s?).try(&.== "Channels")) - - return [] of AboutRelatedChannel if tab.nil? - - items = tab.dig?( - "tabRenderer", "content", - "sectionListRenderer", "contents", 0, - "itemSectionRenderer", "contents", 0, - "gridRenderer", "items" - ).try &.as_a? - - related = [] of AboutRelatedChannel - return related if (items.nil? || items.empty?) - - items.each do |item| - renderer = item["gridChannelRenderer"]? - next if !renderer - - related_id = renderer.dig("channelId").as_s - related_title = renderer.dig("title", "simpleText").as_s - related_author_url = renderer.dig("navigationEndpoint", "browseEndpoint", "canonicalBaseUrl").as_s - related_author_thumbnail = HelperExtractors.get_thumbnails(renderer) - - related << AboutRelatedChannel.new( - ucid: related_id, - author: related_title, - author_url: related_author_url, - author_thumbnail: related_author_thumbnail, - ) +def fetch_related_channels(about_channel : AboutChannel, continuation : String? = nil) : {Array(SearchChannel), String?} + if continuation.nil? + # params is {"2:string":"channels"} encoded + initial_data = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D") + else + initial_data = YoutubeAPI.browse(continuation) end - return related + items, continuation = extract_items(initial_data) + + return items.select(SearchChannel), continuation end diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index e0459cc3..63dd2194 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -29,7 +29,7 @@ struct ChannelVideo json.field "title", self.title json.field "videoId", self.id json.field "videoThumbnails" do - generate_thumbnails(json, self.id) + Invidious::JSONify::APIv1.thumbnails(json, self.id) end json.field "lengthSeconds", self.length_seconds @@ -180,11 +180,16 @@ def fetch_channel(ucid, pull_all_videos : Bool) LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}") - page = 1 + channel = InvidiousChannel.new({ + id: ucid, + author: author, + updated: Time.utc, + deleted: false, + subscribed: nil, + }) LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page") - initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated) - videos = extract_videos(initial_data, author, ucid) + videos, continuation = IV::Channel::Tabs.get_videos(channel) LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed") rss.xpath_nodes("//feed/entry").each do |entry| @@ -197,7 +202,9 @@ def fetch_channel(ucid, pull_all_videos : Bool) views = entry.xpath_node("group/community/statistics").try &.["views"]?.try &.to_i64? views ||= 0_i64 - channel_video = videos.select { |video| video.id == video_id }[0]? + channel_video = videos + .select(SearchVideo) + .select(&.id.== video_id)[0]? length_seconds = channel_video.try &.length_seconds length_seconds ||= 0 @@ -228,58 +235,56 @@ def fetch_channel(ucid, pull_all_videos : Bool) if was_insert LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions") - Invidious::Database::Users.add_notification(video) + if CONFIG.enable_user_notifications + Invidious::Database::Users.add_notification(video) + else + Invidious::Database::Users.feed_needs_update(video) + end else LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated") end end if pull_all_videos - page += 1 - - ids = [] of String - loop do - initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated) - videos = extract_videos(initial_data, author, ucid) - - count = videos.size - videos = videos.map { |video| ChannelVideo.new({ - id: video.id, - title: video.title, - published: video.published, - updated: Time.utc, - ucid: video.ucid, - author: video.author, - length_seconds: video.length_seconds, - live_now: video.live_now, - premiere_timestamp: video.premiere_timestamp, - views: video.views, - }) } - - videos.each do |video| - ids << video.id + # Keep fetching videos using the continuation token retrieved earlier + videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation) + + count = 0 + videos.select(SearchVideo).each do |video| + count += 1 + video = ChannelVideo.new({ + id: video.id, + title: video.title, + published: video.published, + updated: Time.utc, + ucid: video.ucid, + author: video.author, + length_seconds: video.length_seconds, + live_now: video.live_now, + premiere_timestamp: video.premiere_timestamp, + views: video.views, + }) # We are notified of Red videos elsewhere (PubSub), which includes a correct published date, # so since they don't provide a published date here we can safely ignore them. if Time.utc - video.published > 1.minute was_insert = Invidious::Database::ChannelVideos.insert(video) - Invidious::Database::Users.add_notification(video) if was_insert + if was_insert + if CONFIG.enable_user_notifications + Invidious::Database::Users.add_notification(video) + else + Invidious::Database::Users.feed_needs_update(video) + end + end end end break if count < 25 - page += 1 + sleep 500.milliseconds end end - channel = InvidiousChannel.new({ - id: ucid, - author: author, - updated: Time.utc, - deleted: false, - subscribed: nil, - }) - + channel.updated = Time.utc return channel end diff --git a/src/invidious/channels/community.cr b/src/invidious/channels/community.cr index 2a2c74aa..8e300288 100644 --- a/src/invidious/channels/community.cr +++ b/src/invidious/channels/community.cr @@ -138,7 +138,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) json.field "title", video_title json.field "videoId", video_id json.field "videoThumbnails" do - generate_thumbnails(json, video_id) + Invidious::JSONify::APIv1.thumbnails(json, video_id) end json.field "lengthSeconds", decode_length_seconds(attachment["lengthText"]["simpleText"].as_s) diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr index d5628f6a..8dc824b2 100644 --- a/src/invidious/channels/playlists.cr +++ b/src/invidious/channels/playlists.cr @@ -1,93 +1,28 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by) if continuation - response_json = YoutubeAPI.browse(continuation) - continuation_items = response_json["onResponseReceivedActions"]? - .try &.[0]["appendContinuationItemsAction"]["continuationItems"] - - return [] of SearchItem, nil if !continuation_items - - items = [] of SearchItem - continuation_items.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item| - extract_item(item, author, ucid).try { |t| items << t } - } - - continuation = continuation_items.as_a.last["continuationItemRenderer"]? - .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s - else - url = "/channel/#{ucid}/playlists?flow=list&view=1" - - case sort_by - when "last", "last_added" - # - when "oldest", "oldest_created" - url += "&sort=da" - when "newest", "newest_created" - url += "&sort=dd" - else nil # Ignore - end - - response = YT_POOL.client &.get(url) - initial_data = extract_initial_data(response.body) - return [] of SearchItem, nil if !initial_data - - items = extract_items(initial_data, author, ucid) - continuation = response.body.match(/"token":"(?<continuation>[^"]+)"/).try &.["continuation"]? - end - - return items, continuation -end - -# ## NOTE: DEPRECATED -# Reason -> Unstable -# The Protobuf object must be provided with an id of the last playlist from the current "page" -# in order to fetch the next one accurately -# (if the id isn't included, entries shift around erratically between pages, -# leading to repetitions and skip overs) -# -# Since it's impossible to produce the appropriate Protobuf without an id being provided by the user, -# it's better to stick to continuation tokens provided by the first request and onward -def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated = false) - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:base64" => { - "2:string" => "playlists", - "6:varint" => 2_i64, - "7:varint" => 1_i64, - "12:varint" => 1_i64, - "13:string" => "", - "23:varint" => 0_i64, - }, - }, - } - - if cursor - cursor = Base64.urlsafe_encode(cursor, false) if !auto_generated - object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = cursor - end - - if auto_generated - object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x32_i64 + initial_data = YoutubeAPI.browse(continuation) else - object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 1_i64 - case sort - when "oldest", "oldest_created" - object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 2_i64 - when "newest", "newest_created" - object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 3_i64 - when "last", "last_added" - object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 4_i64 - else nil # Ignore - end + params = + case sort_by + when "last", "last_added" + # Equivalent to "&sort=lad" + # {"2:string": "playlists", "3:varint": 4, "4:varint": 1, "6:varint": 1} + "EglwbGF5bGlzdHMYBCABMAE%3D" + when "oldest", "oldest_created" + # formerly "&sort=da" + # Not available anymore :c or maybe ?? + # {"2:string": "playlists", "3:varint": 2, "4:varint": 1, "6:varint": 1} + "EglwbGF5bGlzdHMYAiABMAE%3D" + # {"2:string": "playlists", "3:varint": 1, "4:varint": 1, "6:varint": 1} + # "EglwbGF5bGlzdHMYASABMAE%3D" + when "newest", "newest_created" + # Formerly "&sort=dd" + # {"2:string": "playlists", "3:varint": 3, "4:varint": 1, "6:varint": 1} + "EglwbGF5bGlzdHMYAyABMAE%3D" + end + + initial_data = YoutubeAPI.browse(ucid, params: params || "") end - object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"]))) - object["80226972:embedded"].delete("3:base64") - - continuation = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" + return extract_items(initial_data, author, ucid) end diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index b495e597..befec03d 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -16,6 +16,14 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } + sort_by_numerical = + case sort_by + when "newest" then 1_i64 + when "popular" then 2_i64 + when "oldest" then 3_i64 # Broken as of 10/2022 :c + else 1_i64 # Fallback to "newest" + end + object_inner_1 = { "110:embedded" => { "3:embedded" => { @@ -24,7 +32,7 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so "1:string" => object_inner_2_encoded, "2:string" => "00000000-0000-0000-0000-000000000000", }, - "3:varint" => 1_i64, + "3:varint" => sort_by_numerical, }, }, }, @@ -52,34 +60,138 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so return continuation end -def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest") - continuation = produce_channel_videos_continuation(ucid, page, - auto_generated: auto_generated, sort_by: sort_by, v2: true) - - return YoutubeAPI.browse(continuation) +# Used in bypass_captcha_job.cr +def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) + continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2) + return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" end -def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") - videos = [] of SearchVideo +module Invidious::Channel::Tabs + extend self - # 2.times do |i| - # initial_data = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by) - initial_data = get_channel_videos_response(ucid, 1, auto_generated: auto_generated, sort_by: sort_by) - videos = extract_videos(initial_data, author, ucid) - # end + # ------------------- + # Regular videos + # ------------------- - return videos.size, videos -end + def make_initial_video_ctoken(ucid, sort_by) : String + return produce_channel_videos_continuation(ucid, sort_by: sort_by) + end -def get_latest_videos(ucid) - initial_data = get_channel_videos_response(ucid) - author = initial_data["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s + # Wrapper for AboutChannel, as we still need to call get_videos with + # an author name and ucid directly (e.g in RSS feeds). + # TODO: figure out how to get rid of that + def get_videos(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") + return get_videos( + channel.author, channel.ucid, + continuation: continuation, sort_by: sort_by + ) + end - return extract_videos(initial_data, author, ucid) -end + # Wrapper for InvidiousChannel, as we still need to call get_videos with + # an author name and ucid directly (e.g in RSS feeds). + # TODO: figure out how to get rid of that + def get_videos(channel : InvidiousChannel, *, continuation : String? = nil, sort_by = "newest") + return get_videos( + channel.author, channel.id, + continuation: continuation, sort_by: sort_by + ) + end -# Used in bypass_captcha_job.cr -def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2) - return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" + def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest") + continuation ||= make_initial_video_ctoken(ucid, sort_by) + initial_data = YoutubeAPI.browse(continuation: continuation) + + return extract_items(initial_data, author, ucid) + end + + def get_60_videos(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") + if continuation.nil? + # Fetch the first "page" of video + items, next_continuation = get_videos(channel, sort_by: sort_by) + else + # Fetch a "page" of videos using the given continuation token + items, next_continuation = get_videos(channel, continuation: continuation) + end + + # If there is more to load, then load a second "page" + # and replace the previous continuation token + if !next_continuation.nil? + items_2, next_continuation = get_videos(channel, continuation: next_continuation) + items.concat items_2 + end + + return items, next_continuation + end + + # ------------------- + # Shorts + # ------------------- + + private def fetch_shorts_data(ucid : String, continuation : String? = nil) + if continuation.nil? + # EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts" + # TODO: try to extract the continuation tokens that allows other sorting options + return YoutubeAPI.browse(ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D") + else + return YoutubeAPI.browse(continuation: continuation) + end + end + + def get_shorts(channel : AboutChannel, continuation : String? = nil) + initial_data = self.fetch_shorts_data(channel.ucid, continuation) + + begin + # Try to parse the initial data fetched above + return extract_items(initial_data, channel.author, channel.ucid) + rescue ex : RetryOnceException + # Sometimes, for a completely unknown reason, the "reelItemRenderer" + # object is missing some critical information (it happens once in about + # 20 subsequent requests). Refreshing the page is required to properly + # show the "shorts" tab. + # + # In order to make the experience smoother for the user, we simulate + # said page refresh by fetching again the JSON. If that still doesn't + # work, we raise a BrokenTubeException, as something is really broken. + begin + initial_data = self.fetch_shorts_data(channel.ucid, continuation) + return extract_items(initial_data, channel.author, channel.ucid) + rescue ex : RetryOnceException + raise BrokenTubeException.new "reelPlayerHeaderSupportedRenderers" + end + end + end + + # ------------------- + # Livestreams + # ------------------- + + def get_livestreams(channel : AboutChannel, continuation : String? = nil) + if continuation.nil? + # EgdzdHJlYW1z8gYECgJ6AA%3D%3D is the protobuf object to load "streams" + initial_data = YoutubeAPI.browse(channel.ucid, params: "EgdzdHJlYW1z8gYECgJ6AA%3D%3D") + else + initial_data = YoutubeAPI.browse(continuation: continuation) + end + + return extract_items(initial_data, channel.author, channel.ucid) + end + + def get_60_livestreams(channel : AboutChannel, continuation : String? = nil) + if continuation.nil? + # Fetch the first "page" of streams + items, next_continuation = get_livestreams(channel) + else + # Fetch a "page" of streams using the given continuation token + items, next_continuation = get_livestreams(channel, continuation: continuation) + end + + # If there is more to load, then load a second "page" + # and replace the previous continuation token + if !next_continuation.nil? + items_2, next_continuation = get_livestreams(channel, continuation: next_continuation) + items.concat items_2 + end + + return items, next_continuation + end end diff --git a/src/invidious/config.cr b/src/invidious/config.cr index c9bf43a4..9fc58409 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -110,6 +110,8 @@ class Config property hsts : Bool? = true # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local' property disable_proxy : Bool? | Array(String)? = false + # Enable the user notifications for all users + property enable_user_notifications : Bool = true # URL to the modified source code to be easily AGPL compliant # Will display in the footer, next to the main source code link diff --git a/src/invidious/database/users.cr b/src/invidious/database/users.cr index f62b43ea..0a4a4fd8 100644 --- a/src/invidious/database/users.cr +++ b/src/invidious/database/users.cr @@ -154,6 +154,16 @@ module Invidious::Database::Users # Update (misc) # ------------------- + def feed_needs_update(video : ChannelVideo) + request = <<-SQL + UPDATE users + SET feed_needs_update = true + WHERE $1 = ANY(subscriptions) + SQL + + PG_DB.exec(request, video.ucid) + end + def update_preferences(user : User) request = <<-SQL UPDATE users diff --git a/src/invidious/exceptions.cr b/src/invidious/exceptions.cr index 425c08da..690db907 100644 --- a/src/invidious/exceptions.cr +++ b/src/invidious/exceptions.cr @@ -33,3 +33,8 @@ end class VideoNotAvailableException < Exception end + +# Exception used to indicate that the JSON response from YT is missing +# some important informations, and that the query should be sent again. +class RetryOnceException < Exception +end diff --git a/src/invidious/frontend/channel_page.cr b/src/invidious/frontend/channel_page.cr new file mode 100644 index 00000000..53745dd5 --- /dev/null +++ b/src/invidious/frontend/channel_page.cr @@ -0,0 +1,44 @@ +module Invidious::Frontend::ChannelPage + extend self + + enum TabsAvailable + Videos + Shorts + Streams + Playlists + Community + Channels + end + + def generate_tabs_links(locale : String, channel : AboutChannel, selected_tab : TabsAvailable) + return String.build(1500) do |str| + base_url = "/channel/#{channel.ucid}" + + TabsAvailable.each do |tab| + # Ignore playlists, as it is not supported for auto-generated channels yet + next if (tab.playlists? && channel.auto_generated) + + tab_name = tab.to_s.downcase + + if channel.tabs.includes? tab_name + str << %(<div class="pure-u-1 pure-md-1-3">\n) + + if tab == selected_tab + str << "\t<b>" + str << translate(locale, "channel_tab_#{tab_name}_label") + str << "</b>\n" + else + # Video tab doesn't have the last path component + url = tab.videos? ? base_url : "#{base_url}/#{tab_name}" + + str << %(\t<a href=") << url << %(">) + str << translate(locale, "channel_tab_#{tab_name}_label") + str << "</a>\n" + end + + str << "</div>" + end + end + end + end +end diff --git a/src/invidious/frontend/watch_page.cr b/src/invidious/frontend/watch_page.cr index 80b67641..a9b00860 100644 --- a/src/invidious/frontend/watch_page.cr +++ b/src/invidious/frontend/watch_page.cr @@ -7,7 +7,7 @@ module Invidious::Frontend::WatchPage getter full_videos : Array(Hash(String, JSON::Any)) getter video_streams : Array(Hash(String, JSON::Any)) getter audio_streams : Array(Hash(String, JSON::Any)) - getter captions : Array(Caption) + getter captions : Array(Invidious::Videos::Caption) def initialize( @full_videos, @@ -50,7 +50,7 @@ module Invidious::Frontend::WatchPage video_assets.full_videos.each do |option| mimetype = option["mimeType"].as_s.split(";")[0] - height = itag_to_metadata?(option["itag"]).try &.["height"]? + height = Invidious::Videos::Formats.itag_to_metadata?(option["itag"]).try &.["height"]? value = {"itag": option["itag"], "ext": mimetype.split("/")[1]}.to_json diff --git a/src/invidious/hashtag.cr b/src/invidious/hashtag.cr index afe31a36..bc329205 100644 --- a/src/invidious/hashtag.cr +++ b/src/invidious/hashtag.cr @@ -8,7 +8,8 @@ module Invidious::Hashtag client_config = YoutubeAPI::ClientConfig.new(region: region) response = YoutubeAPI.browse(continuation: ctoken, client_config: client_config) - return extract_items(response) + items, _ = extract_items(response) + return items end def generate_continuation(hashtag : String, cursor : Int) diff --git a/src/invidious/helpers/json_filter.cr b/src/invidious/helpers/json_filter.cr index b8e8f96d..3f4080ba 100644 --- a/src/invidious/helpers/json_filter.cr +++ b/src/invidious/helpers/json_filter.cr @@ -20,7 +20,7 @@ module JSONFilter /^\(|\(\(|\/\(/ end - def self.parse_fields(fields_text : String) : Nil + def self.parse_fields(fields_text : String, &) : Nil if fields_text.empty? raise FieldsParser::ParseError.new "Fields is empty" end @@ -42,7 +42,7 @@ module JSONFilter parse_nest_groups(fields_text) { |nest_list| yield nest_list } end - def self.parse_single_nests(fields_text : String) : Nil + def self.parse_single_nests(fields_text : String, &) : Nil single_nests = remove_nest_groups(fields_text) if !single_nests.empty? @@ -60,7 +60,7 @@ module JSONFilter end end - def self.parse_nest_groups(fields_text : String) : Nil + def self.parse_nest_groups(fields_text : String, &) : Nil nest_stack = [] of NamedTuple(group_name: String, closing_bracket_index: Int64) bracket_pairs = get_bracket_pairs(fields_text, true) diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index 3918bd13..635f0984 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -76,7 +76,7 @@ struct SearchVideo json.field "authorUrl", "/channel/#{self.ucid}" json.field "videoThumbnails" do - generate_thumbnails(json, self.id) + Invidious::JSONify::APIv1.thumbnails(json, self.id) end json.field "description", html_to_content(self.description_html) @@ -155,7 +155,7 @@ struct SearchPlaylist json.field "lengthSeconds", video.length_seconds json.field "videoThumbnails" do - generate_thumbnails(json, video.id) + Invidious::JSONify::APIv1.thumbnails(json, video.id) end end end @@ -265,4 +265,11 @@ class Category end end +struct Continuation + getter token + + def initialize(@token : String) + end +end + alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 8ae5034a..ed0cca38 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -161,21 +161,19 @@ def number_with_separator(number) number.to_s.reverse.gsub(/(\d{3})(?=\d)/, "\\1,").reverse end -def short_text_to_number(short_text : String) : Int32 - case short_text - when .ends_with? "M" - number = short_text.rstrip(" mM").to_f - number *= 1000000 - when .ends_with? "K" - number = short_text.rstrip(" kK").to_f - number *= 1000 - else - number = short_text.rstrip(" ") +def short_text_to_number(short_text : String) : Int64 + matches = /(?<number>\d+(\.\d+)?)\s?(?<suffix>[mMkKbB])?/.match(short_text) + number = matches.try &.["number"].to_f || 0.0 + + case matches.try &.["suffix"].downcase + when "k" then number *= 1_000 + when "m" then number *= 1_000_000 + when "b" then number *= 1_000_000_000 end - number = number.to_i - - return number + return number.to_i64 +rescue ex + return 0_i64 end def number_to_short_text(number) diff --git a/src/invidious/http_server/utils.cr b/src/invidious/http_server/utils.cr new file mode 100644 index 00000000..e3f1fa0f --- /dev/null +++ b/src/invidious/http_server/utils.cr @@ -0,0 +1,20 @@ +module Invidious::HttpServer + module Utils + extend self + + def proxy_video_url(raw_url : String, *, region : String? = nil, absolute : Bool = false) + url = URI.parse(raw_url) + + # Add some URL parameters + params = url.query_params + params["host"] = url.host.not_nil! # Should never be nil, in theory + params["region"] = region if !region.nil? + + if absolute + return "#{HOST_URL}#{url.request_target}?#{params}" + else + return "#{url.request_target}?#{params}" + end + end + end +end diff --git a/src/invidious/jobs/notification_job.cr b/src/invidious/jobs/notification_job.cr index 2f525e08..b445107b 100644 --- a/src/invidious/jobs/notification_job.cr +++ b/src/invidious/jobs/notification_job.cr @@ -1,12 +1,12 @@ class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob - private getter connection_channel : Channel({Bool, Channel(PQ::Notification)}) + private getter connection_channel : ::Channel({Bool, ::Channel(PQ::Notification)}) private getter pg_url : URI def initialize(@connection_channel, @pg_url) end def begin - connections = [] of Channel(PQ::Notification) + connections = [] of ::Channel(PQ::Notification) PG.connect_listen(pg_url, "notifications") { |event| connections.each(&.send(event)) } diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr index 92681408..80812a63 100644 --- a/src/invidious/jobs/refresh_channels_job.cr +++ b/src/invidious/jobs/refresh_channels_job.cr @@ -8,7 +8,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob max_fibers = CONFIG.channel_threads lim_fibers = max_fibers active_fibers = 0 - active_channel = Channel(Bool).new + active_channel = ::Channel(Bool).new backoff = 2.minutes loop do diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr index 4b52c959..4f8130df 100644 --- a/src/invidious/jobs/refresh_feeds_job.cr +++ b/src/invidious/jobs/refresh_feeds_job.cr @@ -7,7 +7,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob def begin max_fibers = CONFIG.feed_threads active_fibers = 0 - active_channel = Channel(Bool).new + active_channel = ::Channel(Bool).new loop do db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs| diff --git a/src/invidious/jobs/subscribe_to_feeds_job.cr b/src/invidious/jobs/subscribe_to_feeds_job.cr index a431a48a..8584fb9c 100644 --- a/src/invidious/jobs/subscribe_to_feeds_job.cr +++ b/src/invidious/jobs/subscribe_to_feeds_job.cr @@ -12,7 +12,7 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob end active_fibers = 0 - active_channel = Channel(Bool).new + active_channel = ::Channel(Bool).new loop do db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > interval '4 days' OR subscribed IS NULL") do |rs| diff --git a/src/invidious/jsonify/api_v1/common.cr b/src/invidious/jsonify/api_v1/common.cr new file mode 100644 index 00000000..64b06465 --- /dev/null +++ b/src/invidious/jsonify/api_v1/common.cr @@ -0,0 +1,18 @@ +require "json" + +module Invidious::JSONify::APIv1 + extend self + + def thumbnails(json : JSON::Builder, id : String) + json.array do + build_thumbnails(id).each do |thumbnail| + json.object do + json.field "quality", thumbnail[:name] + json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg" + json.field "width", thumbnail[:width] + json.field "height", thumbnail[:height] + end + end + end + end +end diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr new file mode 100644 index 00000000..a2b1a35c --- /dev/null +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -0,0 +1,258 @@ +require "json" + +module Invidious::JSONify::APIv1 + extend self + + def video(video : Video, json : JSON::Builder, *, locale : String?, proxy : Bool = false) + json.object do + json.field "type", video.video_type + + json.field "title", video.title + json.field "videoId", video.id + + json.field "error", video.info["reason"] if video.info["reason"]? + + json.field "videoThumbnails" do + self.thumbnails(json, video.id) + end + json.field "storyboards" do + self.storyboards(json, video.id, video.storyboards) + end + + json.field "description", video.description + json.field "descriptionHtml", video.description_html + json.field "published", video.published.to_unix + json.field "publishedText", translate(locale, "`x` ago", recode_date(video.published, locale)) + json.field "keywords", video.keywords + + json.field "viewCount", video.views + json.field "likeCount", video.likes + json.field "dislikeCount", 0_i64 + + json.field "paid", video.paid + json.field "premium", video.premium + json.field "isFamilyFriendly", video.is_family_friendly + json.field "allowedRegions", video.allowed_regions + json.field "genre", video.genre + json.field "genreUrl", video.genre_url + + json.field "author", video.author + json.field "authorId", video.ucid + json.field "authorUrl", "/channel/#{video.ucid}" + + json.field "authorThumbnails" do + json.array do + qualities = {32, 48, 76, 100, 176, 512} + + qualities.each do |quality| + json.object do + json.field "url", video.author_thumbnail.gsub(/=s\d+/, "=s#{quality}") + json.field "width", quality + json.field "height", quality + end + end + end + end + + json.field "subCountText", video.sub_count_text + + json.field "lengthSeconds", video.length_seconds + json.field "allowRatings", video.allow_ratings + json.field "rating", 0_i64 + json.field "isListed", video.is_listed + json.field "liveNow", video.live_now + json.field "isUpcoming", video.is_upcoming + + if video.premiere_timestamp + json.field "premiereTimestamp", video.premiere_timestamp.try &.to_unix + end + + if hlsvp = video.hls_manifest_url + hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL) + json.field "hlsUrl", hlsvp + end + + json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{video.id}" + + json.field "adaptiveFormats" do + json.array do + video.adaptive_fmts.each do |fmt| + json.object do + # Only available on regular videos, not livestreams/OTF streams + if init_range = fmt["initRange"]? + json.field "init", "#{init_range["start"]}-#{init_range["end"]}" + end + if index_range = fmt["indexRange"]? + json.field "index", "#{index_range["start"]}-#{index_range["end"]}" + end + + # Not available on MPEG-4 Timed Text (`text/mp4`) streams (livestreams only) + json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]? + + if proxy + json.field "url", Invidious::HttpServer::Utils.proxy_video_url( + fmt["url"].to_s, absolute: true + ) + else + json.field "url", fmt["url"] + end + + json.field "itag", fmt["itag"].as_i.to_s + json.field "type", fmt["mimeType"] + json.field "clen", fmt["contentLength"]? || "-1" + + # Last modified is a unix timestamp with µS, with the dot omitted. + # E.g: 1638056732(.)141582 + # + # On livestreams, it's not present, so always fall back to the + # current unix timestamp (up to mS precision) for compatibility. + last_modified = fmt["lastModified"]? + last_modified ||= "#{Time.utc.to_unix_ms.to_s}000" + json.field "lmt", last_modified + + json.field "projectionType", fmt["projectionType"] + + if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) + fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 + json.field "fps", fps + json.field "container", fmt_info["ext"] + json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] + + if fmt_info["height"]? + json.field "resolution", "#{fmt_info["height"]}p" + + quality_label = "#{fmt_info["height"]}p" + if fps > 30 + quality_label += "60" + end + json.field "qualityLabel", quality_label + + if fmt_info["width"]? + json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" + end + end + end + + # Livestream chunk infos + json.field "targetDurationSec", fmt["targetDurationSec"].as_i if fmt.has_key?("targetDurationSec") + json.field "maxDvrDurationSec", fmt["maxDvrDurationSec"].as_i if fmt.has_key?("maxDvrDurationSec") + + # Audio-related data + json.field "audioQuality", fmt["audioQuality"] if fmt.has_key?("audioQuality") + json.field "audioSampleRate", fmt["audioSampleRate"].as_s.to_i if fmt.has_key?("audioSampleRate") + json.field "audioChannels", fmt["audioChannels"] if fmt.has_key?("audioChannels") + + # Extra misc stuff + json.field "colorInfo", fmt["colorInfo"] if fmt.has_key?("colorInfo") + json.field "captionTrack", fmt["captionTrack"] if fmt.has_key?("captionTrack") + end + end + end + end + + json.field "formatStreams" do + json.array do + video.fmt_stream.each do |fmt| + json.object do + json.field "url", fmt["url"] + json.field "itag", fmt["itag"].as_i.to_s + json.field "type", fmt["mimeType"] + json.field "quality", fmt["quality"] + + fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) + if fmt_info + fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 + json.field "fps", fps + json.field "container", fmt_info["ext"] + json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] + + if fmt_info["height"]? + json.field "resolution", "#{fmt_info["height"]}p" + + quality_label = "#{fmt_info["height"]}p" + if fps > 30 + quality_label += "60" + end + json.field "qualityLabel", quality_label + + if fmt_info["width"]? + json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" + end + end + end + end + end + end + end + + json.field "captions" do + json.array do + video.captions.each do |caption| + json.object do + json.field "label", caption.name + json.field "language_code", caption.language_code + json.field "url", "/api/v1/captions/#{video.id}?label=#{URI.encode_www_form(caption.name)}" + end + end + end + end + + json.field "recommendedVideos" do + json.array do + video.related_videos.each do |rv| + if rv["id"]? + json.object do + json.field "videoId", rv["id"] + json.field "title", rv["title"] + json.field "videoThumbnails" do + self.thumbnails(json, rv["id"]) + end + + json.field "author", rv["author"] + json.field "authorUrl", "/channel/#{rv["ucid"]?}" + json.field "authorId", rv["ucid"]? + if rv["author_thumbnail"]? + json.field "authorThumbnails" do + json.array do + qualities = {32, 48, 76, 100, 176, 512} + + qualities.each do |quality| + json.object do + json.field "url", rv["author_thumbnail"].gsub(/s\d+-/, "s#{quality}-") + json.field "width", quality + json.field "height", quality + end + end + end + end + end + + json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i + json.field "viewCountText", rv["short_view_count"]? + json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 + end + end + end + end + end + end + end + + def storyboards(json, id, storyboards) + json.array do + storyboards.each do |storyboard| + json.object do + json.field "url", "/api/v1/storyboards/#{id}?width=#{storyboard[:width]}&height=#{storyboard[:height]}" + json.field "templateUrl", storyboard[:url] + json.field "width", storyboard[:width] + json.field "height", storyboard[:height] + json.field "count", storyboard[:count] + json.field "interval", storyboard[:interval] + json.field "storyboardWidth", storyboard[:storyboard_width] + json.field "storyboardHeight", storyboard[:storyboard_height] + json.field "storyboardCount", storyboard[:storyboard_count] + end + end + end + end +end diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index c4eb7507..57f1f53e 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -56,7 +56,7 @@ struct PlaylistVideo json.field "authorUrl", "/channel/#{self.ucid}" json.field "videoThumbnails" do - generate_thumbnails(json, self.id) + Invidious::JSONify::APIv1.thumbnails(json, self.id) end if index diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index bfb8a377..662d1002 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -14,8 +14,6 @@ module Invidious::Routes::API::Manifest begin video = get_video(id, region: region) - rescue ex : VideoRedirect - return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex : NotFoundException haltf env, status_code: 404 rescue ex @@ -31,7 +29,7 @@ module Invidious::Routes::API::Manifest if local uri = URI.parse(url) - url = "#{uri.request_target}host/#{uri.host}/" + url = "#{HOST_URL}#{uri.request_target}host/#{uri.host}/" end "<BaseURL>#{url}</BaseURL>" @@ -44,7 +42,7 @@ module Invidious::Routes::API::Manifest if local adaptive_fmts.each do |fmt| - fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) + fmt["url"] = JSON::Any.new("#{HOST_URL}#{URI.parse(fmt["url"].as_s).request_target}") end end diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 6b81c546..ca2b2734 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -1,13 +1,7 @@ module Invidious::Routes::API::V1::Channels - def self.home(env) - locale = env.get("preferences").as(Preferences).locale - - env.response.content_type = "application/json" - - ucid = env.params.url["ucid"] - sort_by = env.params.query["sort_by"]?.try &.downcase - sort_by ||= "newest" - + # Macro to avoid duplicating some code below + # This sets the `channel` variable, or handles Exceptions. + private macro get_channel begin channel = get_about_info(ucid, locale) rescue ex : ChannelRedirect @@ -18,17 +12,25 @@ module Invidious::Routes::API::V1::Channels rescue ex return error_json(500, ex) end + end - page = 1 - if channel.auto_generated - videos = [] of SearchVideo - count = 0 - else - begin - count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) - rescue ex - return error_json(500, ex) - end + def self.home(env) + locale = env.get("preferences").as(Preferences).locale + ucid = env.params.url["ucid"] + + env.response.content_type = "application/json" + + # Use the private macro defined above. + channel = nil # Make the compiler happy + get_channel() + + # Retrieve "sort by" setting from URL parameters + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" + + begin + videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by) + rescue ex + return error_json(500, ex) end JSON.build do |json| @@ -100,31 +102,13 @@ module Invidious::Routes::API::V1::Channels json.array do # Fetch related channels begin - related_channels = fetch_related_channels(channel) + related_channels, _ = fetch_related_channels(channel) rescue ex - related_channels = [] of AboutRelatedChannel + related_channels = [] of SearchChannel end related_channels.each do |related_channel| - json.object do - json.field "author", related_channel.author - json.field "authorId", related_channel.ucid - json.field "authorUrl", related_channel.author_url - - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - - qualities.each do |quality| - json.object do - json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", quality - end - end - end - end - end + related_channel.to_json(locale, json) end end end # relatedChannels @@ -134,61 +118,112 @@ module Invidious::Routes::API::V1::Channels end def self.latest(env) + # Remove parameters that could affect this endpoint's behavior + env.params.query.delete("sort_by") if env.params.query.has_key?("sort_by") + env.params.query.delete("continuation") if env.params.query.has_key?("continuation") + + return self.videos(env) + end + + def self.videos(env) locale = env.get("preferences").as(Preferences).locale + ucid = env.params.url["ucid"] env.response.content_type = "application/json" - ucid = env.params.url["ucid"] + # Use the private macro defined above. + channel = nil # Make the compiler happy + get_channel() + + # Retrieve some URL parameters + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" + continuation = env.params.query["continuation"]? begin - videos = get_latest_videos(ucid) + videos, next_continuation = Channel::Tabs.get_60_videos( + channel, continuation: continuation, sort_by: sort_by + ) rescue ex return error_json(500, ex) end - JSON.build do |json| - json.array do - videos.each do |video| - video.to_json(locale, json) + return JSON.build do |json| + json.object do + json.field "videos" do + json.array do + videos.each &.to_json(locale, json) + end end + + json.field "continuation", next_continuation if next_continuation end end end - def self.videos(env) + def self.shorts(env) locale = env.get("preferences").as(Preferences).locale + ucid = env.params.url["ucid"] env.response.content_type = "application/json" - ucid = env.params.url["ucid"] - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - sort_by = env.params.query["sort"]?.try &.downcase - sort_by ||= env.params.query["sort_by"]?.try &.downcase - sort_by ||= "newest" + # Use the private macro defined above. + channel = nil # Make the compiler happy + get_channel() + + # Retrieve continuation from URL parameters + continuation = env.params.query["continuation"]? begin - channel = get_about_info(ucid, locale) - rescue ex : ChannelRedirect - env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) - return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) - rescue ex : NotFoundException - return error_json(404, ex) + videos, next_continuation = Channel::Tabs.get_shorts( + channel, continuation: continuation + ) rescue ex return error_json(500, ex) end + return JSON.build do |json| + json.object do + json.field "videos" do + json.array do + videos.each &.to_json(locale, json) + end + end + + json.field "continuation", next_continuation if next_continuation + end + end + end + + def self.streams(env) + locale = env.get("preferences").as(Preferences).locale + ucid = env.params.url["ucid"] + + env.response.content_type = "application/json" + + # Use the private macro defined above. + channel = nil # Make the compiler happy + get_channel() + + # Retrieve continuation from URL parameters + continuation = env.params.query["continuation"]? + begin - count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) + videos, next_continuation = Channel::Tabs.get_60_livestreams( + channel, continuation: continuation + ) rescue ex return error_json(500, ex) end - JSON.build do |json| - json.array do - videos.each do |video| - video.to_json(locale, json) + return JSON.build do |json| + json.object do + json.field "videos" do + json.array do + videos.each &.to_json(locale, json) + end end + + json.field "continuation", next_continuation if next_continuation end end end @@ -204,16 +239,9 @@ module Invidious::Routes::API::V1::Channels env.params.query["sort_by"]?.try &.downcase || "last" - begin - channel = get_about_info(ucid, locale) - rescue ex : ChannelRedirect - env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) - return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) - rescue ex : NotFoundException - return error_json(404, ex) - rescue ex - return error_json(500, ex) - end + # Use the macro defined above + channel = nil # Make the compiler happy + get_channel() items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) @@ -255,6 +283,37 @@ module Invidious::Routes::API::V1::Channels end end + def self.channels(env) + locale = env.get("preferences").as(Preferences).locale + ucid = env.params.url["ucid"] + + env.response.content_type = "application/json" + + # Use the macro defined above + channel = nil # Make the compiler happy + get_channel() + + continuation = env.params.query["continuation"]? + + begin + items, next_continuation = fetch_related_channels(channel, continuation) + rescue ex + return error_json(500, ex) + end + + JSON.build do |json| + json.object do + json.field "relatedChannels" do + json.array do + items.each &.to_json(locale, json) + end + end + + json.field "continuation", next_continuation if next_continuation + end + end + end + def self.search(env) locale = env.get("preferences").as(Preferences).locale region = env.params.query["region"]? diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index 844fedb8..43d360e6 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -124,7 +124,7 @@ module Invidious::Routes::API::V1::Misc json.field "videoThumbnails" do json.array do - generate_thumbnails(json, video.id) + Invidious::JSONify::APIv1.thumbnails(json, video.id) end end diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 1b7b4fa7..79f7bd3f 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -6,19 +6,19 @@ module Invidious::Routes::API::V1::Videos id = env.params.url["id"] region = env.params.query["region"]? + proxy = {"1", "true"}.any? &.== env.params.query["local"]? begin video = get_video(id, region: region) - rescue ex : VideoRedirect - env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) - return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) rescue ex : NotFoundException return error_json(404, ex) rescue ex return error_json(500, ex) end - video.to_json(locale, nil) + return JSON.build do |json| + Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy) + end end def self.captions(env) @@ -41,9 +41,6 @@ module Invidious::Routes::API::V1::Videos begin video = get_video(id, region: region) - rescue ex : VideoRedirect - env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) - return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) rescue ex : NotFoundException haltf env, 404 rescue ex @@ -168,9 +165,6 @@ module Invidious::Routes::API::V1::Videos begin video = get_video(id, region: region) - rescue ex : VideoRedirect - env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) - return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) rescue ex : NotFoundException haltf env, 404 rescue ex @@ -185,7 +179,7 @@ module Invidious::Routes::API::V1::Videos response = JSON.build do |json| json.object do json.field "storyboards" do - generate_storyboards(json, id, storyboards) + Invidious::JSONify::APIv1.storyboards(json, id, storyboards) end end end diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index c6e02cbd..d3969d29 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -7,21 +7,19 @@ module Invidious::Routes::Channels def self.videos(env) data = self.fetch_basic_information(env) - if !data.is_a?(Tuple) - return data - end - locale, user, subscriptions, continuation, ucid, channel = data + return data if !data.is_a?(Tuple) - page = env.params.query["page"]?.try &.to_i? - page ||= 1 + locale, user, subscriptions, continuation, ucid, channel = data sort_by = env.params.query["sort_by"]?.try &.downcase if channel.auto_generated sort_options = {"last", "oldest", "newest"} - sort_by ||= "last" - items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) + items, next_continuation = fetch_channel_playlists( + channel.ucid, channel.author, continuation, (sort_by || "last") + ) + items.uniq! do |item| if item.responds_to?(:title) item.title @@ -33,34 +31,85 @@ module Invidious::Routes::Channels items.each(&.author = "") else sort_options = {"newest", "oldest", "popular"} - sort_by ||= "newest" - count, items = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) + # Fetch items and continuation token + items, next_continuation = Channel::Tabs.get_videos( + channel, continuation: continuation, sort_by: (sort_by || "newest") + ) end + selected_tab = Frontend::ChannelPage::TabsAvailable::Videos templated "channel" end - def self.playlists(env) + def self.shorts(env) data = self.fetch_basic_information(env) - if !data.is_a?(Tuple) - return data + return data if !data.is_a?(Tuple) + + locale, user, subscriptions, continuation, ucid, channel = data + + if !channel.tabs.includes? "shorts" + return env.redirect "/channel/#{channel.ucid}" + end + + # TODO: support sort option for shorts + sort_by = "" + sort_options = [] of String + + # Fetch items and continuation token + items, next_continuation = Channel::Tabs.get_shorts( + channel, continuation: continuation + ) + + selected_tab = Frontend::ChannelPage::TabsAvailable::Shorts + templated "channel" + end + + def self.streams(env) + data = self.fetch_basic_information(env) + return data if !data.is_a?(Tuple) + + locale, user, subscriptions, continuation, ucid, channel = data + + if !channel.tabs.includes? "streams" + return env.redirect "/channel/#{channel.ucid}" end + + # TODO: support sort option for livestreams + sort_by = "" + sort_options = [] of String + + # Fetch items and continuation token + items, next_continuation = Channel::Tabs.get_60_livestreams( + channel, continuation: continuation + ) + + selected_tab = Frontend::ChannelPage::TabsAvailable::Streams + templated "channel" + end + + def self.playlists(env) + data = self.fetch_basic_information(env) + return data if !data.is_a?(Tuple) + locale, user, subscriptions, continuation, ucid, channel = data sort_options = {"last", "oldest", "newest"} sort_by = env.params.query["sort_by"]?.try &.downcase - sort_by ||= "last" if channel.auto_generated return env.redirect "/channel/#{channel.ucid}" end - items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) + items, next_continuation = fetch_channel_playlists( + channel.ucid, channel.author, continuation, (sort_by || "last") + ) + items = items.select(SearchPlaylist).map(&.as(SearchPlaylist)) items.each(&.author = "") - templated "playlists" + selected_tab = Frontend::ChannelPage::TabsAvailable::Playlists + templated "channel" end def self.community(env) @@ -74,12 +123,15 @@ module Invidious::Routes::Channels thin_mode = thin_mode == "true" continuation = env.params.query["continuation"]? - # sort_by = env.params.query["sort_by"]?.try &.downcase if !channel.tabs.includes? "community" return env.redirect "/channel/#{channel.ucid}" end + # TODO: support sort options for community posts + sort_by = "" + sort_options = [] of String + begin items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode)) rescue ex : InfoException @@ -95,6 +147,26 @@ module Invidious::Routes::Channels templated "community" end + def self.channels(env) + data = self.fetch_basic_information(env) + return data if !data.is_a?(Tuple) + + locale, user, subscriptions, continuation, ucid, channel = data + + if channel.auto_generated + return env.redirect "/channel/#{channel.ucid}" + end + + items, next_continuation = fetch_related_channels(channel, continuation) + + # Featured/related channels can't be sorted + sort_options = [] of String + sort_by = nil + + selected_tab = Frontend::ChannelPage::TabsAvailable::Channels + templated "channel" + end + def self.about(env) data = self.fetch_basic_information(env) if !data.is_a?(Tuple) @@ -125,7 +197,7 @@ module Invidious::Routes::Channels end selected_tab = env.request.path.split("/")[-1] - if ["home", "videos", "playlists", "community", "channels", "about"].includes? selected_tab + if {"home", "videos", "shorts", "streams", "playlists", "community", "channels", "about"}.includes? selected_tab url = "/channel/#{ucid}/#{selected_tab}" else url = "/channel/#{ucid}" diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index e6486587..266f7ba4 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -131,8 +131,6 @@ module Invidious::Routes::Embed begin video = get_video(id, region: params.region) - rescue ex : VideoRedirect - return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex : NotFoundException return error_template(404, ex) rescue ex @@ -149,7 +147,7 @@ module Invidious::Routes::Embed # PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.as(User).email) # end - if notifications && notifications.includes? id + if CONFIG.enable_user_notifications && notifications && notifications.includes? id Invidious::Database::Users.remove_notification(user.as(User), id) env.get("user").as(User).notifications.delete(id) notifications.delete(id) diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index b601db94..fb482e33 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -96,12 +96,14 @@ module Invidious::Routes::Feeds videos, notifications = get_subscription_feed(user, max_results, page) - # "updated" here is used for delivering new notifications, so if - # we know a user has looked at their feed e.g. in the past 10 minutes, - # they've already seen a video posted 20 minutes ago, and don't need - # to be notified. - Invidious::Database::Users.clear_notifications(user) - user.notifications = [] of String + if CONFIG.enable_user_notifications + # "updated" here is used for delivering new notifications, so if + # we know a user has looked at their feed e.g. in the past 10 minutes, + # they've already seen a video posted 20 minutes ago, and don't need + # to be notified. + Invidious::Database::Users.clear_notifications(user) + user.notifications = [] of String + end env.set "user", user templated "feeds/subscriptions" @@ -404,13 +406,15 @@ module Invidious::Routes::Feeds video = get_video(id, force_refresh: true) - # Deliver notifications to `/api/v1/auth/notifications` - payload = { - "topic" => video.ucid, - "videoId" => video.id, - "published" => published.to_unix, - }.to_json - PG_DB.exec("NOTIFY notifications, E'#{payload}'") + if CONFIG.enable_user_notifications + # Deliver notifications to `/api/v1/auth/notifications` + payload = { + "topic" => video.ucid, + "videoId" => video.id, + "published" => published.to_unix, + }.to_json + PG_DB.exec("NOTIFY notifications, E'#{payload}'") + end video = ChannelVideo.new({ id: id, @@ -426,7 +430,13 @@ module Invidious::Routes::Feeds }) was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true) - Invidious::Database::Users.add_notification(video) if was_insert + if was_insert + if CONFIG.enable_user_notifications + Invidious::Database::Users.add_notification(video) + else + Invidious::Database::Users.feed_needs_update(video) + end + end end end diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 560f9c19..1e932d11 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -35,6 +35,13 @@ module Invidious::Routes::VideoPlayback end end + # See: https://github.com/iv-org/invidious/issues/3302 + range_header = env.request.headers["Range"]? + if range_header.nil? + range_for_head = query_params["range"]? || "0-640" + headers["Range"] = "bytes=#{range_for_head}" + end + client = make_client(URI.parse(host), region) response = HTTP::Client::Response.new(500) error = "" @@ -70,6 +77,9 @@ module Invidious::Routes::VideoPlayback end end + # Remove the Range header added previously. + headers.delete("Range") if range_header.nil? + if response.status_code >= 400 env.response.content_type = "text/plain" haltf env, response.status_code @@ -91,14 +101,8 @@ module Invidious::Routes::VideoPlayback env.response.headers["Access-Control-Allow-Origin"] = "*" if location = resp.headers["Location"]? - location = URI.parse(location) - location = "#{location.request_target}&host=#{location.host}" - - if region - location += "®ion=#{region}" - end - - return env.redirect location + url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region) + return env.redirect url end IO.copy(resp.body_io, env.response) diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index fe1d8e54..5d3845c3 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -61,8 +61,6 @@ module Invidious::Routes::Watch begin video = get_video(id, region: params.region) - rescue ex : VideoRedirect - return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex : NotFoundException LOGGER.error("get_video not found: #{id} : #{ex.message}") return error_template(404, ex) @@ -82,7 +80,7 @@ module Invidious::Routes::Watch Invidious::Database::Users.mark_watched(user.as(User), id) end - if notifications && notifications.includes? id + if CONFIG.enable_user_notifications && notifications && notifications.includes? id Invidious::Database::Users.remove_notification(user.as(User), id) env.get("user").as(User).notifications.delete(id) notifications.delete(id) diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index f409f13c..491022a5 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -37,7 +37,9 @@ module Invidious::Routing get "/feed/webhook/:token", Routes::Feeds, :push_notifications_get post "/feed/webhook/:token", Routes::Feeds, :push_notifications_post - get "/modify_notifications", Routes::Notifications, :modify + if CONFIG.enable_user_notifications + get "/modify_notifications", Routes::Notifications, :modify + end {% end %} self.register_image_routes @@ -115,14 +117,17 @@ module Invidious::Routing get "/channel/:ucid", Routes::Channels, :home get "/channel/:ucid/home", Routes::Channels, :home get "/channel/:ucid/videos", Routes::Channels, :videos + get "/channel/:ucid/shorts", Routes::Channels, :shorts + get "/channel/:ucid/streams", Routes::Channels, :streams get "/channel/:ucid/playlists", Routes::Channels, :playlists get "/channel/:ucid/community", Routes::Channels, :community + get "/channel/:ucid/channels", Routes::Channels, :channels get "/channel/:ucid/about", Routes::Channels, :about get "/channel/:ucid/live", Routes::Channels, :live get "/user/:user/live", Routes::Channels, :live get "/c/:user/live", Routes::Channels, :live - ["", "/videos", "/playlists", "/community", "/about"].each do |path| + {"", "/videos", "/shorts", "/streams", "/playlists", "/community", "/about"}.each do |path| # /c/LinusTechTips get "/c/:user#{path}", Routes::Channels, :brand_redirect # /user/linustechtips | Not always the same as /c/ @@ -220,6 +225,10 @@ module Invidious::Routing # Channels get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home + get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts + get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams + get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels + {% for route in {"videos", "latest", "playlists", "community", "search"} %} get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}} get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}} @@ -260,8 +269,10 @@ module Invidious::Routing post "/api/v1/auth/tokens/register", {{namespace}}::Authenticated, :register_token post "/api/v1/auth/tokens/unregister", {{namespace}}::Authenticated, :unregister_token - get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications - post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + if CONFIG.enable_user_notifications + get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + end # Misc get "/api/v1/stats", {{namespace}}::Misc, :stats diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr index d1409c06..7e909590 100644 --- a/src/invidious/search/processors.cr +++ b/src/invidious/search/processors.cr @@ -9,7 +9,8 @@ module Invidious::Search client_config = YoutubeAPI::ClientConfig.new(region: query.region) initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config) - return extract_items(initial_data) + items, _ = extract_items(initial_data) + return items end # Search a youtube channel @@ -30,16 +31,7 @@ module Invidious::Search continuation = produce_channel_search_continuation(ucid, query.text, query.page) response_json = YoutubeAPI.browse(continuation) - continuation_items = response_json["onResponseReceivedActions"]? - .try &.[0]["appendContinuationItemsAction"]["continuationItems"] - - return [] of SearchItem if !continuation_items - - items = [] of SearchItem - continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item| - extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t } - end - + items, _ = extract_items(response_json, "", ucid) return items end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index c0ed6e85..d626c7d1 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -1,280 +1,22 @@ -CAPTION_LANGUAGES = { - "", - "English", - "English (auto-generated)", - "English (United Kingdom)", - "English (United States)", - "Afrikaans", - "Albanian", - "Amharic", - "Arabic", - "Armenian", - "Azerbaijani", - "Bangla", - "Basque", - "Belarusian", - "Bosnian", - "Bulgarian", - "Burmese", - "Cantonese (Hong Kong)", - "Catalan", - "Cebuano", - "Chinese", - "Chinese (China)", - "Chinese (Hong Kong)", - "Chinese (Simplified)", - "Chinese (Taiwan)", - "Chinese (Traditional)", - "Corsican", - "Croatian", - "Czech", - "Danish", - "Dutch", - "Dutch (auto-generated)", - "Esperanto", - "Estonian", - "Filipino", - "Finnish", - "French", - "French (auto-generated)", - "Galician", - "Georgian", - "German", - "German (auto-generated)", - "Greek", - "Gujarati", - "Haitian Creole", - "Hausa", - "Hawaiian", - "Hebrew", - "Hindi", - "Hmong", - "Hungarian", - "Icelandic", - "Igbo", - "Indonesian", - "Indonesian (auto-generated)", - "Interlingue", - "Irish", - "Italian", - "Italian (auto-generated)", - "Japanese", - "Japanese (auto-generated)", - "Javanese", - "Kannada", - "Kazakh", - "Khmer", - "Korean", - "Korean (auto-generated)", - "Kurdish", - "Kyrgyz", - "Lao", - "Latin", - "Latvian", - "Lithuanian", - "Luxembourgish", - "Macedonian", - "Malagasy", - "Malay", - "Malayalam", - "Maltese", - "Maori", - "Marathi", - "Mongolian", - "Nepali", - "Norwegian BokmÃ¥l", - "Nyanja", - "Pashto", - "Persian", - "Polish", - "Portuguese", - "Portuguese (auto-generated)", - "Portuguese (Brazil)", - "Punjabi", - "Romanian", - "Russian", - "Russian (auto-generated)", - "Samoan", - "Scottish Gaelic", - "Serbian", - "Shona", - "Sindhi", - "Sinhala", - "Slovak", - "Slovenian", - "Somali", - "Southern Sotho", - "Spanish", - "Spanish (auto-generated)", - "Spanish (Latin America)", - "Spanish (Mexico)", - "Spanish (Spain)", - "Sundanese", - "Swahili", - "Swedish", - "Tajik", - "Tamil", - "Telugu", - "Thai", - "Turkish", - "Turkish (auto-generated)", - "Ukrainian", - "Urdu", - "Uzbek", - "Vietnamese", - "Vietnamese (auto-generated)", - "Welsh", - "Western Frisian", - "Xhosa", - "Yiddish", - "Yoruba", - "Zulu", -} - -REGIONS = {"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW"} - -# See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476 -VIDEO_FORMATS = { - "5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, - "6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, - "13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"}, - "17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"}, - "18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"}, - "22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, - "34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - "35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - - "36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"}, - "37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, - "38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, - "43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, - "44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, - "45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, - "46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, - "59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - "78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - - # 3D videos - "82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - "83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - "84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, - "85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, - "100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, - "101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, - "102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, - - # Apple HTTP Live Streaming - "91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, - "92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, - "93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - "94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - "95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, - "96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, - "132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, - "151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"}, - - # DASH mp4 video - "133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"}, - "134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"}, - "135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, - "136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"}, - "137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"}, - "138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559) - "160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"}, - "212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, - "264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"}, - "298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, - "299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, - "266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"}, - - # Dash mp4 audio - "139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"}, - "140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"}, - "141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"}, - "256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, - "258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, - "325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"}, - "328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"}, - - # Dash webm - "167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, - "168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, - "169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, - "170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, - "218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, - "219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, - "278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"}, - "242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"}, - "243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"}, - "244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, - "245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, - "246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, - "247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"}, - "248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"}, - "271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"}, - # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug) - "272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, - "302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, - "315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - - # Dash webm audio - "171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128}, - "172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256}, - - # Dash webm audio with opus inside - "249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50}, - "250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70}, - "251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160}, - - # av01 video only formats sometimes served with "unknown" codecs - "394" => {"ext" => "mp4", "height" => 144, "vcodec" => "av01.0.05M.08"}, - "395" => {"ext" => "mp4", "height" => 240, "vcodec" => "av01.0.05M.08"}, - "396" => {"ext" => "mp4", "height" => 360, "vcodec" => "av01.0.05M.08"}, - "397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"}, -} - -struct VideoPreferences - include JSON::Serializable - - property annotations : Bool - property autoplay : Bool - property comments : Array(String) - property continue : Bool - property continue_autoplay : Bool - property controls : Bool - property listen : Bool - property local : Bool - property preferred_captions : Array(String) - property player_style : String - property quality : String - property quality_dash : String - property raw : Bool - property region : String? - property related_videos : Bool - property speed : Float32 | Float64 - property video_end : Float64 | Int32 - property video_loop : Bool - property extend_desc : Bool - property video_start : Float64 | Int32 - property volume : Int32 - property vr_mode : Bool - property save_player_pos : Bool +enum VideoType + Video + Livestream + Scheduled end struct Video include DB::Serializable + # Version of the JSON structure + # It prevents us from loading an incompatible version from cache + # (either newer or older, if instances with different versions run + # concurrently, e.g during a version upgrade rollout). + # + # NOTE: don't forget to bump this number if any change is made to + # the `params` structure in videos/parser.cr!!! + # + SCHEMA_VERSION = 2 + property id : String @[DB::Field(converter: Video::JSONConverter)] @@ -282,7 +24,7 @@ struct Video property updated : Time @[DB::Field(ignore: true)] - property captions : Array(Caption)? + @captions = [] of Invidious::Videos::Caption @[DB::Field(ignore: true)] property adaptive_fmts : Array(Hash(String, JSON::Any))? @@ -299,289 +41,45 @@ struct Video end end - def to_json(locale : String?, json : JSON::Builder) - json.object do - json.field "type", "video" - - json.field "title", self.title - json.field "videoId", self.id - - json.field "error", info["reason"] if info["reason"]? - - json.field "videoThumbnails" do - generate_thumbnails(json, self.id) - end - json.field "storyboards" do - generate_storyboards(json, self.id, self.storyboards) - end - - json.field "description", self.description - json.field "descriptionHtml", self.description_html - json.field "published", self.published.to_unix - json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) - json.field "keywords", self.keywords - - json.field "viewCount", self.views - json.field "likeCount", self.likes - json.field "dislikeCount", 0_i64 - - json.field "paid", self.paid - json.field "premium", self.premium - json.field "isFamilyFriendly", self.is_family_friendly - json.field "allowedRegions", self.allowed_regions - json.field "genre", self.genre - json.field "genreUrl", self.genre_url - - json.field "author", self.author - json.field "authorId", self.ucid - json.field "authorUrl", "/channel/#{self.ucid}" - - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - - qualities.each do |quality| - json.object do - json.field "url", self.author_thumbnail.gsub(/=s\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", quality - end - end - end - end - - json.field "subCountText", self.sub_count_text - - json.field "lengthSeconds", self.length_seconds - json.field "allowRatings", self.allow_ratings - json.field "rating", 0_i64 - json.field "isListed", self.is_listed - json.field "liveNow", self.live_now - json.field "isUpcoming", self.is_upcoming - - if self.premiere_timestamp - json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix - end - - if hlsvp = self.hls_manifest_url - hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL) - json.field "hlsUrl", hlsvp - end - - json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{id}" - - json.field "adaptiveFormats" do - json.array do - self.adaptive_fmts.each do |fmt| - json.object do - # Only available on regular videos, not livestreams/OTF streams - if init_range = fmt["initRange"]? - json.field "init", "#{init_range["start"]}-#{init_range["end"]}" - end - if index_range = fmt["indexRange"]? - json.field "index", "#{index_range["start"]}-#{index_range["end"]}" - end - - # Not available on MPEG-4 Timed Text (`text/mp4`) streams (livestreams only) - json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]? - - json.field "url", fmt["url"] - json.field "itag", fmt["itag"].as_i.to_s - json.field "type", fmt["mimeType"] - json.field "clen", fmt["contentLength"]? || "-1" - json.field "lmt", fmt["lastModified"] - json.field "projectionType", fmt["projectionType"] - - if fmt_info = itag_to_metadata?(fmt["itag"]) - fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 - json.field "fps", fps - json.field "container", fmt_info["ext"] - json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] - - if fmt_info["height"]? - json.field "resolution", "#{fmt_info["height"]}p" - - quality_label = "#{fmt_info["height"]}p" - if fps > 30 - quality_label += "60" - end - json.field "qualityLabel", quality_label - - if fmt_info["width"]? - json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" - end - end - end - - # Livestream chunk infos - json.field "targetDurationSec", fmt["targetDurationSec"].as_i if fmt.has_key?("targetDurationSec") - json.field "maxDvrDurationSec", fmt["maxDvrDurationSec"].as_i if fmt.has_key?("maxDvrDurationSec") - - # Audio-related data - json.field "audioQuality", fmt["audioQuality"] if fmt.has_key?("audioQuality") - json.field "audioSampleRate", fmt["audioSampleRate"].as_s.to_i if fmt.has_key?("audioSampleRate") - json.field "audioChannels", fmt["audioChannels"] if fmt.has_key?("audioChannels") - - # Extra misc stuff - json.field "colorInfo", fmt["colorInfo"] if fmt.has_key?("colorInfo") - json.field "captionTrack", fmt["captionTrack"] if fmt.has_key?("captionTrack") - end - end - end - end - - json.field "formatStreams" do - json.array do - self.fmt_stream.each do |fmt| - json.object do - json.field "url", fmt["url"] - json.field "itag", fmt["itag"].as_i.to_s - json.field "type", fmt["mimeType"] - json.field "quality", fmt["quality"] - - fmt_info = itag_to_metadata?(fmt["itag"]) - if fmt_info - fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 - json.field "fps", fps - json.field "container", fmt_info["ext"] - json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] - - if fmt_info["height"]? - json.field "resolution", "#{fmt_info["height"]}p" - - quality_label = "#{fmt_info["height"]}p" - if fps > 30 - quality_label += "60" - end - json.field "qualityLabel", quality_label - - if fmt_info["width"]? - json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" - end - end - end - end - end - end - end - - json.field "captions" do - json.array do - self.captions.each do |caption| - json.object do - json.field "label", caption.name - json.field "language_code", caption.language_code - json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}" - end - end - end - end + # Methods for API v1 JSON - json.field "recommendedVideos" do - json.array do - self.related_videos.each do |rv| - if rv["id"]? - json.object do - json.field "videoId", rv["id"] - json.field "title", rv["title"] - json.field "videoThumbnails" do - generate_thumbnails(json, rv["id"]) - end - - json.field "author", rv["author"] - json.field "authorUrl", "/channel/#{rv["ucid"]?}" - json.field "authorId", rv["ucid"]? - if rv["author_thumbnail"]? - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - - qualities.each do |quality| - json.object do - json.field "url", rv["author_thumbnail"].gsub(/s\d+-/, "s#{quality}-") - json.field "width", quality - json.field "height", quality - end - end - end - end - end - - json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i - json.field "viewCountText", rv["short_view_count"]? - json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 - end - end - end - end - end - end + def to_json(locale : String?, json : JSON::Builder) + Invidious::JSONify::APIv1.video(self, json, locale: locale) end # TODO: remove the locale and follow the crystal convention def to_json(locale : String?, _json : Nil) - JSON.build { |json| to_json(locale, json) } + JSON.build do |json| + Invidious::JSONify::APIv1.video(self, json, locale: locale) + end end def to_json(json : JSON::Builder | Nil = nil) to_json(nil, json) end - def title - info["videoDetails"]["title"]?.try &.as_s || "" - end + # Misc methods - def ucid - info["videoDetails"]["channelId"]?.try &.as_s || "" + def video_type : VideoType + video_type = info["videoType"]?.try &.as_s || "video" + return VideoType.parse?(video_type) || VideoType::Video end - def author - info["videoDetails"]["author"]?.try &.as_s || "" - end - - def length_seconds : Int32 - info.dig?("microformat", "playerMicroformatRenderer", "lengthSeconds").try &.as_s.to_i || - info["videoDetails"]["lengthSeconds"]?.try &.as_s.to_i || 0 - end - - def views : Int64 - info["videoDetails"]["viewCount"]?.try &.as_s.to_i64 || 0_i64 - end - - def likes : Int64 - info["likes"]?.try &.as_i64 || 0_i64 - end - - def dislikes : Int64 - info["dislikes"]?.try &.as_i64 || 0_i64 + def schema_version : Int + return info["version"]?.try &.as_i || 1 end def published : Time - info - .dig?("microformat", "playerMicroformatRenderer", "publishDate") + return info["published"]? .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc end def published=(other : Time) - info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d")) - end - - def allow_ratings - r = info["videoDetails"]["allowRatings"]?.try &.as_bool - r.nil? ? false : r + info["published"] = JSON::Any.new(other.to_s("%Y-%m-%d")) end def live_now - info["microformat"]?.try &.["playerMicroformatRenderer"]? - .try &.["liveBroadcastDetails"]?.try &.["isLiveNow"]?.try &.as_bool || false - end - - def is_listed - info["videoDetails"]["isCrawlable"]?.try &.as_bool || false - end - - def is_upcoming - info["videoDetails"]["isUpcoming"]?.try &.as_bool || false + return (self.video_type == VideoType::Livestream) end def premiere_timestamp : Time? @@ -590,31 +88,11 @@ struct Video .try { |t| Time.parse_rfc3339(t.as_s) } end - def keywords - info["videoDetails"]["keywords"]?.try &.as_a.map &.as_s || [] of String - end - def related_videos info["relatedVideos"]?.try &.as_a.map { |h| h.as_h.transform_values &.as_s } || [] of Hash(String, String) end - def allowed_regions - info - .dig?("microformat", "playerMicroformatRenderer", "availableCountries") - .try &.as_a.map &.as_s || [] of String - end - - def author_thumbnail : String - info["authorThumbnail"]?.try &.as_s || "" - end - - def author_verified : Bool - info["authorVerified"]?.try &.as_bool || false - end - - def sub_count_text : String - info["subCountText"]?.try &.as_s || "-" - end + # Methods for parsing streaming data def fmt_stream return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream @@ -665,6 +143,8 @@ struct Video adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("audio") end + # Misc. methods + def storyboards storyboards = info.dig?("storyboards", "playerStoryboardSpecRenderer", "spec") .try &.as_s.split("|") @@ -728,51 +208,19 @@ struct Video end def paid - reason = info.dig?("playabilityStatus", "reason").try &.as_s || "" - return reason.includes? "requires payment" + return (self.reason || "").includes? "requires payment" end def premium keywords.includes? "YouTube Red" end - def captions : Array(Caption) - return @captions.as(Array(Caption)) if @captions - captions = info["captions"]?.try &.["playerCaptionsTracklistRenderer"]?.try &.["captionTracks"]?.try &.as_a.map do |caption| - name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"] - language_code = caption["languageCode"].to_s - base_url = caption["baseUrl"].to_s - - caption = Caption.new(name.to_s, language_code, base_url) - caption.name = caption.name.split(" - ")[0] - caption + def captions : Array(Invidious::Videos::Caption) + if @captions.empty? && @info.has_key?("captions") + @captions = Invidious::Videos::Caption.from_yt_json(info["captions"]) end - captions ||= [] of Caption - @captions = captions - return @captions.as(Array(Caption)) - end - - def description - description = info - .dig?("microformat", "playerMicroformatRenderer", "description", "simpleText") - .try &.as_s || "" - end - - # TODO - def description=(value : String) - @description = value - end - def description_html - info["descriptionHtml"]?.try &.as_s || "<p></p>" - end - - def description_html=(value : String) - info["descriptionHtml"] = JSON::Any.new(value) - end - - def short_description - info["shortDescription"]?.try &.as_s? || "" + return @captions end def hls_manifest_url : String? @@ -783,25 +231,12 @@ struct Video info.dig?("streamingData", "dashManifestUrl").try &.as_s end - def genre : String - info["genre"]?.try &.as_s || "" - end - def genre_url : String? info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil end - def license : String? - info["license"]?.try &.as_s - end - - def is_family_friendly : Bool - info.dig?("microformat", "playerMicroformatRenderer", "isFamilySafe").try &.as_bool || false - end - def is_vr : Bool? - projection_type = info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s - return {"EQUIRECTANGULAR", "MESH"}.includes? projection_type + return {"EQUIRECTANGULAR", "MESH"}.includes? self.projection_type end def projection_type : String? @@ -811,290 +246,91 @@ struct Video def reason : String? info["reason"]?.try &.as_s end -end - -struct Caption - property name - property language_code - property base_url - - getter name : String - getter language_code : String - getter base_url : String - - setter name - - def initialize(@name, @language_code, @base_url) - end -end - -class VideoRedirect < Exception - property video_id : String - - def initialize(@video_id) - end -end - -# Use to parse both "compactVideoRenderer" and "endScreenVideoRenderer". -# The former is preferred as it has more videos in it. The second has -# the same 11 first entries as the compact rendered. -# -# TODO: "compactRadioRenderer" (Mix) and -# TODO: Use a proper struct/class instead of a hacky JSON object -def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? - return nil if !related["videoId"]? - - # The compact renderer has video length in seconds, where the end - # screen rendered has a full text version ("42:40") - length = related["lengthInSeconds"]?.try &.as_i.to_s - length ||= related.dig?("lengthText", "simpleText").try do |box| - decode_length_seconds(box.as_s).to_s - end - - # Both have "short", so the "long" option shouldn't be required - channel_info = (related["shortBylineText"]? || related["longBylineText"]?) - .try &.dig?("runs", 0) - author = channel_info.try &.dig?("text") - author_verified = has_verified_badge?(related["ownerBadges"]?).to_s + # Macros defining getters/setters for various types of data - ucid = channel_info.try { |ci| HelperExtractors.get_browse_id(ci) } - - # "4,088,033 views", only available on compact renderer - # and when video is not a livestream - view_count = related.dig?("viewCountText", "simpleText") - .try &.as_s.gsub(/\D/, "") - - short_view_count = related.try do |r| - HelperExtractors.get_short_view_count(r).to_s - end - - LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container") - - # TODO: when refactoring video types, make a struct for related videos - # or reuse an existing type, if that fits. - return { - "id" => related["videoId"], - "title" => related["title"]["simpleText"], - "author" => author || JSON::Any.new(""), - "ucid" => JSON::Any.new(ucid || ""), - "length_seconds" => JSON::Any.new(length || "0"), - "view_count" => JSON::Any.new(view_count || "0"), - "short_view_count" => JSON::Any.new(short_view_count || "0"), - "author_verified" => JSON::Any.new(author_verified), - } -end - -def extract_video_info(video_id : String, proxy_region : String? = nil, context_screen : String? = nil) - # Init client config for the API - client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region) - if context_screen == "embed" - client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed - end - - # Fetch data from the player endpoint - player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) - - playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s - - if playability_status != "OK" - subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason") - reason = subreason.try &.[]?("simpleText").try &.as_s - reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("") - reason ||= player_response.dig("playabilityStatus", "reason").as_s + private macro getset_string(name) + # Return {{name.stringify}} from `info` + def {{name.id.underscore}} : String + return info[{{name.stringify}}]?.try &.as_s || "" + end - # Stop here if video is not a scheduled livestream - if playability_status != "LIVE_STREAM_OFFLINE" - return { - "reason" => JSON::Any.new(reason), - } + # Update {{name.stringify}} into `info` + def {{name.id.underscore}}=(value : String) + info[{{name.stringify}}] = JSON::Any.new(value) end - elsif video_id != player_response.dig("videoDetails", "videoId") - # YouTube may return a different video player response than expected. - # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 - raise VideoNotAvailableException.new("The video returned by YouTube isn't the requested one. (WEB client)") - else - reason = nil - end - # Don't fetch the next endpoint if the video is unavailable. - if {"OK", "LIVE_STREAM_OFFLINE"}.any?(playability_status) - next_response = YoutubeAPI.next({"videoId": video_id, "params": ""}) - player_response = player_response.merge(next_response) + {% if flag?(:debug_macros) %} {{debug}} {% end %} end - params = parse_video_info(video_id, player_response) - params["reason"] = JSON::Any.new(reason) if reason - - # Fetch the video streams using an Android client in order to get the decrypted URLs and - # maybe fix throttling issues (#2194).See for the explanation about the decrypted URLs: - # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 - if reason.nil? - if context_screen == "embed" - client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed - else - client_config.client_type = YoutubeAPI::ClientType::Android + private macro getset_string_array(name) + # Return {{name.stringify}} from `info` + def {{name.id.underscore}} : Array(String) + return info[{{name.stringify}}]?.try &.as_a.map &.as_s || [] of String end - android_player = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) - - # Sometimes, the video is available from the web client, but not on Android, so check - # that here, and fallback to the streaming data from the web client if needed. - # See: https://github.com/iv-org/invidious/issues/2549 - if video_id != android_player.dig("videoDetails", "videoId") - # YouTube may return a different video player response than expected. - # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 - raise VideoNotAvailableException.new("The video returned by YouTube isn't the requested one. (ANDROID client)") - elsif android_player["playabilityStatus"]["status"] == "OK" - params["streamingData"] = android_player["streamingData"]? || JSON::Any.new("") - else - params["streamingData"] = player_response["streamingData"]? || JSON::Any.new("") + + # Update {{name.stringify}} into `info` + def {{name.id.underscore}}=(value : Array(String)) + info[{{name.stringify}}] = JSON::Any.new(value) end - end - # TODO: clean that up - {"captions", "microformat", "playabilityStatus", "storyboards", "videoDetails"}.each do |f| - params[f] = player_response[f] if player_response[f]? + {% if flag?(:debug_macros) %} {{debug}} {% end %} end - return params -end - -def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any) - # Top level elements - - main_results = player_response.dig?("contents", "twoColumnWatchNextResults") - - raise BrokenTubeException.new("twoColumnWatchNextResults") if !main_results - - primary_results = main_results.dig?("results", "results", "contents") - - raise BrokenTubeException.new("results") if !primary_results - - video_primary_renderer = primary_results - .as_a.find(&.["videoPrimaryInfoRenderer"]?) - .try &.["videoPrimaryInfoRenderer"] - - video_secondary_renderer = primary_results - .as_a.find(&.["videoSecondaryInfoRenderer"]?) - .try &.["videoSecondaryInfoRenderer"] - - raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer - raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer - - # Related videos - - LOGGER.debug("extract_video_info: parsing related videos...") + {% for op, type in {i32: Int32, i64: Int64} %} + private macro getset_{{op}}(name) + def \{{name.id.underscore}} : {{type}} + return info[\{{name.stringify}}]?.try &.as_i64.to_{{op}} || 0_{{op}} + end - related = [] of JSON::Any + def \{{name.id.underscore}}=(value : Int) + info[\{{name.stringify}}] = JSON::Any.new(value.to_i64) + end - # Parse "compactVideoRenderer" items (under secondary results) - secondary_results = main_results - .dig?("secondaryResults", "secondaryResults", "results") - secondary_results.try &.as_a.each do |element| - if item = element["compactVideoRenderer"]? - related_video = parse_related_video(item) - related << JSON::Any.new(related_video) if related_video + \{% if flag?(:debug_macros) %} \{{debug}} \{% end %} end - end + {% end %} - # If nothing was found previously, fall back to end screen renderer - if related.empty? - # Container for "endScreenVideoRenderer" items - player_overlays = player_response.dig?( - "playerOverlays", "playerOverlayRenderer", - "endScreen", "watchNextEndScreenRenderer", "results" - ) - - player_overlays.try &.as_a.each do |element| - if item = element["endScreenVideoRenderer"]? - related_video = parse_related_video(item) - related << JSON::Any.new(related_video) if related_video - end + private macro getset_bool(name) + # Return {{name.stringify}} from `info` + def {{name.id.underscore}} : Bool + return info[{{name.stringify}}]?.try &.as_bool || false end - end - - # Likes - toplevel_buttons = video_primary_renderer - .try &.dig?("videoActions", "menuRenderer", "topLevelButtons") - - if toplevel_buttons - likes_button = toplevel_buttons.as_a - .find(&.dig?("toggleButtonRenderer", "defaultIcon", "iconType").=== "LIKE") - .try &.["toggleButtonRenderer"] - - if likes_button - likes_txt = (likes_button["defaultText"]? || likes_button["toggledText"]?) - .try &.dig?("accessibility", "accessibilityData", "label") - likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt - - LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"") - LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes + # Update {{name.stringify}} into `info` + def {{name.id.underscore}}=(value : Bool) + info[{{name.stringify}}] = JSON::Any.new(value) end - end - - # Description - - short_description = player_response.dig?("videoDetails", "shortDescription") - - description_html = video_secondary_renderer.try &.dig?("description", "runs") - .try &.as_a.try { |t| content_to_comment_html(t, video_id) } - - # Video metadata - - metadata = video_secondary_renderer - .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows") - .try &.as_a - - genre = player_response.dig?("microformat", "playerMicroformatRenderer", "category") - genre_ucid = nil - license = nil - - metadata.try &.each do |row| - metadata_title = row.dig?("metadataRowRenderer", "title", "simpleText").try &.as_s - contents = row.dig?("metadataRowRenderer", "contents", 0) - if metadata_title == "Category" - contents = contents.try &.dig?("runs", 0) - - genre = contents.try &.["text"]? - genre_ucid = contents.try &.dig?("navigationEndpoint", "browseEndpoint", "browseId") - elsif metadata_title == "License" - license = contents.try &.dig?("runs", 0, "text") - elsif metadata_title == "Licensed to YouTube by" - license = contents.try &.["simpleText"]? - end + {% if flag?(:debug_macros) %} {{debug}} {% end %} end - # Author infos + # Method definitions, using the macros above - if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer") - author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url") - author_verified = has_verified_badge?(author_info["badges"]?) + getset_string author + getset_string authorThumbnail + getset_string description + getset_string descriptionHtml + getset_string genre + getset_string genreUcid + getset_string license + getset_string shortDescription + getset_string subCountText + getset_string title + getset_string ucid - subs_text = author_info["subscriberCountText"]? - .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") } - .try &.as_s.split(" ", 2)[0] - end + getset_string_array allowedRegions + getset_string_array keywords - # Return data - - params = { - "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil), - "relatedVideos" => JSON::Any.new(related), - "likes" => JSON::Any.new(likes || 0_i64), - "dislikes" => JSON::Any.new(0_i64), - "descriptionHtml" => JSON::Any.new(description_html || "<p></p>"), - "genre" => JSON::Any.new(genre.try &.as_s || ""), - "genreUrl" => JSON::Any.new(nil), - "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""), - "license" => JSON::Any.new(license.try &.as_s || ""), - "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), - "authorVerified" => JSON::Any.new(author_verified), - "subCountText" => JSON::Any.new(subs_text || "-"), - } + getset_i32 lengthSeconds + getset_i64 likes + getset_i64 views - return params + getset_bool allowRatings + getset_bool authorVerified + getset_bool isFamilyFriendly + getset_bool isListed + getset_bool isUpcoming end def get_video(id, refresh = true, region = nil, force_refresh = false) @@ -1104,7 +340,8 @@ def get_video(id, refresh = true, region = nil, force_refresh = false) if (refresh && (Time.utc - video.updated > 10.minutes) || (video.premiere_timestamp.try &.< Time.utc)) || - force_refresh + force_refresh || + video.schema_version != Video::SCHEMA_VERSION # cache control begin video = fetch_video(id, region) Invidious::Database::Videos.update(video) @@ -1143,12 +380,6 @@ def fetch_video(id, region) end end - # Try to fetch video info using an embedded client - if info["reason"]? - embed_info = extract_video_info(video_id: id, context_screen: "embed") - info = embed_info if !embed_info["reason"]? - end - if reason = info["reason"]? if reason == "Video unavailable" raise NotFoundException.new(reason.as_s || "") @@ -1166,10 +397,6 @@ def fetch_video(id, region) return video end -def itag_to_metadata?(itag : JSON::Any) - return VIDEO_FORMATS[itag.to_s]? -end - def process_continuation(query, plid, id) continuation = nil if plid @@ -1184,135 +411,6 @@ def process_continuation(query, plid, id) continuation end -def process_video_params(query, preferences) - annotations = query["iv_load_policy"]?.try &.to_i? - autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } - comments = query["comments"]?.try &.split(",").map(&.downcase) - continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe } - continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } - listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe } - local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe } - player_style = query["player_style"]? - preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase) - quality = query["quality"]? - quality_dash = query["quality_dash"]? - region = query["region"]? - related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe } - speed = query["speed"]?.try &.rchop("x").to_f? - video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe } - extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe } - volume = query["volume"]?.try &.to_i? - vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe } - save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe } - - if preferences - # region ||= preferences.region - annotations ||= preferences.annotations.to_unsafe - autoplay ||= preferences.autoplay.to_unsafe - comments ||= preferences.comments - continue ||= preferences.continue.to_unsafe - continue_autoplay ||= preferences.continue_autoplay.to_unsafe - listen ||= preferences.listen.to_unsafe - local ||= preferences.local.to_unsafe - player_style ||= preferences.player_style - preferred_captions ||= preferences.captions - quality ||= preferences.quality - quality_dash ||= preferences.quality_dash - related_videos ||= preferences.related_videos.to_unsafe - speed ||= preferences.speed - video_loop ||= preferences.video_loop.to_unsafe - extend_desc ||= preferences.extend_desc.to_unsafe - volume ||= preferences.volume - vr_mode ||= preferences.vr_mode.to_unsafe - save_player_pos ||= preferences.save_player_pos.to_unsafe - end - - annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe - autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe - comments ||= CONFIG.default_user_preferences.comments - continue ||= CONFIG.default_user_preferences.continue.to_unsafe - continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe - listen ||= CONFIG.default_user_preferences.listen.to_unsafe - local ||= CONFIG.default_user_preferences.local.to_unsafe - player_style ||= CONFIG.default_user_preferences.player_style - preferred_captions ||= CONFIG.default_user_preferences.captions - quality ||= CONFIG.default_user_preferences.quality - quality_dash ||= CONFIG.default_user_preferences.quality_dash - related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe - speed ||= CONFIG.default_user_preferences.speed - video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe - extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe - volume ||= CONFIG.default_user_preferences.volume - vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe - save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe - - annotations = annotations == 1 - autoplay = autoplay == 1 - continue = continue == 1 - continue_autoplay = continue_autoplay == 1 - listen = listen == 1 - local = local == 1 - related_videos = related_videos == 1 - video_loop = video_loop == 1 - extend_desc = extend_desc == 1 - vr_mode = vr_mode == 1 - save_player_pos = save_player_pos == 1 - - if CONFIG.disabled?("dash") && quality == "dash" - quality = "high" - end - - if CONFIG.disabled?("local") && local - local = false - end - - if start = query["t"]? || query["time_continue"]? || query["start"]? - video_start = decode_time(start) - end - video_start ||= 0 - - if query["end"]? - video_end = decode_time(query["end"]) - end - video_end ||= -1 - - raw = query["raw"]?.try &.to_i? - raw ||= 0 - raw = raw == 1 - - controls = query["controls"]?.try &.to_i? - controls ||= 1 - controls = controls >= 1 - - params = VideoPreferences.new({ - annotations: annotations, - autoplay: autoplay, - comments: comments, - continue: continue, - continue_autoplay: continue_autoplay, - controls: controls, - listen: listen, - local: local, - player_style: player_style, - preferred_captions: preferred_captions, - quality: quality, - quality_dash: quality_dash, - raw: raw, - region: region, - related_videos: related_videos, - speed: speed, - video_end: video_end, - video_loop: video_loop, - extend_desc: extend_desc, - video_start: video_start, - volume: volume, - vr_mode: vr_mode, - save_player_pos: save_player_pos, - }) - - return params -end - def build_thumbnails(id) return { {host: HOST_URL, height: 720, width: 1280, name: "maxres", url: "maxres"}, @@ -1326,34 +424,3 @@ def build_thumbnails(id) {host: HOST_URL, height: 90, width: 120, name: "end", url: "3"}, } end - -def generate_thumbnails(json, id) - json.array do - build_thumbnails(id).each do |thumbnail| - json.object do - json.field "quality", thumbnail[:name] - json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg" - json.field "width", thumbnail[:width] - json.field "height", thumbnail[:height] - end - end - end -end - -def generate_storyboards(json, id, storyboards) - json.array do - storyboards.each do |storyboard| - json.object do - json.field "url", "/api/v1/storyboards/#{id}?width=#{storyboard[:width]}&height=#{storyboard[:height]}" - json.field "templateUrl", storyboard[:url] - json.field "width", storyboard[:width] - json.field "height", storyboard[:height] - json.field "count", storyboard[:count] - json.field "interval", storyboard[:interval] - json.field "storyboardWidth", storyboard[:storyboard_width] - json.field "storyboardHeight", storyboard[:storyboard_height] - json.field "storyboardCount", storyboard[:storyboard_count] - end - end - end -end diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr new file mode 100644 index 00000000..4642c1a7 --- /dev/null +++ b/src/invidious/videos/caption.cr @@ -0,0 +1,168 @@ +require "json" + +module Invidious::Videos + struct Caption + property name : String + property language_code : String + property base_url : String + + def initialize(@name, @language_code, @base_url) + end + + # Parse the JSON structure from Youtube + def self.from_yt_json(container : JSON::Any) : Array(Caption) + caption_tracks = container + .dig?("playerCaptionsTracklistRenderer", "captionTracks") + .try &.as_a + + captions_list = [] of Caption + return captions_list if caption_tracks.nil? + + caption_tracks.each do |caption| + name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"] + name = name.to_s.split(" - ")[0] + + language_code = caption["languageCode"].to_s + base_url = caption["baseUrl"].to_s + + captions_list << Caption.new(name, language_code, base_url) + end + + return captions_list + end + + # List of all caption languages available on Youtube. + LANGUAGES = { + "", + "English", + "English (auto-generated)", + "English (United Kingdom)", + "English (United States)", + "Afrikaans", + "Albanian", + "Amharic", + "Arabic", + "Armenian", + "Azerbaijani", + "Bangla", + "Basque", + "Belarusian", + "Bosnian", + "Bulgarian", + "Burmese", + "Cantonese (Hong Kong)", + "Catalan", + "Cebuano", + "Chinese", + "Chinese (China)", + "Chinese (Hong Kong)", + "Chinese (Simplified)", + "Chinese (Taiwan)", + "Chinese (Traditional)", + "Corsican", + "Croatian", + "Czech", + "Danish", + "Dutch", + "Dutch (auto-generated)", + "Esperanto", + "Estonian", + "Filipino", + "Finnish", + "French", + "French (auto-generated)", + "Galician", + "Georgian", + "German", + "German (auto-generated)", + "Greek", + "Gujarati", + "Haitian Creole", + "Hausa", + "Hawaiian", + "Hebrew", + "Hindi", + "Hmong", + "Hungarian", + "Icelandic", + "Igbo", + "Indonesian", + "Indonesian (auto-generated)", + "Interlingue", + "Irish", + "Italian", + "Italian (auto-generated)", + "Japanese", + "Japanese (auto-generated)", + "Javanese", + "Kannada", + "Kazakh", + "Khmer", + "Korean", + "Korean (auto-generated)", + "Kurdish", + "Kyrgyz", + "Lao", + "Latin", + "Latvian", + "Lithuanian", + "Luxembourgish", + "Macedonian", + "Malagasy", + "Malay", + "Malayalam", + "Maltese", + "Maori", + "Marathi", + "Mongolian", + "Nepali", + "Norwegian BokmÃ¥l", + "Nyanja", + "Pashto", + "Persian", + "Polish", + "Portuguese", + "Portuguese (auto-generated)", + "Portuguese (Brazil)", + "Punjabi", + "Romanian", + "Russian", + "Russian (auto-generated)", + "Samoan", + "Scottish Gaelic", + "Serbian", + "Shona", + "Sindhi", + "Sinhala", + "Slovak", + "Slovenian", + "Somali", + "Southern Sotho", + "Spanish", + "Spanish (auto-generated)", + "Spanish (Latin America)", + "Spanish (Mexico)", + "Spanish (Spain)", + "Sundanese", + "Swahili", + "Swedish", + "Tajik", + "Tamil", + "Telugu", + "Thai", + "Turkish", + "Turkish (auto-generated)", + "Ukrainian", + "Urdu", + "Uzbek", + "Vietnamese", + "Vietnamese (auto-generated)", + "Welsh", + "Western Frisian", + "Xhosa", + "Yiddish", + "Yoruba", + "Zulu", + } + end +end diff --git a/src/invidious/videos/formats.cr b/src/invidious/videos/formats.cr new file mode 100644 index 00000000..e98e7257 --- /dev/null +++ b/src/invidious/videos/formats.cr @@ -0,0 +1,116 @@ +module Invidious::Videos::Formats + def self.itag_to_metadata?(itag : JSON::Any) + return FORMATS[itag.to_s]? + end + + # See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476 + private FORMATS = { + "5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, + "6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, + "13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"}, + "17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"}, + "18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"}, + "22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, + "34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + + "36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"}, + "37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, + "38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, + "43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, + "44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, + "45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, + "46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, + "59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + + # 3D videos + "82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, + "85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, + "100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, + "101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, + "102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, + + # Apple HTTP Live Streaming + "91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, + "92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, + "93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, + "96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, + "132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, + "151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"}, + + # DASH mp4 video + "133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"}, + "134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"}, + "135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, + "136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"}, + "137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"}, + "138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559) + "160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"}, + "212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, + "264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"}, + "298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, + "299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, + "266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"}, + + # Dash mp4 audio + "139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"}, + "140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"}, + "141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"}, + "256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, + "258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, + "325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"}, + "328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"}, + + # Dash webm + "167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"}, + "242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"}, + "243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"}, + "244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, + "245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, + "246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, + "247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"}, + "248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"}, + "271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"}, + # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug) + "272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, + "302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, + "315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + + # Dash webm audio + "171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128}, + "172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256}, + + # Dash webm audio with opus inside + "249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50}, + "250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70}, + "251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160}, + + # av01 video only formats sometimes served with "unknown" codecs + "394" => {"ext" => "mp4", "height" => 144, "vcodec" => "av01.0.05M.08"}, + "395" => {"ext" => "mp4", "height" => 240, "vcodec" => "av01.0.05M.08"}, + "396" => {"ext" => "mp4", "height" => 360, "vcodec" => "av01.0.05M.08"}, + "397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"}, + } +end diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr new file mode 100644 index 00000000..5c323975 --- /dev/null +++ b/src/invidious/videos/parser.cr @@ -0,0 +1,373 @@ +require "json" + +# Use to parse both "compactVideoRenderer" and "endScreenVideoRenderer". +# The former is preferred as it has more videos in it. The second has +# the same 11 first entries as the compact rendered. +# +# TODO: "compactRadioRenderer" (Mix) and +# TODO: Use a proper struct/class instead of a hacky JSON object +def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? + return nil if !related["videoId"]? + + # The compact renderer has video length in seconds, where the end + # screen rendered has a full text version ("42:40") + length = related["lengthInSeconds"]?.try &.as_i.to_s + length ||= related.dig?("lengthText", "simpleText").try do |box| + decode_length_seconds(box.as_s).to_s + end + + # Both have "short", so the "long" option shouldn't be required + channel_info = (related["shortBylineText"]? || related["longBylineText"]?) + .try &.dig?("runs", 0) + + author = channel_info.try &.dig?("text") + author_verified = has_verified_badge?(related["ownerBadges"]?).to_s + + ucid = channel_info.try { |ci| HelperExtractors.get_browse_id(ci) } + + # "4,088,033 views", only available on compact renderer + # and when video is not a livestream + view_count = related.dig?("viewCountText", "simpleText") + .try &.as_s.gsub(/\D/, "") + + short_view_count = related.try do |r| + HelperExtractors.get_short_view_count(r).to_s + end + + LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container") + + # TODO: when refactoring video types, make a struct for related videos + # or reuse an existing type, if that fits. + return { + "id" => related["videoId"], + "title" => related["title"]["simpleText"], + "author" => author || JSON::Any.new(""), + "ucid" => JSON::Any.new(ucid || ""), + "length_seconds" => JSON::Any.new(length || "0"), + "view_count" => JSON::Any.new(view_count || "0"), + "short_view_count" => JSON::Any.new(short_view_count || "0"), + "author_verified" => JSON::Any.new(author_verified), + } +end + +def extract_video_info(video_id : String, proxy_region : String? = nil) + # Init client config for the API + client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region) + + # Fetch data from the player endpoint + # 8AEB param is used to fetch YouTube stories + player_response = YoutubeAPI.player(video_id: video_id, params: "8AEB", client_config: client_config) + + playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s + + if playability_status != "OK" + subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason") + reason = subreason.try &.[]?("simpleText").try &.as_s + reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("") + reason ||= player_response.dig("playabilityStatus", "reason").as_s + + # Stop here if video is not a scheduled livestream or + # for LOGIN_REQUIRED when videoDetails element is not found because retrying won't help + if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) || + playability_status == "LOGIN_REQUIRED" && !player_response.dig?("videoDetails") + return { + "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), + "reason" => JSON::Any.new(reason), + } + end + elsif video_id != player_response.dig("videoDetails", "videoId") + # YouTube may return a different video player response than expected. + # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 + raise VideoNotAvailableException.new("The video returned by YouTube isn't the requested one. (WEB client)") + else + reason = nil + end + + # Don't fetch the next endpoint if the video is unavailable. + if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) + next_response = YoutubeAPI.next({"videoId": video_id, "params": ""}) + player_response = player_response.merge(next_response) + end + + params = parse_video_info(video_id, player_response) + params["reason"] = JSON::Any.new(reason) if reason + + new_player_response = nil + + if reason.nil? + # Fetch the video streams using an Android client in order to get the + # decrypted URLs and maybe fix throttling issues (#2194). See the + # following issue for an explanation about decrypted URLs: + # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 + client_config.client_type = YoutubeAPI::ClientType::Android + new_player_response = try_fetch_streaming_data(video_id, client_config) + elsif !reason.includes?("your country") # Handled separately + # The Android embedded client could help here + client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed + new_player_response = try_fetch_streaming_data(video_id, client_config) + end + + # Last hope + if new_player_response.nil? + client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed + new_player_response = try_fetch_streaming_data(video_id, client_config) + end + + # Replace player response and reset reason + if !new_player_response.nil? + player_response = new_player_response + params.delete("reason") + end + + {"captions", "playabilityStatus", "playerConfig", "storyboards", "streamingData"}.each do |f| + params[f] = player_response[f] if player_response[f]? + end + + # Data structure version, for cache control + params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64) + + return params +end + +def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)? + LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.") + # 8AEB param is used to fetch YouTube stories + response = YoutubeAPI.player(video_id: id, params: "8AEB", client_config: client_config) + + playability_status = response["playabilityStatus"]["status"] + LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.") + + if id != response.dig("videoDetails", "videoId") + # YouTube may return a different video player response than expected. + # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 + raise VideoNotAvailableException.new( + "The video returned by YouTube isn't the requested one. (#{client_config.client_type} client)" + ) + elsif playability_status == "OK" + return response + else + return nil + end +end + +def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any) + # Top level elements + + main_results = player_response.dig?("contents", "twoColumnWatchNextResults") + + raise BrokenTubeException.new("twoColumnWatchNextResults") if !main_results + + # Primary results are not available on Music videos + # See: https://github.com/iv-org/invidious/pull/3238#issuecomment-1207193725 + if primary_results = main_results.dig?("results", "results", "contents") + video_primary_renderer = primary_results + .as_a.find(&.["videoPrimaryInfoRenderer"]?) + .try &.["videoPrimaryInfoRenderer"] + + video_secondary_renderer = primary_results + .as_a.find(&.["videoSecondaryInfoRenderer"]?) + .try &.["videoSecondaryInfoRenderer"] + + raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer + raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer + end + + video_details = player_response.dig?("videoDetails") + microformat = player_response.dig?("microformat", "playerMicroformatRenderer") + + raise BrokenTubeException.new("videoDetails") if !video_details + raise BrokenTubeException.new("microformat") if !microformat + + # Basic video infos + + title = video_details["title"]?.try &.as_s + + # We have to try to extract viewCount from videoPrimaryInfoRenderer first, + # then from videoDetails, as the latter is "0" for livestreams (we want + # to get the amount of viewers watching). + views_txt = video_primary_renderer + .try &.dig?("viewCount", "videoViewCountRenderer", "viewCount", "runs", 0, "text") + views_txt ||= video_details["viewCount"]? + views = views_txt.try &.as_s.gsub(/\D/, "").to_i64? + + length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"]) + .try &.as_s.to_i64 + + published = microformat["publishDate"]? + .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc + + premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp") + .try { |t| Time.parse_rfc3339(t.as_s) } + + live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow") + .try &.as_bool || false + + # Extra video infos + + allowed_regions = microformat["availableCountries"]? + .try &.as_a.map &.as_s || [] of String + + allow_ratings = video_details["allowRatings"]?.try &.as_bool + family_friendly = microformat["isFamilySafe"].try &.as_bool + is_listed = video_details["isCrawlable"]?.try &.as_bool + is_upcoming = video_details["isUpcoming"]?.try &.as_bool + + keywords = video_details["keywords"]? + .try &.as_a.map &.as_s || [] of String + + # Related videos + + LOGGER.debug("extract_video_info: parsing related videos...") + + related = [] of JSON::Any + + # Parse "compactVideoRenderer" items (under secondary results) + secondary_results = main_results + .dig?("secondaryResults", "secondaryResults", "results") + secondary_results.try &.as_a.each do |element| + if item = element["compactVideoRenderer"]? + related_video = parse_related_video(item) + related << JSON::Any.new(related_video) if related_video + end + end + + # If nothing was found previously, fall back to end screen renderer + if related.empty? + # Container for "endScreenVideoRenderer" items + player_overlays = player_response.dig?( + "playerOverlays", "playerOverlayRenderer", + "endScreen", "watchNextEndScreenRenderer", "results" + ) + + player_overlays.try &.as_a.each do |element| + if item = element["endScreenVideoRenderer"]? + related_video = parse_related_video(item) + related << JSON::Any.new(related_video) if related_video + end + end + end + + # Likes + + toplevel_buttons = video_primary_renderer + .try &.dig?("videoActions", "menuRenderer", "topLevelButtons") + + if toplevel_buttons + likes_button = toplevel_buttons.try &.as_a + .find(&.dig?("toggleButtonRenderer", "defaultIcon", "iconType").=== "LIKE") + .try &.["toggleButtonRenderer"] + + # New format as of september 2022 + likes_button ||= toplevel_buttons.try &.as_a + .find(&.["segmentedLikeDislikeButtonRenderer"]?) + .try &.dig?( + "segmentedLikeDislikeButtonRenderer", + "likeButton", "toggleButtonRenderer" + ) + + if likes_button + # Note: The like count from `toggledText` is off by one, as it would + # represent the new like count in the event where the user clicks on "like". + likes_txt = (likes_button["defaultText"]? || likes_button["toggledText"]?) + .try &.dig?("accessibility", "accessibilityData", "label") + likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt + + LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"") + LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes + end + end + + # Description + + description = microformat.dig?("description", "simpleText").try &.as_s || "" + short_description = player_response.dig?("videoDetails", "shortDescription") + + description_html = video_secondary_renderer.try &.dig?("description", "runs") + .try &.as_a.try { |t| content_to_comment_html(t, video_id) } + + # Video metadata + + metadata = video_secondary_renderer + .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows") + .try &.as_a + + genre = microformat["category"]? + genre_ucid = nil + license = nil + + metadata.try &.each do |row| + metadata_title = extract_text(row.dig?("metadataRowRenderer", "title")) + contents = row.dig?("metadataRowRenderer", "contents", 0) + + if metadata_title == "Category" + contents = contents.try &.dig?("runs", 0) + + genre = contents.try &.["text"]? + genre_ucid = contents.try &.dig?("navigationEndpoint", "browseEndpoint", "browseId") + elsif metadata_title == "License" + license = contents.try &.dig?("runs", 0, "text") + elsif metadata_title == "Licensed to YouTube by" + license = contents.try &.["simpleText"]? + end + end + + # Author infos + + author = video_details["author"]?.try &.as_s + ucid = video_details["channelId"]?.try &.as_s + + if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer") + author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url") + author_verified = has_verified_badge?(author_info["badges"]?) + + subs_text = author_info["subscriberCountText"]? + .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") } + .try &.as_s.split(" ", 2)[0] + end + + # Return data + + if live_now + video_type = VideoType::Livestream + elsif !premiere_timestamp.nil? + video_type = VideoType::Scheduled + published = premiere_timestamp || Time.utc + else + video_type = VideoType::Video + end + + params = { + "videoType" => JSON::Any.new(video_type.to_s), + # Basic video infos + "title" => JSON::Any.new(title || ""), + "views" => JSON::Any.new(views || 0_i64), + "likes" => JSON::Any.new(likes || 0_i64), + "lengthSeconds" => JSON::Any.new(length_txt || 0_i64), + "published" => JSON::Any.new(published.to_rfc3339), + # Extra video infos + "allowedRegions" => JSON::Any.new(allowed_regions.map { |v| JSON::Any.new(v) }), + "allowRatings" => JSON::Any.new(allow_ratings || false), + "isFamilyFriendly" => JSON::Any.new(family_friendly || false), + "isListed" => JSON::Any.new(is_listed || false), + "isUpcoming" => JSON::Any.new(is_upcoming || false), + "keywords" => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }), + # Related videos + "relatedVideos" => JSON::Any.new(related), + # Description + "description" => JSON::Any.new(description || ""), + "descriptionHtml" => JSON::Any.new(description_html || "<p></p>"), + "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil), + # Video metadata + "genre" => JSON::Any.new(genre.try &.as_s || ""), + "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""), + "license" => JSON::Any.new(license.try &.as_s || ""), + # Author infos + "author" => JSON::Any.new(author || ""), + "ucid" => JSON::Any.new(ucid || ""), + "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), + "authorVerified" => JSON::Any.new(author_verified || false), + "subCountText" => JSON::Any.new(subs_text || "-"), + } + + return params +end diff --git a/src/invidious/videos/regions.cr b/src/invidious/videos/regions.cr new file mode 100644 index 00000000..575f8c25 --- /dev/null +++ b/src/invidious/videos/regions.cr @@ -0,0 +1,27 @@ +# List of geographical regions that Youtube recognizes. +# This is used to determine if a video is either restricted to a list +# of allowed regions (= whitelisted) or if it can't be watched in +# a set of regions (= blacklisted). +REGIONS = { + "AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", + "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", + "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", + "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", + "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", + "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", + "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", + "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", + "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", + "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", + "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", + "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", + "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", + "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", + "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", + "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", + "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", + "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", + "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", + "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", + "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW", +} diff --git a/src/invidious/videos/video_preferences.cr b/src/invidious/videos/video_preferences.cr new file mode 100644 index 00000000..34cf7ff0 --- /dev/null +++ b/src/invidious/videos/video_preferences.cr @@ -0,0 +1,156 @@ +struct VideoPreferences + include JSON::Serializable + + property annotations : Bool + property autoplay : Bool + property comments : Array(String) + property continue : Bool + property continue_autoplay : Bool + property controls : Bool + property listen : Bool + property local : Bool + property preferred_captions : Array(String) + property player_style : String + property quality : String + property quality_dash : String + property raw : Bool + property region : String? + property related_videos : Bool + property speed : Float32 | Float64 + property video_end : Float64 | Int32 + property video_loop : Bool + property extend_desc : Bool + property video_start : Float64 | Int32 + property volume : Int32 + property vr_mode : Bool + property save_player_pos : Bool +end + +def process_video_params(query, preferences) + annotations = query["iv_load_policy"]?.try &.to_i? + autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } + comments = query["comments"]?.try &.split(",").map(&.downcase) + continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe } + continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } + listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe } + local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe } + player_style = query["player_style"]? + preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase) + quality = query["quality"]? + quality_dash = query["quality_dash"]? + region = query["region"]? + related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe } + speed = query["speed"]?.try &.rchop("x").to_f? + video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe } + extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe } + volume = query["volume"]?.try &.to_i? + vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe } + save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe } + + if preferences + # region ||= preferences.region + annotations ||= preferences.annotations.to_unsafe + autoplay ||= preferences.autoplay.to_unsafe + comments ||= preferences.comments + continue ||= preferences.continue.to_unsafe + continue_autoplay ||= preferences.continue_autoplay.to_unsafe + listen ||= preferences.listen.to_unsafe + local ||= preferences.local.to_unsafe + player_style ||= preferences.player_style + preferred_captions ||= preferences.captions + quality ||= preferences.quality + quality_dash ||= preferences.quality_dash + related_videos ||= preferences.related_videos.to_unsafe + speed ||= preferences.speed + video_loop ||= preferences.video_loop.to_unsafe + extend_desc ||= preferences.extend_desc.to_unsafe + volume ||= preferences.volume + vr_mode ||= preferences.vr_mode.to_unsafe + save_player_pos ||= preferences.save_player_pos.to_unsafe + end + + annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe + autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe + comments ||= CONFIG.default_user_preferences.comments + continue ||= CONFIG.default_user_preferences.continue.to_unsafe + continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe + listen ||= CONFIG.default_user_preferences.listen.to_unsafe + local ||= CONFIG.default_user_preferences.local.to_unsafe + player_style ||= CONFIG.default_user_preferences.player_style + preferred_captions ||= CONFIG.default_user_preferences.captions + quality ||= CONFIG.default_user_preferences.quality + quality_dash ||= CONFIG.default_user_preferences.quality_dash + related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe + speed ||= CONFIG.default_user_preferences.speed + video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe + extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe + volume ||= CONFIG.default_user_preferences.volume + vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe + save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe + + annotations = annotations == 1 + autoplay = autoplay == 1 + continue = continue == 1 + continue_autoplay = continue_autoplay == 1 + listen = listen == 1 + local = local == 1 + related_videos = related_videos == 1 + video_loop = video_loop == 1 + extend_desc = extend_desc == 1 + vr_mode = vr_mode == 1 + save_player_pos = save_player_pos == 1 + + if CONFIG.disabled?("dash") && quality == "dash" + quality = "high" + end + + if CONFIG.disabled?("local") && local + local = false + end + + if start = query["t"]? || query["time_continue"]? || query["start"]? + video_start = decode_time(start) + end + video_start ||= 0 + + if query["end"]? + video_end = decode_time(query["end"]) + end + video_end ||= -1 + + raw = query["raw"]?.try &.to_i? + raw ||= 0 + raw = raw == 1 + + controls = query["controls"]?.try &.to_i? + controls ||= 1 + controls = controls >= 1 + + params = VideoPreferences.new({ + annotations: annotations, + autoplay: autoplay, + comments: comments, + continue: continue, + continue_autoplay: continue_autoplay, + controls: controls, + listen: listen, + local: local, + player_style: player_style, + preferred_captions: preferred_captions, + quality: quality, + quality_dash: quality_dash, + raw: raw, + region: region, + related_videos: related_videos, + speed: speed, + video_end: video_end, + video_loop: video_loop, + extend_desc: extend_desc, + video_start: video_start, + volume: volume, + vr_mode: vr_mode, + save_player_pos: save_player_pos, + }) + + return params +end diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr index 1295423e..931dd407 100644 --- a/src/invidious/views/channel.ecr +++ b/src/invidious/views/channel.ecr @@ -1,8 +1,24 @@ -<% ucid = channel.ucid %> -<% author = HTML.escape(channel.author) %> -<% channel_profile_pic = URI.parse(channel.author_thumbnail).request_target %> +<%- + ucid = channel.ucid + author = HTML.escape(channel.author) + channel_profile_pic = URI.parse(channel.author_thumbnail).request_target + + relative_url = + case selected_tab + when .shorts? then "/channel/#{ucid}/shorts" + when .streams? then "/channel/#{ucid}/streams" + when .playlists? then "/channel/#{ucid}/playlists" + when .channels? then "/channel/#{ucid}/channels" + else + "/channel/#{ucid}" + end + + youtube_url = "https://www.youtube.com#{relative_url}" + redirect_url = Invidious::Frontend::Misc.redirect_url(env) +-%> <% content_for "header" do %> +<%- if selected_tab.videos? -%> <meta name="description" content="<%= channel.description %>"> <meta property="og:site_name" content="Invidious"> <meta property="og:url" content="<%= HOST_URL %>/channel/<%= ucid %>"> @@ -14,91 +30,14 @@ <meta name="twitter:title" content="<%= author %>"> <meta name="twitter:description" content="<%= channel.description %>"> <meta name="twitter:image" content="/ggpht<%= channel_profile_pic %>"> -<link rel="alternate" href="https://www.youtube.com/channel/<%= ucid %>"> -<title><%= author %> - Invidious</title> <link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/channel/<%= ucid %>" /> -<% end %> - -<% if channel.banner %> - <div class="h-box"> - <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>"> - </div> +<%- end -%> - <div class="h-box"> - <hr> - </div> +<link rel="alternate" href="<%= youtube_url %>"> +<title><%= author %> - Invidious</title> <% end %> -<div class="pure-g h-box"> - <div class="pure-u-2-3"> - <div class="channel-profile"> - <img src="/ggpht<%= channel_profile_pic %>"> - <span><%= author %></span><% if !channel.verified.nil? && channel.verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end %> - </div> - </div> - <div class="pure-u-1-3"> - <h3 style="text-align:right"> - <a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a> - </h3> - </div> -</div> - -<div class="h-box"> - <div id="descriptionWrapper"> - <p><span style="white-space:pre-wrap"><%= channel.description_html %></span></p> - </div> -</div> - -<div class="h-box"> - <% sub_count_text = number_to_short_text(channel.sub_count) %> - <%= rendered "components/subscribe_widget" %> -</div> - -<div class="pure-g h-box"> - <div class="pure-u-1-3"> - <a href="https://www.youtube.com/channel/<%= ucid %>"><%= translate(locale, "View channel on YouTube") %></a> - <div class="pure-u-1 pure-md-1-3"> - <% if env.get("preferences").as(Preferences).automatic_instance_redirect%> - <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a> - <% else %> - <a href="https://redirect.invidious.io<%= env.request.path %>"><%= translate(locale, "Switch Invidious Instance") %></a> - <% end %> - </div> - <% if !channel.auto_generated %> - <div class="pure-u-1 pure-md-1-3"> - <b><%= translate(locale, "Videos") %></b> - </div> - <% end %> - <div class="pure-u-1 pure-md-1-3"> - <% if channel.auto_generated %> - <b><%= translate(locale, "Playlists") %></b> - <% else %> - <a href="/channel/<%= ucid %>/playlists"><%= translate(locale, "Playlists") %></a> - <% end %> - </div> - <div class="pure-u-1 pure-md-1-3"> - <% if channel.tabs.includes? "community" %> - <a href="/channel/<%= ucid %>/community"><%= translate(locale, "Community") %></a> - <% end %> - </div> - </div> - <div class="pure-u-1-3"></div> - <div class="pure-u-1-3"> - <div class="pure-g" style="text-align:right"> - <% sort_options.each do |sort| %> - <div class="pure-u-1 pure-md-1-3"> - <% if sort_by == sort %> - <b><%= translate(locale, sort) %></b> - <% else %> - <a href="/channel/<%= ucid %>?page=<%= page %>&sort_by=<%= sort %>"> - <%= translate(locale, sort) %> - </a> - <% end %> - </div> - <% end %> - </div> - </div> -</div> +<%= rendered "components/channel_info" %> <div class="h-box"> <hr> @@ -113,17 +52,10 @@ <script src="/js/watched_widget.js"></script> <div class="pure-g h-box"> - <div class="pure-u-1 pure-u-lg-1-5"> - <% if page > 1 %> - <a href="/channel/<%= ucid %>?page=<%= page - 1 %><% if sort_by != "newest" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>"> - <%= translate(locale, "Previous page") %> - </a> - <% end %> - </div> - <div class="pure-u-1 pure-u-lg-3-5"></div> + <div class="pure-u-1 pure-u-md-4-5"></div> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> - <% if count == 60 %> - <a href="/channel/<%= ucid %>?page=<%= page + 1 %><% if sort_by != "newest" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>"> + <% if next_continuation %> + <a href="<%= relative_url %>?continuation=<%= next_continuation %><% if sort_options.any? sort_by %>&sort_by=<%= sort_by %><% end %>"> <%= translate(locale, "Next page") %> </a> <% end %> diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr index 3bc29e55..9e11d562 100644 --- a/src/invidious/views/community.ecr +++ b/src/invidious/views/community.ecr @@ -1,71 +1,21 @@ -<% ucid = channel.ucid %> -<% author = HTML.escape(channel.author) %> +<%- + ucid = channel.ucid + author = HTML.escape(channel.author) + channel_profile_pic = URI.parse(channel.author_thumbnail).request_target -<% content_for "header" do %> -<title><%= author %> - Invidious</title> -<% end %> + relative_url = "/channel/#{ucid}/community" + youtube_url = "https://www.youtube.com#{relative_url}" + redirect_url = Invidious::Frontend::Misc.redirect_url(env) -<% if channel.banner %> - <div class="h-box"> - <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>"> - </div> + selected_tab = Invidious::Frontend::ChannelPage::TabsAvailable::Community +-%> - <div class="h-box"> - <hr> - </div> +<% content_for "header" do %> +<link rel="alternate" href="<%= youtube_url %>"> +<title><%= author %> - Invidious</title> <% end %> -<div class="pure-g h-box"> - <div class="pure-u-2-3"> - <div class="channel-profile"> - <img src="/ggpht<%= URI.parse(channel.author_thumbnail).request_target %>"> - <span><%= author %></span><% if !channel.verified.nil? && channel.verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end %> - </div> - </div> - <div class="pure-u-1-3" style="text-align:right"> - <h3 style="text-align:right"> - <a href="/feed/channel/<%= channel.ucid %>"><i class="icon ion-logo-rss"></i></a> - </h3> - </div> -</div> - -<div class="h-box"> - <div id="descriptionWrapper"> - <p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %></span></p> - </div> -</div> - -<div class="h-box"> - <% sub_count_text = number_to_short_text(channel.sub_count) %> - <%= rendered "components/subscribe_widget" %> -</div> - -<div class="pure-g h-box"> - <div class="pure-u-1-3"> - <a href="https://www.youtube.com/channel/<%= channel.ucid %>/community"><%= translate(locale, "View channel on YouTube") %></a> - <div class="pure-u-1 pure-md-1-3"> - <% if env.get("preferences").as(Preferences).automatic_instance_redirect%> - <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a> - <% else %> - <a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a> - <% end %> - </div> - <% if !channel.auto_generated %> - <div class="pure-u-1 pure-md-1-3"> - <a href="/channel/<%= channel.ucid %>"><%= translate(locale, "Videos") %></a> - </div> - <% end %> - <div class="pure-u-1 pure-md-1-3"> - <a href="/channel/<%= channel.ucid %>/playlists"><%= translate(locale, "Playlists") %></a> - </div> - <div class="pure-u-1 pure-md-1-3"> - <% if channel.tabs.includes? "community" %> - <b><%= translate(locale, "Community") %></b> - <% end %> - </div> - </div> - <div class="pure-u-2-3"></div> -</div> +<%= rendered "components/channel_info" %> <div class="h-box"> <hr> diff --git a/src/invidious/views/components/channel_info.ecr b/src/invidious/views/components/channel_info.ecr new file mode 100644 index 00000000..f216359f --- /dev/null +++ b/src/invidious/views/components/channel_info.ecr @@ -0,0 +1,60 @@ +<% if channel.banner %> + <div class="h-box"> + <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>"> + </div> + + <div class="h-box"> + <hr> + </div> +<% end %> + +<div class="pure-g h-box"> + <div class="pure-u-2-3"> + <div class="channel-profile"> + <img src="/ggpht<%= channel_profile_pic %>"> + <span><%= author %></span><% if !channel.verified.nil? && channel.verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end %> + </div> + </div> + <div class="pure-u-1-3"> + <h3 style="text-align:right"> + <a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a> + </h3> + </div> +</div> + +<div class="h-box"> + <div id="descriptionWrapper"> + <p><span style="white-space:pre-wrap"><%= channel.description_html %></span></p> + </div> +</div> + +<div class="h-box"> + <% sub_count_text = number_to_short_text(channel.sub_count) %> + <%= rendered "components/subscribe_widget" %> +</div> + +<div class="pure-g h-box"> + <div class="pure-u-1-2"> + <div class="pure-u-1 pure-md-1-3"> + <a href="<%= youtube_url %>"><%= translate(locale, "View channel on YouTube") %></a> + </div> + <div class="pure-u-1 pure-md-1-3"> + <a href="<%= redirect_url %>"><%= translate(locale, "Switch Invidious Instance") %></a> + </div> + + <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, selected_tab) %> + </div> + <div class="pure-u-1-2"> + <div class="pure-g" style="text-align:end"> + <% sort_options.each do |sort| %> + <div class="pure-u-1 pure-md-1-3"> + <% if sort_by == sort %> + <b><%= translate(locale, sort) %></b> + <% else %> + <a href="<%= relative_url %>?sort_by=<%= sort %>"><%= translate(locale, sort) %></a> + <% end %> + </div> + <% end %> + </div> + </div> +</div> diff --git a/src/invidious/views/feeds/subscriptions.ecr b/src/invidious/views/feeds/subscriptions.ecr index add1eefc..d4e93240 100644 --- a/src/invidious/views/feeds/subscriptions.ecr +++ b/src/invidious/views/feeds/subscriptions.ecr @@ -23,6 +23,8 @@ </div> </div> +<% if CONFIG.enable_user_notifications %> + <center> <%= translate_count(locale, "subscriptions_unseen_notifs_count", notifications.size) %> </center> @@ -39,6 +41,8 @@ <% end %> </div> +<% end %> + <div class="h-box"> <hr> </div> diff --git a/src/invidious/views/playlists.ecr b/src/invidious/views/playlists.ecr deleted file mode 100644 index 6ce8b033..00000000 --- a/src/invidious/views/playlists.ecr +++ /dev/null @@ -1,110 +0,0 @@ -<% ucid = channel.ucid %> -<% author = HTML.escape(channel.author) %> - -<% content_for "header" do %> -<title><%= author %> - Invidious</title> -<% end %> - -<% if channel.banner %> - <div class="h-box"> - <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>"> - </div> - - <div class="h-box"> - <hr> - </div> -<% end %> - -<div class="pure-g h-box"> - <div class="pure-u-2-3"> - <div class="channel-profile"> - <img src="/ggpht<%= URI.parse(channel.author_thumbnail).request_target %>"> - <span><%= author %></span><% if !channel.verified.nil? && channel.verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end %> - </div> - </div> - <div class="pure-u-1-3" style="text-align:right"> - <h3 style="text-align:right"> - <a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a> - </h3> - </div> -</div> - -<div class="h-box"> - <div id="descriptionWrapper"> - <p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content if !channel.description_html.empty? %></span></p> - </div> -</div> - -<div class="h-box"> - <% sub_count_text = number_to_short_text(channel.sub_count) %> - <%= rendered "components/subscribe_widget" %> -</div> - -<div class="pure-g h-box"> - <div class="pure-g pure-u-1-3"> - <div class="pure-u-1 pure-md-1-3"> - <a href="https://www.youtube.com/channel/<%= ucid %>/playlists"><%= translate(locale, "View channel on YouTube") %></a> - </div> - - <div class="pure-u-1 pure-md-1-3"> - <% if env.get("preferences").as(Preferences).automatic_instance_redirect%> - <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a> - <% else %> - <a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a> - <% end %> - </div> - - <div class="pure-u-1 pure-md-1-3"> - <a href="/channel/<%= ucid %>"><%= translate(locale, "Videos") %></a> - </div> - <div class="pure-u-1 pure-md-1-3"> - <% if !channel.auto_generated %> - <b><%= translate(locale, "Playlists") %></b> - <% end %> - </div> - <div class="pure-u-1 pure-md-1-3"> - <% if channel.tabs.includes? "community" %> - <a href="/channel/<%= ucid %>/community"><%= translate(locale, "Community") %></a> - <% end %> - </div> - </div> - <div class="pure-u-1-3"></div> - <div class="pure-u-1-3"> - <div class="pure-g" style="text-align:right"> - <% {"last", "oldest", "newest"}.each do |sort| %> - <div class="pure-u-1 pure-md-1-3"> - <% if sort_by == sort %> - <b><%= translate(locale, sort) %></b> - <% else %> - <a href="/channel/<%= ucid %>/playlists?sort_by=<%= sort %>"> - <%= translate(locale, sort) %> - </a> - <% end %> - </div> - <% end %> - </div> - </div> -</div> - -<div class="h-box"> - <hr> -</div> - -<div class="pure-g"> -<% items.each do |item| %> - <%= rendered "components/item" %> -<% end %> -</div> - -<script src="/js/watched_widget.js"></script> - -<div class="pure-g h-box"> - <div class="pure-u-1 pure-u-md-4-5"></div> - <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> - <% if continuation %> - <a href="/channel/<%= ucid %>/playlists?continuation=<%= continuation %><% if sort_by != "last" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>"> - <%= translate(locale, "Next page") %> - </a> - <% end %> - </div> -</div> diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index 98f72eba..77265679 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -54,7 +54,7 @@ <div class="pure-u-1-4"> <a id="notification_ticker" title="<%= translate(locale, "Subscriptions") %>" href="/feed/subscriptions" class="pure-menu-heading"> <% notification_count = env.get("user").as(Invidious::User).notifications.size %> - <% if notification_count > 0 %> + <% if CONFIG.enable_user_notifications && notification_count > 0 %> <span id="notification_count"><%= notification_count %></span> <i class="icon ion-ios-notifications"></i> <% else %> <i class="icon ion-ios-notifications-outline"></i> @@ -170,7 +170,9 @@ }.to_pretty_json %> </script> + <% if CONFIG.enable_user_notifications %> <script src="/js/notifications.js?v=<%= ASSET_COMMIT %>"></script> + <% end %> <% end %> </body> diff --git a/src/invidious/views/user/data_control.ecr b/src/invidious/views/user/data_control.ecr index 74ccc06c..a451159f 100644 --- a/src/invidious/views/user/data_control.ecr +++ b/src/invidious/views/user/data_control.ecr @@ -14,7 +14,7 @@ <div class="pure-control-group"> <label for="import_youtube"> - <a rel="noopener" target="_blank" href="https://github.com/iv-org/documentation/blob/master/Export-YouTube-subscriptions.md"> + <a rel="noopener" target="_blank" href="https://github.com/iv-org/documentation/blob/master/docs/export-youtube-subscriptions.md"> <%= translate(locale, "Import YouTube subscriptions") %> </a> </label> diff --git a/src/invidious/views/user/preferences.ecr b/src/invidious/views/user/preferences.ecr index dbb5e9db..dfda1434 100644 --- a/src/invidious/views/user/preferences.ecr +++ b/src/invidious/views/user/preferences.ecr @@ -89,7 +89,7 @@ <label for="captions[0]"><%= translate(locale, "preferences_captions_label") %></label> <% preferences.captions.each_with_index do |caption, index| %> <select class="pure-u-1-6" name="captions[<%= index %>]" id="captions[<%= index %>]"> - <% CAPTION_LANGUAGES.each do |option| %> + <% Invidious::Videos::Caption::LANGUAGES.each do |option| %> <option value="<%= option %>" <% if preferences.captions[index] == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option> <% end %> </select> @@ -244,6 +244,7 @@ <input name="unseen_only" id="unseen_only" type="checkbox" <% if preferences.unseen_only %>checked<% end %>> </div> + <% if CONFIG.enable_user_notifications %> <div class="pure-control-group"> <label for="notifications_only"><%= translate(locale, "preferences_notifications_only_label") %></label> <input name="notifications_only" id="notifications_only" type="checkbox" <% if preferences.notifications_only %>checked<% end %>> @@ -255,6 +256,7 @@ <a href="#" data-onclick="notification_requestPermission"><%= translate(locale, "Enable web notifications") %></a> </div> <% end %> + <% end %> <% end %> <% if env.get?("user") && CONFIG.admins.includes? env.get?("user").as(Invidious::User).email %> diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 8112930d..65d107b2 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -7,7 +7,7 @@ require "../helpers/serialized_yt_data" private ITEM_CONTAINER_EXTRACTOR = { Extractors::YouTubeTabs, Extractors::SearchResults, - Extractors::Continuation, + Extractors::ContinuationContent, } private ITEM_PARSERS = { @@ -18,8 +18,11 @@ private ITEM_PARSERS = { Parsers::CategoryRendererParser, Parsers::RichItemRendererParser, Parsers::ReelItemRendererParser, + Parsers::ContinuationItemRendererParser, } +private alias InitialData = Hash(String, JSON::Any) + record AuthorFallback, name : String, id : String # Namespace for logic relating to parsing InnerTube data into various datastructs. @@ -170,7 +173,7 @@ private module Parsers # Always simpleText # TODO change default value to nil subscriber_count = item_contents.dig?("subscriberCountText", "simpleText") - .try { |s| short_text_to_number(s.as_s.split(" ")[0]) } || 0 + .try { |s| short_text_to_number(s.as_s.split(" ")[0]).to_i32 } || 0 # Auto-generated channels doesn't have videoCountText # Taken from: https://github.com/iv-org/invidious/pull/2228#discussion_r717620922 @@ -345,14 +348,9 @@ private module Parsers content_container = item_contents["contents"] end - raw_contents = content_container["items"]?.try &.as_a - if !raw_contents.nil? - raw_contents.each do |item| - result = extract_item(item) - if !result.nil? - contents << result - end - end + content_container["items"]?.try &.as_a.each do |item| + result = parse_item(item, author_fallback.name, author_fallback.id) + contents << result if result.is_a?(SearchItem) end Category.new({ @@ -384,7 +382,9 @@ private module Parsers end private def self.parse(item_contents, author_fallback) - return VideoRendererParser.process(item_contents, author_fallback) + child = VideoRendererParser.process(item_contents, author_fallback) + child ||= ReelItemRendererParser.process(item_contents, author_fallback) + return child end def self.parser_name @@ -408,9 +408,19 @@ private module Parsers private def self.parse(item_contents, author_fallback) video_id = item_contents["videoId"].as_s - video_details_container = item_contents.dig( + reel_player_overlay = item_contents.dig( "navigationEndpoint", "reelWatchEndpoint", - "overlay", "reelPlayerOverlayRenderer", + "overlay", "reelPlayerOverlayRenderer" + ) + + # Sometimes, the "reelPlayerOverlayRenderer" object is missing the + # important part of the response. We use this exception to tell + # the calling function to fetch the content again. + if !reel_player_overlay.as_h.has_key?("reelPlayerHeaderSupportedRenderers") + raise RetryOnceException.new + end + + video_details_container = reel_player_overlay.dig( "reelPlayerHeaderSupportedRenderers", "reelPlayerHeaderRenderer" ) @@ -436,9 +446,9 @@ private module Parsers # View count - view_count_text = video_details_container.dig?("viewCountText", "simpleText") - view_count_text ||= video_details_container - .dig?("viewCountText", "accessibility", "accessibilityData", "label") + # View count used to be in the reelWatchEndpoint, but that changed? + view_count_text = item_contents.dig?("viewCountText", "simpleText") + view_count_text ||= video_details_container.dig?("viewCountText", "simpleText") view_count = view_count_text.try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64 @@ -450,8 +460,8 @@ private module Parsers regex_match = /- (?<min>\d+ minutes? )?(?<sec>\d+ seconds?)+ -/.match(a11y_data) - minutes = regex_match.try &.["min"].to_i(strict: false) || 0 - seconds = regex_match.try &.["sec"].to_i(strict: false) || 0 + minutes = regex_match.try &.["min"]?.try &.to_i(strict: false) || 0 + seconds = regex_match.try &.["sec"]?.try &.to_i(strict: false) || 0 duration = (minutes*60 + seconds) @@ -475,6 +485,35 @@ private module Parsers return {{@type.name}} end end + + # Parses an InnerTube continuationItemRenderer into a Continuation. + # Returns nil when the given object isn't a continuationItemRenderer. + # + # continuationItemRenderer contains various metadata ued to load more + # content (i.e when the user scrolls down). The interesting bit is the + # protobuf object known as the "continutation token". Previously, those + # were generated from sratch, but recent (as of 11/2022) Youtube changes + # are forcing us to extract them from replies. + # + module ContinuationItemRendererParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["continuationItemRenderer"]? + return self.parse(item_contents) + end + end + + private def self.parse(item_contents) + token = item_contents + .dig?("continuationEndpoint", "continuationCommand", "token") + .try &.as_s + + return Continuation.new(token) if token + end + + def self.parser_name + return {{@type.name}} + end + end end # The following are the extractors for extracting an array of items from @@ -510,7 +549,7 @@ private module Extractors # }] # module YouTubeTabs - def self.process(initial_data : Hash(String, JSON::Any)) + def self.process(initial_data : InitialData) if target = initial_data["twoColumnBrowseResultsRenderer"]? self.extract(target) end @@ -575,7 +614,7 @@ private module Extractors # } # module SearchResults - def self.process(initial_data : Hash(String, JSON::Any)) + def self.process(initial_data : InitialData) if target = initial_data["twoColumnSearchResultsRenderer"]? self.extract(target) end @@ -608,8 +647,8 @@ private module Extractors # The way they are structured is too varied to be accurately written down here. # However, they all eventually lead to an array of parsable items after traversing # through the JSON structure. - module Continuation - def self.process(initial_data : Hash(String, JSON::Any)) + module ContinuationContent + def self.process(initial_data : InitialData) if target = initial_data["continuationContents"]? self.extract(target) elsif target = initial_data["appendContinuationItemsAction"]? @@ -691,8 +730,7 @@ end # Parses an item from Youtube's JSON response into a more usable structure. # The end result can either be a SearchVideo, SearchPlaylist or SearchChannel. -def extract_item(item : JSON::Any, author_fallback : String? = "", - author_id_fallback : String? = "") +def parse_item(item : JSON::Any, author_fallback : String? = "", author_id_fallback : String? = "") # We "allow" nil values but secretly use empty strings instead. This is to save us the # hassle of modifying every author_fallback and author_id_fallback arg usage # which is more often than not nil. @@ -702,24 +740,23 @@ def extract_item(item : JSON::Any, author_fallback : String? = "", # Each parser automatically validates the data given to see if the data is # applicable to itself. If not nil is returned and the next parser is attempted. ITEM_PARSERS.each do |parser| - LOGGER.trace("extract_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)") + LOGGER.trace("parse_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)") if result = parser.process(item, author_fallback) - LOGGER.debug("extract_item: Successfully parsed via #{parser.parser_name}") - + LOGGER.debug("parse_item: Successfully parsed via #{parser.parser_name}") return result else - LOGGER.trace("extract_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...") + LOGGER.trace("parse_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...") end end end # Parses multiple items from YouTube's initial JSON response into a more usable structure. # The end result is an array of SearchItem. -def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, - author_id_fallback : String? = nil) : Array(SearchItem) - items = [] of SearchItem - +# +# This function yields the container so that items can be parsed separately. +# +def extract_items(initial_data : InitialData, &block) if unpackaged_data = initial_data["contents"]?.try &.as_h elsif unpackaged_data = initial_data["response"]?.try &.as_h elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 0).try &.as_h @@ -727,24 +764,37 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri unpackaged_data = initial_data end - # This is identical to the parser cycling of extract_item(). + # This is identical to the parser cycling of parse_item(). ITEM_CONTAINER_EXTRACTOR.each do |extractor| LOGGER.trace("extract_items: Attempting to extract item container using \"#{extractor.extractor_name}\" (cycling...)") if container = extractor.process(unpackaged_data) LOGGER.debug("extract_items: Successfully unpacked container with \"#{extractor.extractor_name}\"") # Extract items in container - container.each do |item| - if parsed_result = extract_item(item, author_fallback, author_id_fallback) - items << parsed_result - end - end - - break + container.each { |item| yield item } else LOGGER.trace("extract_items: Extractor \"#{extractor.extractor_name}\" does not apply. Cycling to the next one...") end end +end + +# Wrapper using the block function above +def extract_items( + initial_data : InitialData, + author_fallback : String? = nil, + author_id_fallback : String? = nil +) : {Array(SearchItem), String?} + items = [] of SearchItem + continuation = nil + + extract_items(initial_data) do |item| + parsed = parse_item(item, author_fallback, author_id_fallback) + + case parsed + when .is_a?(Continuation) then continuation = parsed.token + when .is_a?(SearchItem) then items << parsed + end + end - return items + return items, continuation end diff --git a/src/invidious/yt_backend/extractors_utils.cr b/src/invidious/yt_backend/extractors_utils.cr index f8245160..0cb3c079 100644 --- a/src/invidious/yt_backend/extractors_utils.cr +++ b/src/invidious/yt_backend/extractors_utils.cr @@ -68,10 +68,10 @@ rescue ex return false end -def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) - extracted = extract_items(initial_data, author_fallback, author_id_fallback) +def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) : Array(SearchVideo) + extracted, _ = extract_items(initial_data, author_fallback, author_id_fallback) - target = [] of SearchItem + target = [] of (SearchItem | Continuation) extracted.each do |i| if i.is_a?(Category) i.contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video } @@ -79,28 +79,11 @@ def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : Str target << i end end - return target.select(SearchVideo).map(&.as(SearchVideo)) + + return target.select(SearchVideo) end def extract_selected_tab(tabs) # Extract the selected tab from the array of tabs Youtube returns return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"]?.try &.as_bool)[0]["tabRenderer"] end - -def fetch_continuation_token(items : Array(JSON::Any)) - # Fetches the continuation token from an array of items - return items.last["continuationItemRenderer"]? - .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s -end - -def fetch_continuation_token(initial_data : Hash(String, JSON::Any)) - # Fetches the continuation token from initial data - if initial_data["onResponseReceivedActions"]? - continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"] - else - tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]) - continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"] - end - - return fetch_continuation_token(continuation_items.as_a) -end diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 02327025..91a9332c 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -43,7 +43,7 @@ module YoutubeAPI ClientType::Web => { name: "WEB", name_proto: "1", - version: "2.20220804.07.00", + version: "2.20221118.01.00", api_key: DEFAULT_API_KEY, screen: "WATCH_FULL_SCREEN", os_name: "Windows", |
