summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/invidious/channels/channels.cr36
-rw-r--r--src/invidious/comments.cr73
-rw-r--r--src/invidious/database/users.cr2
-rw-r--r--src/invidious/helpers/utils.cr53
-rw-r--r--src/invidious/routes/watch.cr6
-rw-r--r--src/invidious/trending.cr19
-rw-r--r--src/invidious/user/imports.cr2
-rw-r--r--src/invidious/videos/description.cr53
-rw-r--r--src/invidious/videos/parser.cr2
-rw-r--r--src/invidious/yt_backend/extractors_utils.cr13
10 files changed, 123 insertions, 136 deletions
diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr
index 63dd2194..c3d6124f 100644
--- a/src/invidious/channels/channels.cr
+++ b/src/invidious/channels/channels.cr
@@ -159,12 +159,18 @@ def fetch_channel(ucid, pull_all_videos : Bool)
LOGGER.debug("fetch_channel: #{ucid}")
LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}")
+ namespaces = {
+ "yt" => "http://www.youtube.com/xml/schemas/2015",
+ "media" => "http://search.yahoo.com/mrss/",
+ "default" => "http://www.w3.org/2005/Atom",
+ }
+
LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed")
rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body
LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed")
- rss = XML.parse_html(rss)
+ rss = XML.parse(rss)
- author = rss.xpath_node(%q(//feed/title))
+ author = rss.xpath_node("//default:feed/default:title", namespaces)
if !author
raise InfoException.new("Deleted or invalid channel")
end
@@ -192,15 +198,23 @@ def fetch_channel(ucid, pull_all_videos : Bool)
videos, continuation = IV::Channel::Tabs.get_videos(channel)
LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
- rss.xpath_nodes("//feed/entry").each do |entry|
- video_id = entry.xpath_node("videoid").not_nil!.content
- title = entry.xpath_node("title").not_nil!.content
- published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content)
- updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content)
- author = entry.xpath_node("author/name").not_nil!.content
- ucid = entry.xpath_node("channelid").not_nil!.content
- views = entry.xpath_node("group/community/statistics").try &.["views"]?.try &.to_i64?
- views ||= 0_i64
+ rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry|
+ video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
+ title = entry.xpath_node("default:title", namespaces).not_nil!.content
+
+ published = Time.parse_rfc3339(
+ entry.xpath_node("default:published", namespaces).not_nil!.content
+ )
+ updated = Time.parse_rfc3339(
+ entry.xpath_node("default:updated", namespaces).not_nil!.content
+ )
+
+ author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
+ ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
+
+ views = entry
+ .xpath_node("media:group/media:community/media:statistics", namespaces)
+ .try &.["views"]?.try &.to_i64? || 0_i64
channel_video = videos
.select(SearchVideo)
diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr
index fd2be73d..01556099 100644
--- a/src/invidious/comments.cr
+++ b/src/invidious/comments.cr
@@ -372,32 +372,25 @@ def template_youtube_comments(comments, locale, thin_mode, is_replies = false)
</div>
END_HTML
when "video"
- html << <<-END_HTML
- <div class="pure-g">
- <div class="pure-u-1 pure-u-md-1-2">
- <div style="position:relative;width:100%;height:0;padding-bottom:56.25%;margin-bottom:5px">
- END_HTML
-
if attachment["error"]?
html << <<-END_HTML
+ <div class="pure-g video-iframe-wrapper">
<p>#{attachment["error"]}</p>
+ </div>
END_HTML
else
html << <<-END_HTML
- <iframe id='ivplayer' style='position:absolute;width:100%;height:100%;left:0;top:0' src='/embed/#{attachment["videoId"]?}?autoplay=0' style='border:none;'></iframe>
+ <div class="pure-g video-iframe-wrapper">
+ <iframe class="video-iframe" src='/embed/#{attachment["videoId"]?}?autoplay=0'></iframe>
+ </div>
END_HTML
end
-
- html << <<-END_HTML
- </div>
- </div>
- </div>
- END_HTML
else nil # Ignore
end
end
html << <<-END_HTML
+ <p>
<span title="#{Time.unix(child["published"].as_i64).to_s(translate(locale, "%A %B %-d, %Y"))}">#{translate(locale, "`x` ago", recode_date(Time.unix(child["published"].as_i64), locale))} #{child["isEdited"] == true ? translate(locale, "(edited)") : ""}</span>
|
END_HTML
@@ -416,6 +409,7 @@ def template_youtube_comments(comments, locale, thin_mode, is_replies = false)
html << <<-END_HTML
<i class="icon ion-ios-thumbs-up"></i> #{number_with_separator(child["likeCount"])}
+ </p>
END_HTML
if child["creatorHeart"]?
@@ -604,7 +598,7 @@ def text_to_parsed_content(text : String) : JSON::Any
currentNode = {"text" => urlMatch[0], "navigationEndpoint" => {"urlEndpoint" => {"url" => urlMatch[0]}}}
currentNodes << (JSON.parse(currentNode.to_json))
# If text remain after match create new simple node with text after match
- afterNode = {"text" => splittedLastNode.size > 0 ? splittedLastNode[1] : ""}
+ afterNode = {"text" => splittedLastNode.size > 1 ? splittedLastNode[1] : ""}
currentNodes << (JSON.parse(afterNode.to_json))
end
@@ -635,55 +629,8 @@ def content_to_comment_html(content, video_id : String? = "")
text = HTML.escape(run["text"].as_s)
- if run["navigationEndpoint"]?
- if url = run["navigationEndpoint"]["urlEndpoint"]?.try &.["url"].as_s
- url = URI.parse(url)
- displayed_url = text
-
- if url.host == "youtu.be"
- url = "/watch?v=#{url.request_target.lstrip('/')}"
- elsif url.host.nil? || url.host.not_nil!.ends_with?("youtube.com")
- if url.path == "/redirect"
- # Sometimes, links can be corrupted (why?) so make sure to fallback
- # nicely. See https://github.com/iv-org/invidious/issues/2682
- url = url.query_params["q"]? || ""
- displayed_url = url
- else
- url = url.request_target
- displayed_url = "youtube.com#{url}"
- end
- end
-
- text = %(<a href="#{url}">#{reduce_uri(displayed_url)}</a>)
- elsif watch_endpoint = run["navigationEndpoint"]["watchEndpoint"]?
- start_time = watch_endpoint["startTimeSeconds"]?.try &.as_i
- link_video_id = watch_endpoint["videoId"].as_s
-
- url = "/watch?v=#{link_video_id}"
- url += "&t=#{start_time}" if !start_time.nil?
-
- # If the current video ID (passed through from the caller function)
- # is the same as the video ID in the link, add HTML attributes for
- # the JS handler function that bypasses page reload.
- #
- # See: https://github.com/iv-org/invidious/issues/3063
- if link_video_id == video_id
- start_time ||= 0
- text = %(<a href="#{url}" data-onclick="jump_to_time" data-jump-time="#{start_time}">#{reduce_uri(text)}</a>)
- else
- text = %(<a href="#{url}">#{text}</a>)
- end
- elsif url = run.dig?("navigationEndpoint", "commandMetadata", "webCommandMetadata", "url").try &.as_s
- if text.starts_with?(/\s?[@#]/)
- # Handle "pings" in comments and hasthags differently
- # See:
- # - https://github.com/iv-org/invidious/issues/3038
- # - https://github.com/iv-org/invidious/issues/3062
- text = %(<a href="#{url}">#{text}</a>)
- else
- text = %(<a href="#{url}">#{reduce_uri(url)}</a>)
- end
- end
+ if navigationEndpoint = run.dig?("navigationEndpoint")
+ text = parse_link_endpoint(navigationEndpoint, text, video_id)
end
text = "<b>#{text}</b>" if run["bold"]?
diff --git a/src/invidious/database/users.cr b/src/invidious/database/users.cr
index 0a4a4fd8..d54e6a76 100644
--- a/src/invidious/database/users.cr
+++ b/src/invidious/database/users.cr
@@ -52,7 +52,7 @@ module Invidious::Database::Users
def mark_watched(user : User, vid : String)
request = <<-SQL
UPDATE users
- SET watched = array_append(watched, $1)
+ SET watched = array_append(array_remove(watched, $1), $1)
WHERE email = $2
SQL
diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr
index 500a2582..bcf7c963 100644
--- a/src/invidious/helpers/utils.cr
+++ b/src/invidious/helpers/utils.cr
@@ -389,3 +389,56 @@ def reduce_uri(uri : URI | String, max_length : Int32 = 50, suffix : String = "â
end
return str
end
+
+# Get the html link from a NavigationEndpoint or an innertubeCommand
+def parse_link_endpoint(endpoint : JSON::Any, text : String, video_id : String)
+ if url = endpoint.dig?("urlEndpoint", "url").try &.as_s
+ url = URI.parse(url)
+ displayed_url = text
+
+ if url.host == "youtu.be"
+ url = "/watch?v=#{url.request_target.lstrip('/')}"
+ elsif url.host.nil? || url.host.not_nil!.ends_with?("youtube.com")
+ if url.path == "/redirect"
+ # Sometimes, links can be corrupted (why?) so make sure to fallback
+ # nicely. See https://github.com/iv-org/invidious/issues/2682
+ url = url.query_params["q"]? || ""
+ displayed_url = url
+ else
+ url = url.request_target
+ displayed_url = "youtube.com#{url}"
+ end
+ end
+
+ text = %(<a href="#{url}">#{reduce_uri(displayed_url)}</a>)
+ elsif watch_endpoint = endpoint.dig?("watchEndpoint")
+ start_time = watch_endpoint["startTimeSeconds"]?.try &.as_i
+ link_video_id = watch_endpoint["videoId"].as_s
+
+ url = "/watch?v=#{link_video_id}"
+ url += "&t=#{start_time}" if !start_time.nil?
+
+ # If the current video ID (passed through from the caller function)
+ # is the same as the video ID in the link, add HTML attributes for
+ # the JS handler function that bypasses page reload.
+ #
+ # See: https://github.com/iv-org/invidious/issues/3063
+ if link_video_id == video_id
+ start_time ||= 0
+ text = %(<a href="#{url}" data-onclick="jump_to_time" data-jump-time="#{start_time}">#{reduce_uri(text)}</a>)
+ else
+ text = %(<a href="#{url}">#{text}</a>)
+ end
+ elsif url = endpoint.dig?("commandMetadata", "webCommandMetadata", "url").try &.as_s
+ if text.starts_with?(/\s?[@#]/)
+ # Handle "pings" in comments and hasthags differently
+ # See:
+ # - https://github.com/iv-org/invidious/issues/3038
+ # - https://github.com/iv-org/invidious/issues/3062
+ text = %(<a href="#{url}">#{text}</a>)
+ else
+ text = %(<a href="#{url}">#{reduce_uri(url)}</a>)
+ end
+ end
+ return text
+end
diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr
index 5d3845c3..813cb0f4 100644
--- a/src/invidious/routes/watch.cr
+++ b/src/invidious/routes/watch.cr
@@ -76,7 +76,7 @@ module Invidious::Routes::Watch
end
env.params.query.delete_all("iv_load_policy")
- if watched && preferences.watch_history && !watched.includes? id
+ if watched && preferences.watch_history
Invidious::Database::Users.mark_watched(user.as(User), id)
end
@@ -259,9 +259,7 @@ module Invidious::Routes::Watch
case action
when "action_mark_watched"
- if !user.watched.includes? id
- Invidious::Database::Users.mark_watched(user, id)
- end
+ Invidious::Database::Users.mark_watched(user, id)
when "action_mark_unwatched"
Invidious::Database::Users.mark_unwatched(user, id)
else
diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr
index 74bab1bd..2d9f8a83 100644
--- a/src/invidious/trending.cr
+++ b/src/invidious/trending.cr
@@ -20,6 +20,21 @@ def fetch_trending(trending_type, region, locale)
items, _ = extract_items(initial_data)
- # Return items, but ignore categories (e.g featured content)
- return items.reject!(Category), plid
+ extracted = [] of SearchItem
+
+ items.each do |itm|
+ if itm.is_a?(Category)
+ # Ignore the smaller categories, as they generally contain a sponsored
+ # channel, which brings a lot of noise on the trending page.
+ # See: https://github.com/iv-org/invidious/issues/2989
+ next if itm.contents.size < 24
+
+ extracted.concat extract_category(itm)
+ else
+ extracted << itm
+ end
+ end
+
+ # Deduplicate items before returning results
+ return extracted.select(SearchVideo).uniq!(&.id), plid
end
diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr
index 20ae0d47..aa947456 100644
--- a/src/invidious/user/imports.cr
+++ b/src/invidious/user/imports.cr
@@ -48,7 +48,7 @@ struct Invidious::User
if data["watch_history"]?
user.watched += data["watch_history"].as_a.map(&.as_s)
- user.watched.uniq!
+ user.watched.reverse!.uniq!.reverse!
Invidious::Database::Users.update_watch_history(user)
end
diff --git a/src/invidious/videos/description.cr b/src/invidious/videos/description.cr
index 2017955d..542cb416 100644
--- a/src/invidious/videos/description.cr
+++ b/src/invidious/videos/description.cr
@@ -1,51 +1,6 @@
require "json"
require "uri"
-def parse_command(command : JSON::Any?, string : String) : String?
- on_tap = command.dig?("onTap", "innertubeCommand")
-
- # 3rd party URL, extract original URL from YouTube tracking URL
- if url_endpoint = on_tap.try &.["urlEndpoint"]?
- youtube_url = URI.parse url_endpoint["url"].as_s
-
- original_url = youtube_url.query_params["q"]?
- if original_url.nil?
- return ""
- else
- return "<a href=\"#{original_url}\">#{original_url}</a>"
- end
- # 1st party watch URL
- elsif watch_endpoint = on_tap.try &.["watchEndpoint"]?
- video_id = watch_endpoint["videoId"].as_s
- time = watch_endpoint["startTimeSeconds"].as_i
-
- url = "/watch?v=#{video_id}&t=#{time}s"
-
- # if string is a timestamp, use the string instead
- # this is a lazy regex for validating timestamps
- if /(?:\d{1,2}:){1,2}\d{2}/ =~ string
- return "<a href=\"#{url}\">#{string}</a>"
- else
- return "<a href=\"#{url}\">#{url}</a>"
- end
- # hashtag/other browse URLs
- elsif browse_endpoint = on_tap.try &.dig?("commandMetadata", "webCommandMetadata")
- url = browse_endpoint["url"].try &.as_s
-
- # remove unnecessary character in a channel name
- if browse_endpoint["webPageType"]?.try &.as_s == "WEB_PAGE_TYPE_CHANNEL"
- name = string.match(/@[\w\d.-]+/)
- if name.try &.[0]?
- return "<a href=\"#{url}\">#{name.try &.[0]}</a>"
- end
- end
-
- return "<a href=\"#{url}\">#{string}</a>"
- end
-
- return "(unknown YouTube desc command)"
-end
-
private def copy_string(str : String::Builder, iter : Iterator, count : Int) : Int
copied = 0
while copied < count
@@ -62,7 +17,7 @@ private def copy_string(str : String::Builder, iter : Iterator, count : Int) : I
return copied
end
-def parse_description(desc : JSON::Any?) : String?
+def parse_description(desc, video_id : String) : String?
return "" if desc.nil?
content = desc["content"].as_s
@@ -94,7 +49,11 @@ def parse_description(desc : JSON::Any?) : String?
copy_string(str2, iter, cmd_length)
end
- str << parse_command(command, cmd_content)
+ link = cmd_content
+ if on_tap = command.dig?("onTap", "innertubeCommand")
+ link = parse_link_endpoint(on_tap, cmd_content, video_id)
+ end
+ str << link
index += cmd_length
end
diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr
index 1c6d118d..2e8eecc3 100644
--- a/src/invidious/videos/parser.cr
+++ b/src/invidious/videos/parser.cr
@@ -287,7 +287,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
# description_html = video_secondary_renderer.try &.dig?("description", "runs")
# .try &.as_a.try { |t| content_to_comment_html(t, video_id) }
- description_html = parse_description(video_secondary_renderer.try &.dig?("attributedDescription"))
+ description_html = parse_description(video_secondary_renderer.try &.dig?("attributedDescription"), video_id)
# Video metadata
diff --git a/src/invidious/yt_backend/extractors_utils.cr b/src/invidious/yt_backend/extractors_utils.cr
index b247dca8..11d95958 100644
--- a/src/invidious/yt_backend/extractors_utils.cr
+++ b/src/invidious/yt_backend/extractors_utils.cr
@@ -68,16 +68,17 @@ rescue ex
return false
end
-# This function extracts the SearchItems from a Category.
+# This function extracts SearchVideo items from a Category.
# Categories are commonly returned in search results and trending pages.
def extract_category(category : Category) : Array(SearchVideo)
- items = [] of SearchItem
+ return category.contents.select(SearchVideo)
+end
- category.contents.each do |item|
- target << cate_i if item.is_a?(SearchItem)
+# :ditto:
+def extract_category(category : Category, &)
+ category.contents.select(SearchVideo).each do |item|
+ yield item
end
-
- return items
end
def extract_selected_tab(tabs)
generated by cgit v1.2.3 (git 2.39.1) at 2025-12-17 16:44:19 +0000