Merge pull request #3137 from SamantazFox/add-hashtags

Add hashtags

9 files changed, 207 insertions, 38 deletions

diff --git a/src/invidious.cr b/src/invidious.cr
index dd240852..4952b365 100644
--- a/src/invidious.cr
+++ b/src/invidious.cr
@@ -385,6 +385,7 @@ end
   Invidious::Routing.get "/opensearch.xml", Invidious::Routes::Search, :opensearch
   Invidious::Routing.get "/results", Invidious::Routes::Search, :results
   Invidious::Routing.get "/search", Invidious::Routes::Search, :search
+  Invidious::Routing.get "/hashtag/:hashtag", Invidious::Routes::Search, :hashtag
 
   # User routes
   define_user_routes()
diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr
index da71e9a8..565f2bca 100644
--- a/src/invidious/channels/about.cr
+++ b/src/invidious/channels/about.cr
@@ -61,6 +61,7 @@ def get_about_info(ucid, locale) : AboutChannel
     author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s
     author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s
     author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s
+    author_verified = has_verified_badge?(initdata.dig?("header", "c4TabbedHeaderRenderer", "badges"))
 
     ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s
 
@@ -71,9 +72,6 @@ def get_about_info(ucid, locale) : AboutChannel
     # if banner.includes? "channels/c4/default_banner"
     #  banner = nil
     # end
-    # author_verified_badges = initdata["header"]?.try &.["c4TabbedHeaderRenderer"]?.try &.["badges"]?
-    author_verified_badge = initdata["header"].dig?("c4TabbedHeaderRenderer", "badges", 0, "metadataBadgeRenderer", "tooltip")
-    author_verified = (author_verified_badge && author_verified_badge == "Verified")
 
     description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]?
 
diff --git a/src/invidious/hashtag.cr b/src/invidious/hashtag.cr
new file mode 100644
index 00000000..afe31a36
--- /dev/null
+++ b/src/invidious/hashtag.cr
@@ -0,0 +1,44 @@
+module Invidious::Hashtag
+  extend self
+
+  def fetch(hashtag : String, page : Int, region : String? = nil) : Array(SearchItem)
+    cursor = (page - 1) * 60
+    ctoken = generate_continuation(hashtag, cursor)
+
+    client_config = YoutubeAPI::ClientConfig.new(region: region)
+    response = YoutubeAPI.browse(continuation: ctoken, client_config: client_config)
+
+    return extract_items(response)
+  end
+
+  def generate_continuation(hashtag : String, cursor : Int)
+    object = {
+      "80226972:embedded" => {
+        "2:string" => "FEhashtag",
+        "3:base64" => {
+          "1:varint" => cursor.to_i64,
+        },
+        "7:base64" => {
+          "325477796:embedded" => {
+            "1:embedded" => {
+              "2:0:embedded" => {
+                "2:string"  => '#' + hashtag,
+                "4:varint"  => 0_i64,
+                "11:string" => "",
+              },
+              "4:string" => "browse-feedFEhashtag",
+            },
+            "2:string" => hashtag,
+          },
+        },
+      },
+    }
+
+    continuation = object.try { |i| Protodec::Any.cast_json(i) }
+      .try { |i| Protodec::Any.from_json(i) }
+      .try { |i| Base64.urlsafe_encode(i) }
+      .try { |i| URI.encode_www_form(i) }
+
+    return continuation
+  end
+end
diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr
index b5b58399..2e6043f7 100644
--- a/src/invidious/routes/feeds.cr
+++ b/src/invidious/routes/feeds.cr
@@ -182,7 +182,7 @@ module Invidious::Routes::Feeds
         paid:               false,
         premium:            false,
         premiere_timestamp: nil,
-        author_verified:    false, # 	¯\_(ツ)_/¯
+        author_verified:    false,
       })
     end
 
diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr
index e60d0081..6f8bffea 100644
--- a/src/invidious/routes/search.cr
+++ b/src/invidious/routes/search.cr
@@ -63,4 +63,35 @@ module Invidious::Routes::Search
       templated "search"
     end
   end
+
+  def self.hashtag(env : HTTP::Server::Context)
+    locale = env.get("preferences").as(Preferences).locale
+
+    hashtag = env.params.url["hashtag"]?
+    if hashtag.nil? || hashtag.empty?
+      return error_template(400, "Invalid request")
+    end
+
+    page = env.params.query["page"]?
+    if page.nil?
+      page = 1
+    else
+      page = Math.max(1, page.to_i)
+      env.params.query.delete_all("page")
+    end
+
+    begin
+      videos = Invidious::Hashtag.fetch(hashtag, page)
+    rescue ex
+      return error_template(500, ex)
+    end
+
+    params = env.params.query.empty? ? "" : "&#{env.params.query}"
+
+    hashtag_encoded = URI.encode_www_form(hashtag, space_to_plus: false)
+    url_prev_page = "/hashtag/#{hashtag_encoded}?page=#{page - 1}#{params}"
+    url_next_page = "/hashtag/#{hashtag_encoded}?page=#{page + 1}#{params}"
+
+    templated "hashtag"
+  end
 end
diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr
index f65b05bb..1504e390 100644
--- a/src/invidious/videos.cr
+++ b/src/invidious/videos.cr
@@ -853,6 +853,7 @@ end
 # the same 11 first entries as the compact rendered.
 #
 # TODO: "compactRadioRenderer" (Mix) and
+# TODO: Use a proper struct/class instead of a hacky JSON object
 def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
   return nil if !related["videoId"]?
 
@@ -868,11 +869,7 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
     .try &.dig?("runs", 0)
 
   author = channel_info.try &.dig?("text")
-  author_verified_badge = related["ownerBadges"]?.try do |badges_array|
-    badges_array.as_a.find(&.dig("metadataBadgeRenderer", "tooltip").as_s.== "Verified")
-  end
-
-  author_verified = (author_verified_badge && author_verified_badge.size > 0).to_s
+  author_verified = has_verified_badge?(related["ownerBadges"]?).to_s
 
   ucid = channel_info.try { |ci| HelperExtractors.get_browse_id(ci) }
 
@@ -1089,17 +1086,19 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_
 
   # Author infos
 
-  author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer")
-  author_thumbnail = author_info.try &.dig?("thumbnail", "thumbnails", 0, "url")
+  if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer")
+    author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url")
+    params["authorThumbnail"] = JSON::Any.new(author_thumbnail.try &.as_s || "")
 
-  author_verified_badge = author_info.try &.dig?("badges", 0, "metadataBadgeRenderer", "tooltip")
-  author_verified = (!author_verified_badge.nil? && author_verified_badge == "Verified")
-  params["authorVerified"] = JSON::Any.new(author_verified)
+    author_verified = has_verified_badge?(author_info["badges"]?)
+    params["authorVerified"] = JSON::Any.new(author_verified)
 
-  params["authorThumbnail"] = JSON::Any.new(author_thumbnail.try &.as_s || "")
+    subs_text = author_info["subscriberCountText"]?
+      .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") }
+      .try &.as_s.split(" ", 2)[0]
 
-  params["subCountText"] = JSON::Any.new(author_info.try &.["subscriberCountText"]?
-    .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") }.try &.as_s.split(" ", 2)[0] || "-")
+    params["subCountText"] = JSON::Any.new(subs_text || "-")
+  end
 
   # Return data
 
diff --git a/src/invidious/views/hashtag.ecr b/src/invidious/views/hashtag.ecr
new file mode 100644
index 00000000..0ecfe832
--- /dev/null
+++ b/src/invidious/views/hashtag.ecr
@@ -0,0 +1,39 @@
+<% content_for "header" do %>
+<title><%= HTML.escape(hashtag) %> - Invidious</title>
+<% end %>
+
+<hr/>
+
+<div class="pure-g h-box v-box">
+    <div class="pure-u-1 pure-u-lg-1-5">
+        <%- if page > 1 -%>
+            <a href="<%= url_prev_page %>"><%= translate(locale, "Previous page") %></a>
+        <%- end -%>
+    </div>
+    <div class="pure-u-1 pure-u-lg-3-5"></div>
+    <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
+        <%- if videos.size >= 60 -%>
+            <a href="<%= url_next_page %>"><%= translate(locale, "Next page") %></a>
+        <%- end -%>
+    </div>
+</div>
+
+<div class="pure-g">
+    <%- videos.each do |item| -%>
+        <%= rendered "components/item" %>
+    <%- end -%>
+</div>
+
+<div class="pure-g h-box">
+    <div class="pure-u-1 pure-u-lg-1-5">
+        <%- if page > 1 -%>
+            <a href="<%= url_prev_page %>"><%= translate(locale, "Previous page") %></a>
+        <%- end -%>
+    </div>
+    <div class="pure-u-1 pure-u-lg-3-5"></div>
+    <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
+        <%- if videos.size >= 60 -%>
+            <a href="<%= url_next_page %>"><%= translate(locale, "Next page") %></a>
+        <%- end -%>
+    </div>
+</div>
diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr
index a2ec7d59..c4326cab 100644
--- a/src/invidious/yt_backend/extractors.cr
+++ b/src/invidious/yt_backend/extractors.cr
@@ -1,3 +1,5 @@
+require "../helpers/serialized_yt_data"
+
 # This file contains helper methods to parse the Youtube API json data into
 # neat little packages we can use
 
@@ -14,6 +16,7 @@ private ITEM_PARSERS = {
   Parsers::GridPlaylistRendererParser,
   Parsers::PlaylistRendererParser,
   Parsers::CategoryRendererParser,
+  Parsers::RichItemRendererParser,
 }
 
 record AuthorFallback, name : String, id : String
@@ -57,6 +60,8 @@ private module Parsers
         author_id = author_fallback.id
       end
 
+      author_verified = has_verified_badge?(item_contents["ownerBadges"]?)
+
       # For live videos (and possibly recently premiered videos) there is no published information.
       # Instead, in its place is the amount of people currently watching. This behavior should be replicated
       # on Invidious once all features of livestreams are supported. On an unrelated note, defaulting to the current
@@ -102,11 +107,7 @@ private module Parsers
       premium = false
 
       premiere_timestamp = item_contents.dig?("upcomingEventData", "startTime").try { |t| Time.unix(t.as_s.to_i64) }
-      author_verified_badge = item_contents["ownerBadges"]?.try do |badges_array|
-        badges_array.as_a.find(&.dig("metadataBadgeRenderer", "tooltip").as_s.== "Verified")
-      end
 
-      author_verified = (author_verified_badge && author_verified_badge.size > 0)
       item_contents["badges"]?.try &.as_a.each do |badge|
         b = badge["metadataBadgeRenderer"]
         case b["label"].as_s
@@ -133,7 +134,7 @@ private module Parsers
         live_now:           live_now,
         premium:            premium,
         premiere_timestamp: premiere_timestamp,
-        author_verified:    author_verified || false,
+        author_verified:    author_verified,
       })
     end
 
@@ -161,12 +162,9 @@ private module Parsers
     private def self.parse(item_contents, author_fallback)
       author = extract_text(item_contents["title"]) || author_fallback.name
       author_id = item_contents["channelId"]?.try &.as_s || author_fallback.id
-      author_verified_badge = item_contents["ownerBadges"]?.try do |badges_array|
-        badges_array.as_a.find(&.dig("metadataBadgeRenderer", "tooltip").as_s.== "Verified")
-      end
-
-      author_verified = (author_verified_badge && author_verified_badge.size > 0)
+      author_verified = has_verified_badge?(item_contents["ownerBadges"]?)
       author_thumbnail = HelperExtractors.get_thumbnails(item_contents)
+
       # When public subscriber count is disabled, the subscriberCountText isn't sent by InnerTube.
       # Always simpleText
       # TODO change default value to nil
@@ -188,7 +186,7 @@ private module Parsers
         video_count:      video_count,
         description_html: description_html,
         auto_generated:   auto_generated,
-        author_verified:  author_verified || false,
+        author_verified:  author_verified,
       })
     end
 
@@ -216,11 +214,9 @@ private module Parsers
     private def self.parse(item_contents, author_fallback)
       title = extract_text(item_contents["title"]) || ""
       plid = item_contents["playlistId"]?.try &.as_s || ""
-      author_verified_badge = item_contents["ownerBadges"]?.try do |badges_array|
-        badges_array.as_a.find(&.dig("metadataBadgeRenderer", "tooltip").as_s.== "Verified")
-      end
 
-      author_verified = (author_verified_badge && author_verified_badge.size > 0)
+      author_verified = has_verified_badge?(item_contents["ownerBadges"]?)
+
       video_count = HelperExtractors.get_video_count(item_contents)
       playlist_thumbnail = HelperExtractors.get_thumbnails(item_contents)
 
@@ -232,7 +228,7 @@ private module Parsers
         video_count:     video_count,
         videos:          [] of SearchPlaylistVideo,
         thumbnail:       playlist_thumbnail,
-        author_verified: author_verified || false,
+        author_verified: author_verified,
       })
     end
 
@@ -266,11 +262,8 @@ private module Parsers
       author_info = item_contents.dig?("shortBylineText", "runs", 0)
       author = author_info.try &.["text"].as_s || author_fallback.name
       author_id = author_info.try { |x| HelperExtractors.get_browse_id(x) } || author_fallback.id
-      author_verified_badge = item_contents["ownerBadges"]?.try do |badges_array|
-        badges_array.as_a.find(&.dig("metadataBadgeRenderer", "tooltip").as_s.== "Verified")
-      end
+      author_verified = has_verified_badge?(item_contents["ownerBadges"]?)
 
-      author_verified = (author_verified_badge && author_verified_badge.size > 0)
       videos = item_contents["videos"]?.try &.as_a.map do |v|
         v = v["childVideoRenderer"]
         v_title = v.dig?("title", "simpleText").try &.as_s || ""
@@ -293,7 +286,7 @@ private module Parsers
         video_count:     video_count,
         videos:          videos,
         thumbnail:       playlist_thumbnail,
-        author_verified: author_verified || false,
+        author_verified: author_verified,
       })
     end
 
@@ -374,6 +367,29 @@ private module Parsers
       return {{@type.name}}
     end
   end
+
+  # Parses an InnerTube richItemRenderer into a SearchVideo.
+  # Returns nil when the given object isn't a shelfRenderer
+  #
+  # A richItemRenderer seems to be a simple wrapper for a videoRenderer, used
+  # by the result page for hashtags. It is located inside a continuationItems
+  # container.
+  #
+  module RichItemRendererParser
+    def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+      if item_contents = item.dig?("richItemRenderer", "content")
+        return self.parse(item_contents, author_fallback)
+      end
+    end
+
+    private def self.parse(item_contents, author_fallback)
+      return VideoRendererParser.process(item_contents, author_fallback)
+    end
+
+    def self.parser_name
+      return {{@type.name}}
+    end
+  end
 end
 
 # The following are the extractors for extracting an array of items from
@@ -501,6 +517,8 @@ private module Extractors
         self.extract(target)
       elsif target = initial_data["appendContinuationItemsAction"]?
         self.extract(target)
+      elsif target = initial_data["reloadContinuationItemsCommand"]?
+        self.extract(target)
       end
     end
 
diff --git a/src/invidious/yt_backend/extractors_utils.cr b/src/invidious/yt_backend/extractors_utils.cr
index add5f488..3d5e5787 100644
--- a/src/invidious/yt_backend/extractors_utils.cr
+++ b/src/invidious/yt_backend/extractors_utils.cr
@@ -29,6 +29,45 @@ def extract_text(item : JSON::Any?) : String?
   end
 end
 
+# Check if an "ownerBadges" or a "badges" element contains a verified badge.
+# There is currently two known types of verified badges:
+#
+# "ownerBadges": [{
+#   "metadataBadgeRenderer": {
+#     "icon": { "iconType": "CHECK_CIRCLE_THICK" },
+#     "style": "BADGE_STYLE_TYPE_VERIFIED",
+#     "tooltip": "Verified",
+#     "accessibilityData": { "label": "Verified" }
+#    }
+# }],
+#
+# "ownerBadges": [{
+#   "metadataBadgeRenderer": {
+#     "icon": { "iconType": "OFFICIAL_ARTIST_BADGE" },
+#     "style": "BADGE_STYLE_TYPE_VERIFIED_ARTIST",
+#     "tooltip": "Official Artist Channel",
+#     "accessibilityData": { "label": "Official Artist Channel" }
+#   }
+# }],
+#
+def has_verified_badge?(badges : JSON::Any?)
+  return false if badges.nil?
+
+  badges.as_a.each do |badge|
+    style = badge.dig("metadataBadgeRenderer", "style").as_s
+
+    return true if style == "BADGE_STYLE_TYPE_VERIFIED"
+    return true if style == "BADGE_STYLE_TYPE_VERIFIED_ARTIST"
+  end
+
+  return false
+rescue ex
+  LOGGER.debug("Unable to parse owner badges. Got exception: #{ex.message}")
+  LOGGER.trace("Owner badges data: #{badges.to_json}")
+
+  return false
+end
+
 def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
   extracted = extract_items(initial_data, author_fallback, author_id_fallback)