Implement workaround for broken shorts objects

3 files changed, 46 insertions, 15 deletions

diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr
index bea406c1..befec03d 100644
--- a/src/invidious/channels/videos.cr
+++ b/src/invidious/channels/videos.cr
@@ -127,16 +127,38 @@ module Invidious::Channel::Tabs
   #  Shorts
   # -------------------
 
-  def get_shorts(channel : AboutChannel, continuation : String? = nil)
+  private def fetch_shorts_data(ucid : String, continuation : String? = nil)
     if continuation.nil?
       # EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts"
       # TODO: try to extract the continuation tokens that allows other sorting options
-      initial_data = YoutubeAPI.browse(channel.ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D")
+      return YoutubeAPI.browse(ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D")
     else
-      initial_data = YoutubeAPI.browse(continuation: continuation)
+      return YoutubeAPI.browse(continuation: continuation)
     end
+  end
 
-    return extract_items(initial_data, channel.author, channel.ucid)
+  def get_shorts(channel : AboutChannel, continuation : String? = nil)
+    initial_data = self.fetch_shorts_data(channel.ucid, continuation)
+
+    begin
+      # Try to parse the initial data fetched above
+      return extract_items(initial_data, channel.author, channel.ucid)
+    rescue ex : RetryOnceException
+      # Sometimes, for a completely unknown reason, the "reelItemRenderer"
+      # object is missing some critical information (it happens once in about
+      # 20 subsequent requests). Refreshing the page is required to properly
+      # show the "shorts" tab.
+      #
+      # In order to make the experience smoother for the user, we simulate
+      # said page refresh by fetching again the JSON. If that still doesn't
+      # work, we raise a BrokenTubeException, as something is really broken.
+      begin
+        initial_data = self.fetch_shorts_data(channel.ucid, continuation)
+        return extract_items(initial_data, channel.author, channel.ucid)
+      rescue ex : RetryOnceException
+        raise BrokenTubeException.new "reelPlayerHeaderSupportedRenderers"
+      end
+    end
   end
 
   # -------------------
diff --git a/src/invidious/exceptions.cr b/src/invidious/exceptions.cr
index 425c08da..690db907 100644
--- a/src/invidious/exceptions.cr
+++ b/src/invidious/exceptions.cr
@@ -33,3 +33,8 @@ end
 
 class VideoNotAvailableException < Exception
 end
+
+# Exception used to indicate that the JSON response from YT is missing
+# some important informations, and that the query should be sent again.
+class RetryOnceException < Exception
+end
diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr
index bca0dcbd..65d107b2 100644
--- a/src/invidious/yt_backend/extractors.cr
+++ b/src/invidious/yt_backend/extractors.cr
@@ -408,19 +408,23 @@ private module Parsers
     private def self.parse(item_contents, author_fallback)
       video_id = item_contents["videoId"].as_s
 
-      begin
-        video_details_container = item_contents.dig(
-          "navigationEndpoint", "reelWatchEndpoint",
-          "overlay", "reelPlayerOverlayRenderer",
-          "reelPlayerHeaderSupportedRenderers",
-          "reelPlayerHeaderRenderer"
-        )
-      rescue ex : KeyError
-        # Extract key name from original message
-        key = /"([^"]+)"/.match(ex.message || "").try &.[1]?
-        raise BrokenTubeException.new(key || "reelPlayerOverlayRenderer")
+      reel_player_overlay = item_contents.dig(
+        "navigationEndpoint", "reelWatchEndpoint",
+        "overlay", "reelPlayerOverlayRenderer"
+      )
+
+      # Sometimes, the "reelPlayerOverlayRenderer" object is missing the
+      # important part of the response. We use this exception to tell
+      # the calling function to fetch the content again.
+      if !reel_player_overlay.as_h.has_key?("reelPlayerHeaderSupportedRenderers")
+        raise RetryOnceException.new
       end
 
+      video_details_container = reel_player_overlay.dig(
+        "reelPlayerHeaderSupportedRenderers",
+        "reelPlayerHeaderRenderer"
+      )
+
       # Author infos
 
       author = video_details_container