Videos: Use android test suite client (#4650)

"Temporary" fix, until a better and more permanent solution is found.

Closes issue 4584

2 files changed, 26 insertions, 15 deletions

diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr
index e3816d79..0e1a947c 100644
--- a/src/invidious/videos/parser.cr
+++ b/src/invidious/videos/parser.cr
@@ -107,11 +107,7 @@ def extract_video_info(video_id : String)
     # decrypted URLs and maybe fix throttling issues (#2194). See the
     # following issue for an explanation about decrypted URLs:
     # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
-    client_config.client_type = YoutubeAPI::ClientType::Android
-    new_player_response = try_fetch_streaming_data(video_id, client_config)
-  elsif !reason.includes?("your country") # Handled separately
-    # The Android embedded client could help here
-    client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed
+    client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite
     new_player_response = try_fetch_streaming_data(video_id, client_config)
   end
 
@@ -123,8 +119,9 @@ def extract_video_info(video_id : String)
 
   # Replace player response and reset reason
   if !new_player_response.nil?
-    # Preserve storyboard data before replacement
+    # Preserve captions & storyboard data before replacement
     new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]?
+    new_player_response["captions"] = player_response["captions"] if player_response["captions"]?
 
     player_response = new_player_response
     params.delete("reason")
@@ -142,9 +139,7 @@ end
 
 def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)?
   LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.")
-  # CgIIAdgDAQ%3D%3D is a workaround for streaming URLs that returns a 403.
-  # https://github.com/LuanRT/YouTube.js/pull/624
-  response = YoutubeAPI.player(video_id: id, params: "CgIIAdgDAQ%3D%3D", client_config: client_config)
+  response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config)
 
   playability_status = response["playabilityStatus"]["status"]
   LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.")
@@ -152,7 +147,7 @@ def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConf
   if id != response.dig("videoDetails", "videoId")
     # YouTube may return a different video player response than expected.
     # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
-    raise VideoNotAvailableException.new(
+    raise InfoException.new(
       "The video returned by YouTube isn't the requested one. (#{client_config.client_type} client)"
     )
   elsif playability_status == "OK"
diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr
index 8235898f..727ce9a3 100644
--- a/src/invidious/yt_backend/youtube_api.cr
+++ b/src/invidious/yt_backend/youtube_api.cr
@@ -6,18 +6,22 @@ module YoutubeAPI
   extend self
 
   private DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
+  private ANDROID_API_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w"
 
   # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history
-  private ANDROID_APP_VERSION = "19.09.36"
-  private ANDROID_USER_AGENT  = "com.google.android.youtube/19.09.36 (Linux; U; Android 12; US) gzip"
+  private ANDROID_APP_VERSION = "19.14.42"
+  private ANDROID_USER_AGENT  = "com.google.android.youtube/19.14.42 (Linux; U; Android 12; US) gzip"
   private ANDROID_SDK_VERSION = 31_i64
   private ANDROID_VERSION     = "12"
 
+  private ANDROID_TS_APP_VERSION = "1.9"
+  private ANDROID_TS_USER_AGENT  = "com.google.android.youtube/1.9 (Linux; U; Android 12; US) gzip"
+
   # For Apple device names, see https://gist.github.com/adamawolf/3048717
   # For iOS versions, see https://en.wikipedia.org/wiki/IOS_version_history#Releases,
   # then go to the dedicated article of the major version you want.
-  private IOS_APP_VERSION = "19.09.3"
-  private IOS_USER_AGENT  = "com.google.ios.youtube/19.09.3 (iPhone14,5; U; CPU iOS 17_4 like Mac OS X;)"
+  private IOS_APP_VERSION = "19.16.3"
+  private IOS_USER_AGENT  = "com.google.ios.youtube/19.16.3 (iPhone14,5; U; CPU iOS 17_4 like Mac OS X;)"
   private IOS_VERSION     = "17.4.0.21E219" # Major.Minor.Patch.Build
 
   private WINDOWS_VERSION = "10.0"
@@ -32,6 +36,7 @@ module YoutubeAPI
     Android
     AndroidEmbeddedPlayer
     AndroidScreenEmbed
+    AndroidTestSuite
 
     IOS
     IOSEmbedded
@@ -89,7 +94,7 @@ module YoutubeAPI
       name:                "ANDROID",
       name_proto:          "3",
       version:             ANDROID_APP_VERSION,
-      api_key:             "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w",
+      api_key:             ANDROID_API_KEY,
       android_sdk_version: ANDROID_SDK_VERSION,
       user_agent:          ANDROID_USER_AGENT,
       os_name:             "Android",
@@ -114,6 +119,17 @@ module YoutubeAPI
       os_version:          ANDROID_VERSION,
       platform:            "MOBILE",
     },
+    ClientType::AndroidTestSuite => {
+      name:                "ANDROID_TESTSUITE",
+      name_proto:          "30",
+      version:             ANDROID_TS_APP_VERSION,
+      api_key:             ANDROID_API_KEY,
+      android_sdk_version: ANDROID_SDK_VERSION,
+      user_agent:          ANDROID_TS_USER_AGENT,
+      os_name:             "Android",
+      os_version:          ANDROID_VERSION,
+      platform:            "MOBILE",
+    },
 
     # IOS