Add support for timedtext captions (#3533)

2 files changed, 106 insertions, 30 deletions

diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr
index 79f7bd3f..f312211e 100644
--- a/src/invidious/routes/api/v1/videos.cr
+++ b/src/invidious/routes/api/v1/videos.cr
@@ -93,45 +93,50 @@ module Invidious::Routes::API::V1::Videos
     # as well as some other markup that makes it cumbersome, so we try to fix that here
     if caption.name.includes? "auto-generated"
       caption_xml = YT_POOL.client &.get(url).body
-      caption_xml = XML.parse(caption_xml)
 
-      webvtt = String.build do |str|
-        str << <<-END_VTT
-        WEBVTT
-        Kind: captions
-        Language: #{tlang || caption.language_code}
+      if caption_xml.starts_with?("<?xml")
+        webvtt = caption.timedtext_to_vtt(caption_xml, tlang)
+      else
+        caption_xml = XML.parse(caption_xml)
 
+        webvtt = String.build do |str|
+          str << <<-END_VTT
+          WEBVTT
+          Kind: captions
+          Language: #{tlang || caption.language_code}
 
-        END_VTT
 
-        caption_nodes = caption_xml.xpath_nodes("//transcript/text")
-        caption_nodes.each_with_index do |node, i|
-          start_time = node["start"].to_f.seconds
-          duration = node["dur"]?.try &.to_f.seconds
-          duration ||= start_time
+          END_VTT
 
-          if caption_nodes.size > i + 1
-            end_time = caption_nodes[i + 1]["start"].to_f.seconds
-          else
-            end_time = start_time + duration
-          end
+          caption_nodes = caption_xml.xpath_nodes("//transcript/text")
+          caption_nodes.each_with_index do |node, i|
+            start_time = node["start"].to_f.seconds
+            duration = node["dur"]?.try &.to_f.seconds
+            duration ||= start_time
 
-          start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}"
-          end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}"
+            if caption_nodes.size > i + 1
+              end_time = caption_nodes[i + 1]["start"].to_f.seconds
+            else
+              end_time = start_time + duration
+            end
 
-          text = HTML.unescape(node.content)
-          text = text.gsub(/<font color="#[a-fA-F0-9]{6}">/, "")
-          text = text.gsub(/<\/font>/, "")
-          if md = text.match(/(?<name>.*) : (?<text>.*)/)
-            text = "<v #{md["name"]}>#{md["text"]}</v>"
-          end
+            start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}"
+            end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}"
 
-          str << <<-END_CUE
-          #{start_time} --> #{end_time}
-          #{text}
+            text = HTML.unescape(node.content)
+            text = text.gsub(/<font color="#[a-fA-F0-9]{6}">/, "")
+            text = text.gsub(/<\/font>/, "")
+            if md = text.match(/(?<name>.*) : (?<text>.*)/)
+              text = "<v #{md["name"]}>#{md["text"]}</v>"
+            end
 
+            str << <<-END_CUE
+            #{start_time} --> #{end_time}
+            #{text}
 
-          END_CUE
+
+            END_CUE
+          end
         end
       end
     else
@@ -141,7 +146,12 @@ module Invidious::Routes::API::V1::Videos
       #
       # See: https://github.com/iv-org/invidious/issues/2391
       webvtt = YT_POOL.client &.get("#{url}&format=vtt").body
-        .gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1")
+      if webvtt.starts_with?("<?xml")
+        webvtt = caption.timedtext_to_vtt(webvtt)
+      else
+        webvtt = YT_POOL.client &.get("#{url}&format=vtt").body
+          .gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1")
+      end
     end
 
     if title = env.params.query["title"]?
diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr
index 4642c1a7..13f81a31 100644
--- a/src/invidious/videos/caption.cr
+++ b/src/invidious/videos/caption.cr
@@ -31,6 +31,72 @@ module Invidious::Videos
       return captions_list
     end
 
+    def timedtext_to_vtt(timedtext : String, tlang = nil) : String
+      # In the future, we could just directly work with the url. This is more of a POC
+      cues = [] of XML::Node
+      tree = XML.parse(timedtext)
+      tree = tree.children.first
+
+      tree.children.each do |item|
+        if item.name == "body"
+          item.children.each do |cue|
+            if cue.name == "p" && !(cue.children.size == 1 && cue.children[0].content == "\n")
+              cues << cue
+            end
+          end
+          break
+        end
+      end
+      result = String.build do |result|
+        result << <<-END_VTT
+        WEBVTT
+        Kind: captions
+        Language: #{tlang || @language_code}
+
+
+        END_VTT
+
+        result << "\n\n"
+
+        cues.each_with_index do |node, i|
+          start_time = node["t"].to_f.milliseconds
+
+          duration = node["d"]?.try &.to_f.milliseconds
+
+          duration ||= start_time
+
+          if cues.size > i + 1
+            end_time = cues[i + 1]["t"].to_f.milliseconds
+          else
+            end_time = start_time + duration
+          end
+
+          # start_time
+          result << start_time.hours.to_s.rjust(2, '0')
+          result << ':' << start_time.minutes.to_s.rjust(2, '0')
+          result << ':' << start_time.seconds.to_s.rjust(2, '0')
+          result << '.' << start_time.milliseconds.to_s.rjust(3, '0')
+
+          result << " --> "
+
+          # end_time
+          result << end_time.hours.to_s.rjust(2, '0')
+          result << ':' << end_time.minutes.to_s.rjust(2, '0')
+          result << ':' << end_time.seconds.to_s.rjust(2, '0')
+          result << '.' << end_time.milliseconds.to_s.rjust(3, '0')
+
+          result << "\n"
+
+          node.children.each do |s|
+            result << s.content
+          end
+          result << "\n"
+          result << "\n"
+        end
+      end
+      return result
+    end
+
     # List of all caption languages available on Youtube.
     LANGUAGES = {
       "",