Merge pull request #116 from omarroth/add-playlists

Add playlist page and endpoint

7 files changed, 311 insertions, 59 deletions

diff --git a/src/invidious.cr b/src/invidious.cr
index 1f33c2db..f91e4c72 100644
--- a/src/invidious.cr
+++ b/src/invidious.cr
@@ -368,6 +368,31 @@ get "/embed/:id" do |env|
   rendered "embed"
 end
 
+# Playlists
+get "/playlist" do |env|
+  plid = env.params.query["list"]?
+  if !plid
+    next env.redirect "/"
+  end
+
+  page = env.params.query["page"]?.try &.to_i?
+  page ||= 1
+
+  if plid
+    begin
+      videos = extract_playlist(plid, page)
+    rescue ex
+      error_message = ex.message
+      next templated "error"
+    end
+    playlist = fetch_playlist(plid)
+  else
+    next env.redirect "/"
+  end
+
+  templated "playlist"
+end
+
 # Search
 
 get "/results" do |env|
@@ -1534,31 +1559,13 @@ get "/channel/:ucid" do |env|
   rss = XML.parse_html(rss.body)
   author = rss.xpath_node("//feed/author/name").not_nil!.content
 
-  url = produce_playlist_url(ucid, (page - 1) * 100)
-  response = client.get(url)
-  response = JSON.parse(response.body)
-
-  if !response["content_html"]?
-    error_message = "This channel does not exist."
+  begin
+    videos = extract_playlist(ucid, page)
+  rescue ex
+    error_message = ex.message
     next templated "error"
   end
 
-  document = XML.parse_html(response["content_html"].as_s)
-  anchor = document.xpath_node(%q(//div[@class="pl-video-owner"]/a))
-  if !anchor
-    videos = [] of ChannelVideo
-    next templated "channel"
-  end
-
-  videos = [] of ChannelVideo
-  document.xpath_nodes(%q(//a[contains(@class,"pl-video-title-link")])).each do |node|
-    href = URI.parse(node["href"])
-    id = HTTP::Params.parse(href.query.not_nil!)["v"]
-    title = node.content
-
-    videos << ChannelVideo.new(id, title, Time.now, Time.now, "", "")
-  end
-
   templated "channel"
 end
 
@@ -2363,6 +2370,65 @@ get "/api/v1/search" do |env|
   response
 end
 
+get "/api/v1/playlists/:plid" do |env|
+  plid = env.params.url["plid"]
+
+  page = env.params.query["page"]?.try &.to_i?
+  page ||= 1
+
+  begin
+    videos = extract_playlist(plid, page)
+  rescue ex
+    env.response.content_type = "application/json"
+    response = {"error" => "Playlist is empty"}.to_json
+    halt env, status_code: 404, response: response
+  end
+
+  playlist = fetch_playlist(plid)
+
+  response = JSON.build do |json|
+    json.object do
+      json.field "title", playlist.title
+      json.field "id", playlist.id
+
+      json.field "author", playlist.author
+      json.field "authorId", playlist.ucid
+      json.field "authorUrl", "/channel/#{playlist.ucid}"
+
+      json.field "description", playlist.description
+      json.field "videoCount", playlist.video_count
+
+      json.field "viewCount", playlist.views
+      json.field "updated", playlist.updated.epoch
+
+      json.field "videos" do
+        json.array do
+          videos.each do |video|
+            json.object do
+              json.field "title", video.title
+              json.field "id", video.id
+
+              json.field "author", video.author
+              json.field "authorId", video.ucid
+              json.field "authorUrl", "/channel/#{video.ucid}"
+
+              json.field "videoThumbnails" do
+                generate_thumbnails(json, video.id)
+              end
+
+              json.field "index", video.index
+              json.field "lengthSeconds", video.length_seconds
+            end
+          end
+        end
+      end
+    end
+  end
+
+  env.response.content_type = "application/json"
+  response
+end
+
 get "/api/manifest/dash/id/videoplayback" do |env|
   env.response.headers["Access-Control-Allow-Origin"] = "*"
   env.redirect "/videoplayback?#{env.params.query}"
diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr
index b7898d5e..d9f86f69 100644
--- a/src/invidious/comments.cr
+++ b/src/invidious/comments.cr
@@ -262,7 +262,7 @@ def fill_links(html, scheme, host)
   end
 
   if host == "www.youtube.com"
-    html = html.xpath_node(%q(//p[@id="eow-description"])).not_nil!.to_xml
+    html = html.xpath_node(%q(//body)).not_nil!.to_xml
   else
     html = html.to_xml(options: XML::SaveOptions::NO_DECL)
   end
diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr
index 7104e1e9..5f29ee8d 100644
--- a/src/invidious/helpers/helpers.cr
+++ b/src/invidious/helpers/helpers.cr
@@ -116,40 +116,6 @@ def login_req(login_form, f_req)
   return HTTP::Params.encode(data)
 end
 
-def produce_playlist_url(id, index)
-  if id.starts_with? "UC"
-    id = "UU" + id.lchop("UC")
-  end
-  ucid = "VL" + id
-
-  continuation = [0x08_u8] + write_var_int(index)
-  slice = continuation.to_unsafe.to_slice(continuation.size)
-  slice = Base64.urlsafe_encode(slice, false)
-
-  # Inner Base64
-  continuation = "PT:" + slice
-  continuation = [0x7a_u8, continuation.bytes.size.to_u8] + continuation.bytes
-  slice = continuation.to_unsafe.to_slice(continuation.size)
-  slice = Base64.urlsafe_encode(slice)
-  slice = URI.escape(slice)
-
-  # Outer Base64
-  continuation = [0x1a.to_u8, slice.bytes.size.to_u8] + slice.bytes
-  continuation = ucid.bytes + continuation
-  continuation = [0x12_u8, ucid.size.to_u8] + continuation
-  continuation = [0xe2_u8, 0xa9_u8, 0x85_u8, 0xb2_u8, 2_u8, continuation.size.to_u8] + continuation
-
-  # Wrap bytes
-  slice = continuation.to_unsafe.to_slice(continuation.size)
-  slice = Base64.urlsafe_encode(slice)
-  slice = URI.escape(slice)
-  continuation = slice
-
-  url = "/browse_ajax?action_continuation=1&continuation=#{continuation}"
-
-  return url
-end
-
 def produce_videos_url(ucid, page = 1)
   page = "#{page}"
 
diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr
index 4690a405..fe3e4e24 100644
--- a/src/invidious/helpers/utils.cr
+++ b/src/invidious/helpers/utils.cr
@@ -64,10 +64,23 @@ end
 
 def decode_date(string : String)
   # String matches 'YYYY'
-  if string.match(/\d{4}/)
+  if string.match(/^\d{4}/)
     return Time.new(string.to_i, 1, 1)
   end
 
+  # Try to parse as format Jul 10, 2000
+  begin
+    return Time.parse(string, "%b %-d, %Y", Time::Location.local)
+  rescue ex
+  end
+
+  case string
+  when "today"
+    return Time.now
+  when "yesterday"
+    return Time.now - 1.day
+  end
+
   # String matches format "20 hours ago", "4 months ago"...
   date = string.split(" ")[-3, 3]
   delta = date[0].to_i
diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr
new file mode 100644
index 00000000..9bd5724d
--- /dev/null
+++ b/src/invidious/playlists.cr
@@ -0,0 +1,160 @@
+class Playlist
+  add_mapping({
+    title:       String,
+    id:          String,
+    author:      String,
+    ucid:        String,
+    description: String,
+    video_count: Int32,
+    views:       Int64,
+    updated:     Time,
+  })
+end
+
+class PlaylistVideo
+  add_mapping({
+    title:          String,
+    id:             String,
+    author:         String,
+    ucid:           String,
+    length_seconds: Int32,
+    published:      Time,
+    playlists:      Array(String),
+    index:          Int32,
+  })
+end
+
+def extract_playlist(plid, page)
+  index = (page - 1) * 100
+  url = produce_playlist_url(plid, index)
+
+  client = make_client(YT_URL)
+  response = client.get(url)
+  response = JSON.parse(response.body)
+  if !response["content_html"]? || response["content_html"].as_s.empty?
+    raise "Playlist does not exist"
+  end
+
+  videos = [] of PlaylistVideo
+
+  document = XML.parse_html(response["content_html"].as_s)
+  anchor = document.xpath_node(%q(//div[@class="pl-video-owner"]/a))
+  if anchor
+    document.xpath_nodes(%q(.//tr[contains(@class, "pl-video")])).each_with_index do |video, offset|
+      anchor = video.xpath_node(%q(.//td[@class="pl-video-title"]))
+      if !anchor
+        next
+      end
+
+      title = anchor.xpath_node(%q(.//a)).not_nil!.content.strip(" \n")
+      id = anchor.xpath_node(%q(.//a)).not_nil!["href"].lchop("/watch?v=")[0, 11]
+
+      anchor = anchor.xpath_node(%q(.//div[@class="pl-video-owner"]/a))
+      if anchor
+        author = anchor.content
+        ucid = anchor["href"].split("/")[2]
+      else
+        author = ""
+        ucid = ""
+      end
+
+      anchor = video.xpath_node(%q(.//td[@class="pl-video-time"]/div/div[1]))
+      if anchor && !anchor.content.empty?
+        length_seconds = decode_length_seconds(anchor.content)
+      else
+        length_seconds = 0
+      end
+
+      videos << PlaylistVideo.new(
+        title,
+        id,
+        author,
+        ucid,
+        length_seconds,
+        Time.now,
+        [plid],
+        index + offset,
+      )
+    end
+  end
+
+  return videos
+end
+
+def produce_playlist_url(id, index)
+  if id.starts_with? "UC"
+    id = "UU" + id.lchop("UC")
+  end
+  ucid = "VL" + id
+
+  continuation = [0x08_u8] + write_var_int(index)
+  slice = continuation.to_unsafe.to_slice(continuation.size)
+  slice = Base64.urlsafe_encode(slice, false)
+
+  # Inner Base64
+  continuation = "PT:" + slice
+  continuation = [0x7a_u8, continuation.bytes.size.to_u8] + continuation.bytes
+  slice = continuation.to_unsafe.to_slice(continuation.size)
+  slice = Base64.urlsafe_encode(slice)
+  slice = URI.escape(slice)
+
+  # Outer Base64
+  continuation = [0x1a.to_u8, slice.bytes.size.to_u8] + slice.bytes
+  continuation = ucid.bytes + continuation
+  continuation = [0x12_u8, ucid.size.to_u8] + continuation
+  continuation = [0xe2_u8, 0xa9_u8, 0x85_u8, 0xb2_u8, 2_u8, continuation.size.to_u8] + continuation
+
+  # Wrap bytes
+  slice = continuation.to_unsafe.to_slice(continuation.size)
+  slice = Base64.urlsafe_encode(slice)
+  slice = URI.escape(slice)
+  continuation = slice
+
+  url = "/browse_ajax?action_continuation=1&continuation=#{continuation}"
+
+  return url
+end
+
+def fetch_playlist(plid)
+  client = make_client(YT_URL)
+  response = client.get("/playlist?list=#{plid}&disable_polymer=1")
+  document = XML.parse_html(response.body)
+
+  title = document.xpath_node(%q(//h1[@class="pl-header-title"])).not_nil!.content
+  title = title.strip(" \n")
+
+  description = document.xpath_node(%q(//span[@class="pl-header-description-text"]/div/div[1]))
+  description ||= document.xpath_node(%q(//span[@class="pl-header-description-text"]))
+
+  if description
+    description = description.to_xml.strip(" \n")
+    description = description.split("<button ")[0]
+    description = fill_links(description, "https", "www.youtube.com")
+    description = add_alt_links(description)
+  else
+    description = ""
+  end
+
+  anchor = document.xpath_node(%q(//ul[@class="pl-header-details"])).not_nil!
+  author = anchor.xpath_node(%q(.//li[1]/a)).not_nil!.content
+  ucid = anchor.xpath_node(%q(.//li[1]/a)).not_nil!["href"].split("/")[2]
+
+  video_count = anchor.xpath_node(%q(.//li[2])).not_nil!.content.delete("videos").to_i
+  views = anchor.xpath_node(%q(.//li[3])).not_nil!.content.delete("views,").to_i64
+
+  updated = anchor.xpath_node(%q(.//li[4])).not_nil!.content.lchop("Last updated on ").lchop("Updated ")
+  updated = decode_date(updated)
+
+  playlist = Playlist.new(
+    title,
+    plid,
+    author,
+    ucid,
+    description,
+    video_count,
+    views,
+    updated
+  )
+
+  return playlist
+end
diff --git a/src/invidious/views/components/video.ecr b/src/invidious/views/components/video.ecr
index e92e4eb8..779016da 100644
--- a/src/invidious/views/components/video.ecr
+++ b/src/invidious/views/components/video.ecr
@@ -1,6 +1,11 @@
 <div class="pure-u-1 pure-u-md-1-4">
     <div class="h-box">
-        <a style="width:100%;" href="/watch?v=<%= video.id %>">
+        <% if video.responds_to?(:playlists) %>
+        <% params = "&list=#{video.playlists[0]}" %>
+        <% else %>
+        <% params = nil %>
+        <% end %>
+        <a style="width:100%;" href="/watch?v=<%= video.id %><%= params %>">
             <% if env.get?("user") && env.get("user").as(User).preferences.thin_mode %>
             <% else %>
             <img style="width:100%;" src="https://i.ytimg.com/vi/<%= video.id %>/mqdefault.jpg"/>
diff --git a/src/invidious/views/playlist.ecr b/src/invidious/views/playlist.ecr
new file mode 100644
index 00000000..e5fbf395
--- /dev/null
+++ b/src/invidious/views/playlist.ecr
@@ -0,0 +1,42 @@
+<% content_for "header" do %>
+<title><%= playlist.title %> - Invidious</title>
+<% end %>
+
+<div class="pure-g h-box">
+    <div class="pure-u-2-3">
+        <h3><%= playlist.title %></h3>
+    </div>
+</div>
+<div class="pure-g h-box">
+    <div class="pure-u-1 pure-u-md-1-4">
+        <a href="/channel/<%= playlist.ucid %>">
+            <b><%= playlist.author %></b>
+        </a>
+    </div>
+</div>
+
+<div class="h-box">
+    <p><%= playlist.description %></p>
+</div>
+
+<% videos.each_slice(4) do |slice| %>
+<div class="pure-g">
+    <% slice.each do |video| %>
+        <%= rendered "components/video" %>
+    <% end %>
+</div> 
+<% end %>
+
+<div class="pure-g h-box">
+    <div class="pure-u-1 pure-u-md-1-5">
+    <% if page >= 2 %>
+        <a href="/playlist?list=<%= playlist.id %>&page=<%= page - 1 %>">Next page</a>
+    <% end %>
+    </div>
+    <div class="pure-u-1 pure-u-md-3-5"></div>
+    <div style="text-align:right;" class="pure-u-1 pure-u-md-1-5">
+    <% if videos.size == 100 %>
+        <a href="/playlist?list=<%= playlist.id %>&page=<%= page + 1 %>">Next page</a>
+    <% end %>
+    </div>
+</div>
\ No newline at end of file