From 6aa65593ef0dbadc0ef2735cd1d1bca0788370f1 Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 12 Aug 2021 23:31:12 -0700 Subject: Extract API routes from invidious.cr (2/?) - Video playback endpoints - Search feed api - Video info api --- src/invidious.cr | 575 +------------------------- src/invidious/channels/channels.cr | 1 + src/invidious/helpers/macros.cr | 9 + src/invidious/routes/api/manifest.cr | 237 +++++++++++ src/invidious/routes/api/v1/channels.cr | 243 +++++++++++ src/invidious/routes/api/v1/feeds.cr | 46 +++ src/invidious/routes/api/v1/misc.cr | 13 + src/invidious/routes/api/v1/routes.cr | 37 ++ src/invidious/routes/api/v1/search.cr | 101 +++++ src/invidious/routes/api/v1/videos.cr | 372 +++++++++++++++++ src/invidious/routes/api_v1/channels.cr | 244 ----------- src/invidious/routes/api_v1/feeds.cr | 46 --- src/invidious/routes/api_v1/misc.cr | 13 - src/invidious/routes/api_v1/routes.cr | 30 -- src/invidious/routes/api_v1/search.cr | 24 -- src/invidious/routes/api_v1/video_playback.cr | 2 - src/invidious/routes/api_v1/widgets.cr | 386 ----------------- src/invidious/routes/video_playback.cr | 290 +++++++++++++ 18 files changed, 1351 insertions(+), 1318 deletions(-) create mode 100644 src/invidious/routes/api/manifest.cr create mode 100644 src/invidious/routes/api/v1/channels.cr create mode 100644 src/invidious/routes/api/v1/feeds.cr create mode 100644 src/invidious/routes/api/v1/misc.cr create mode 100644 src/invidious/routes/api/v1/routes.cr create mode 100644 src/invidious/routes/api/v1/search.cr create mode 100644 src/invidious/routes/api/v1/videos.cr delete mode 100644 src/invidious/routes/api_v1/channels.cr delete mode 100644 src/invidious/routes/api_v1/feeds.cr delete mode 100644 src/invidious/routes/api_v1/misc.cr delete mode 100644 src/invidious/routes/api_v1/routes.cr delete mode 100644 src/invidious/routes/api_v1/search.cr delete mode 100644 src/invidious/routes/api_v1/video_playback.cr delete mode 100644 src/invidious/routes/api_v1/widgets.cr create mode 100644 src/invidious/routes/video_playback.cr (limited to 'src') diff --git a/src/invidious.cr b/src/invidious.cr index 6ac099f3..85852b9a 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -364,6 +364,8 @@ Invidious::Routing.post "/preferences", Invidious::Routes::PreferencesRoute, :up Invidious::Routing.get "/toggle_theme", Invidious::Routes::PreferencesRoute, :toggle_theme define_v1_api_routes() +define_api_manifest_routes() +define_video_playback_routes() # Users @@ -1639,69 +1641,6 @@ end # API Endpoints -get "/api/v1/videos/:id" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - id = env.params.url["id"] - region = env.params.query["region"]? - - begin - video = get_video(id, PG_DB, region: region) - rescue ex : VideoRedirect - env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) - next error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) - rescue ex - next error_json(500, ex) - end - - video.to_json(locale) -end - -get "/api/v1/search" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - region = env.params.query["region"]? - - env.response.content_type = "application/json" - - query = env.params.query["q"]? - query ||= "" - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - sort_by = env.params.query["sort_by"]?.try &.downcase - sort_by ||= "relevance" - - date = env.params.query["date"]?.try &.downcase - date ||= "" - - duration = env.params.query["duration"]?.try &.downcase - duration ||= "" - - features = env.params.query["features"]?.try &.split(",").map { |feature| feature.downcase } - features ||= [] of String - - content_type = env.params.query["type"]?.try &.downcase - content_type ||= "video" - - begin - search_params = produce_search_params(page, sort_by, date, content_type, duration, features) - rescue ex - next error_json(400, ex) - end - - count, search_results = search(query, search_params, region).as(Tuple) - JSON.build do |json| - json.array do - search_results.each do |item| - item.to_json(locale, json) - end - end - end -end - {"/api/v1/playlists/:plid", "/api/v1/auth/playlists/:plid"}.each do |route| get route do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? @@ -2245,516 +2184,6 @@ post "/api/v1/auth/tokens/unregister" do |env| env.response.status_code = 204 end -get "/api/manifest/dash/id/videoplayback" do |env| - env.response.headers.delete("Content-Type") - env.response.headers["Access-Control-Allow-Origin"] = "*" - env.redirect "/videoplayback?#{env.params.query}" -end - -get "/api/manifest/dash/id/videoplayback/*" do |env| - env.response.headers.delete("Content-Type") - env.response.headers["Access-Control-Allow-Origin"] = "*" - env.redirect env.request.path.lchop("/api/manifest/dash/id") -end - -get "/api/manifest/dash/id/:id" do |env| - env.response.headers.add("Access-Control-Allow-Origin", "*") - env.response.content_type = "application/dash+xml" - - local = env.params.query["local"]?.try &.== "true" - id = env.params.url["id"] - region = env.params.query["region"]? - - # Since some implementations create playlists based on resolution regardless of different codecs, - # we can opt to only add a source to a representation if it has a unique height within that representation - unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe } - - begin - video = get_video(id, PG_DB, region: region) - rescue ex : VideoRedirect - next env.redirect env.request.resource.gsub(id, ex.video_id) - rescue ex - env.response.status_code = 403 - next - end - - if dashmpd = video.dash_manifest_url - manifest = YT_POOL.client &.get(URI.parse(dashmpd).request_target).body - - manifest = manifest.gsub(/[^<]+<\/BaseURL>/) do |baseurl| - url = baseurl.lchop("") - url = url.rchop("") - - if local - uri = URI.parse(url) - url = "#{uri.request_target}host/#{uri.host}/" - end - - "#{url}" - end - - next manifest - end - - adaptive_fmts = video.adaptive_fmts - - if local - adaptive_fmts.each do |fmt| - fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) - end - end - - audio_streams = video.audio_streams - video_streams = video.video_streams.sort_by { |stream| {stream["width"].as_i, stream["fps"].as_i} }.reverse - - XML.build(indent: " ", encoding: "UTF-8") do |xml| - xml.element("MPD", "xmlns": "urn:mpeg:dash:schema:mpd:2011", - "profiles": "urn:mpeg:dash:profile:full:2011", minBufferTime: "PT1.5S", type: "static", - mediaPresentationDuration: "PT#{video.length_seconds}S") do - xml.element("Period") do - i = 0 - - {"audio/mp4", "audio/webm"}.each do |mime_type| - mime_streams = audio_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type } - next if mime_streams.empty? - - xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true) do - mime_streams.each do |fmt| - codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"') - bandwidth = fmt["bitrate"].as_i - itag = fmt["itag"].as_i - url = fmt["url"].as_s - - xml.element("Representation", id: fmt["itag"], codecs: codecs, bandwidth: bandwidth) do - xml.element("AudioChannelConfiguration", schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011", - value: "2") - xml.element("BaseURL") { xml.text url } - xml.element("SegmentBase", indexRange: "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}") do - xml.element("Initialization", range: "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}") - end - end - end - end - - i += 1 - end - - potential_heights = {4320, 2160, 1440, 1080, 720, 480, 360, 240, 144} - - {"video/mp4", "video/webm"}.each do |mime_type| - mime_streams = video_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type } - next if mime_streams.empty? - - heights = [] of Int32 - xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, scanType: "progressive") do - mime_streams.each do |fmt| - codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"') - bandwidth = fmt["bitrate"].as_i - itag = fmt["itag"].as_i - url = fmt["url"].as_s - width = fmt["width"].as_i - height = fmt["height"].as_i - - # Resolutions reported by YouTube player (may not accurately reflect source) - height = potential_heights.min_by { |i| (height - i).abs } - next if unique_res && heights.includes? height - heights << height - - xml.element("Representation", id: itag, codecs: codecs, width: width, height: height, - startWithSAP: "1", maxPlayoutRate: "1", - bandwidth: bandwidth, frameRate: fmt["fps"]) do - xml.element("BaseURL") { xml.text url } - xml.element("SegmentBase", indexRange: "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}") do - xml.element("Initialization", range: "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}") - end - end - end - end - - i += 1 - end - end - end - end -end - -get "/api/manifest/hls_variant/*" do |env| - response = YT_POOL.client &.get(env.request.path) - - if response.status_code != 200 - env.response.status_code = response.status_code - next - end - - local = env.params.query["local"]?.try &.== "true" - - env.response.content_type = "application/x-mpegURL" - env.response.headers.add("Access-Control-Allow-Origin", "*") - - manifest = response.body - - if local - manifest = manifest.gsub("https://www.youtube.com", HOST_URL) - manifest = manifest.gsub("index.m3u8", "index.m3u8?local=true") - end - - manifest -end - -get "/api/manifest/hls_playlist/*" do |env| - response = YT_POOL.client &.get(env.request.path) - - if response.status_code != 200 - env.response.status_code = response.status_code - next - end - - local = env.params.query["local"]?.try &.== "true" - - env.response.content_type = "application/x-mpegURL" - env.response.headers.add("Access-Control-Allow-Origin", "*") - - manifest = response.body - - if local - manifest = manifest.gsub(/^https:\/\/r\d---.{11}\.c\.youtube\.com[^\n]*/m) do |match| - path = URI.parse(match).path - - path = path.lchop("/videoplayback/") - path = path.rchop("/") - - path = path.gsub(/mime\/\w+\/\w+/) do |mimetype| - mimetype = mimetype.split("/") - mimetype[0] + "/" + mimetype[1] + "%2F" + mimetype[2] - end - - path = path.split("/") - - raw_params = {} of String => Array(String) - path.each_slice(2) do |pair| - key, value = pair - value = URI.decode_www_form(value) - - if raw_params[key]? - raw_params[key] << value - else - raw_params[key] = [value] - end - end - - raw_params = HTTP::Params.new(raw_params) - if fvip = raw_params["hls_chunk_host"].match(/r(?\d+)---/) - raw_params["fvip"] = fvip["fvip"] - end - - raw_params["local"] = "true" - - "#{HOST_URL}/videoplayback?#{raw_params}" - end - end - - manifest -end - -# YouTube /videoplayback links expire after 6 hours, -# so we have a mechanism here to redirect to the latest version -get "/latest_version" do |env| - if env.params.query["download_widget"]? - download_widget = JSON.parse(env.params.query["download_widget"]) - - id = download_widget["id"].as_s - title = download_widget["title"].as_s - - if label = download_widget["label"]? - env.redirect "/api/v1/captions/#{id}?label=#{label}&title=#{title}" - next - else - itag = download_widget["itag"].as_s.to_i - local = "true" - end - end - - id ||= env.params.query["id"]? - itag ||= env.params.query["itag"]?.try &.to_i - - region = env.params.query["region"]? - - local ||= env.params.query["local"]? - local ||= "false" - local = local == "true" - - if !id || !itag - env.response.status_code = 400 - next - end - - video = get_video(id, PG_DB, region: region) - - fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag } - url = fmt.try &.["url"]?.try &.as_s - - if !url - env.response.status_code = 404 - next - end - - url = URI.parse(url).request_target.not_nil! if local - url = "#{url}&title=#{title}" if title - - env.redirect url -end - -options "/videoplayback" do |env| - env.response.headers.delete("Content-Type") - env.response.headers["Access-Control-Allow-Origin"] = "*" - env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" - env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range" -end - -options "/videoplayback/*" do |env| - env.response.headers.delete("Content-Type") - env.response.headers["Access-Control-Allow-Origin"] = "*" - env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" - env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range" -end - -options "/api/manifest/dash/id/videoplayback" do |env| - env.response.headers.delete("Content-Type") - env.response.headers["Access-Control-Allow-Origin"] = "*" - env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" - env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range" -end - -options "/api/manifest/dash/id/videoplayback/*" do |env| - env.response.headers.delete("Content-Type") - env.response.headers["Access-Control-Allow-Origin"] = "*" - env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" - env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range" -end - -get "/videoplayback/*" do |env| - path = env.request.path - - path = path.lchop("/videoplayback/") - path = path.rchop("/") - - path = path.gsub(/mime\/\w+\/\w+/) do |mimetype| - mimetype = mimetype.split("/") - mimetype[0] + "/" + mimetype[1] + "%2F" + mimetype[2] - end - - path = path.split("/") - - raw_params = {} of String => Array(String) - path.each_slice(2) do |pair| - key, value = pair - value = URI.decode_www_form(value) - - if raw_params[key]? - raw_params[key] << value - else - raw_params[key] = [value] - end - end - - query_params = HTTP::Params.new(raw_params) - - env.response.headers["Access-Control-Allow-Origin"] = "*" - env.redirect "/videoplayback?#{query_params}" -end - -get "/videoplayback" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - query_params = env.params.query - - fvip = query_params["fvip"]? || "3" - mns = query_params["mn"]?.try &.split(",") - mns ||= [] of String - - if query_params["region"]? - region = query_params["region"] - query_params.delete("region") - end - - if query_params["host"]? && !query_params["host"].empty? - host = "https://#{query_params["host"]}" - query_params.delete("host") - else - host = "https://r#{fvip}---#{mns.pop}.googlevideo.com" - end - - url = "/videoplayback?#{query_params.to_s}" - - headers = HTTP::Headers.new - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end - - client = make_client(URI.parse(host), region) - response = HTTP::Client::Response.new(500) - error = "" - 5.times do - begin - response = client.head(url, headers) - - if response.headers["Location"]? - location = URI.parse(response.headers["Location"]) - env.response.headers["Access-Control-Allow-Origin"] = "*" - - new_host = "#{location.scheme}://#{location.host}" - if new_host != host - host = new_host - client.close - client = make_client(URI.parse(new_host), region) - end - - url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" - else - break - end - rescue Socket::Addrinfo::Error - if !mns.empty? - mn = mns.pop - end - fvip = "3" - - host = "https://r#{fvip}---#{mn}.googlevideo.com" - client = make_client(URI.parse(host), region) - rescue ex - error = ex.message - end - end - - if response.status_code >= 400 - env.response.status_code = response.status_code - env.response.content_type = "text/plain" - next error - end - - if url.includes? "&file=seg.ts" - if CONFIG.disabled?("livestreams") - next error_template(403, "Administrator has disabled this endpoint.") - end - - begin - client.get(url, headers) do |response| - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if location = response.headers["Location"]? - location = URI.parse(location) - location = "#{location.request_target}&host=#{location.host}" - - if region - location += "®ion=#{region}" - end - - next env.redirect location - end - - IO.copy(response.body_io, env.response) - end - rescue ex - end - else - if query_params["title"]? && CONFIG.disabled?("downloads") || - CONFIG.disabled?("dash") - next error_template(403, "Administrator has disabled this endpoint.") - end - - content_length = nil - first_chunk = true - range_start, range_end = parse_range(env.request.headers["Range"]?) - chunk_start = range_start - chunk_end = range_end - - if !chunk_end || chunk_end - chunk_start > HTTP_CHUNK_SIZE - chunk_end = chunk_start + HTTP_CHUNK_SIZE - 1 - end - - # TODO: Record bytes written so we can restart after a chunk fails - while true - if !range_end && content_length - range_end = content_length - end - - if range_end && chunk_start > range_end - break - end - - if range_end && chunk_end > range_end - chunk_end = range_end - end - - headers["Range"] = "bytes=#{chunk_start}-#{chunk_end}" - - begin - client.get(url, headers) do |response| - if first_chunk - if !env.request.headers["Range"]? && response.status_code == 206 - env.response.status_code = 200 - else - env.response.status_code = response.status_code - end - - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) && key.downcase != "content-range" - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if location = response.headers["Location"]? - location = URI.parse(location) - location = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" - - env.redirect location - break - end - - if title = query_params["title"]? - # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ - env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}" - end - - if !response.headers.includes_word?("Transfer-Encoding", "chunked") - content_length = response.headers["Content-Range"].split("/")[-1].to_i64 - if env.request.headers["Range"]? - env.response.headers["Content-Range"] = "bytes #{range_start}-#{range_end || (content_length - 1)}/#{content_length}" - env.response.content_length = ((range_end.try &.+ 1) || content_length) - range_start - else - env.response.content_length = content_length - end - end - end - - proxy_file(response, env) - end - rescue ex - if ex.message != "Error reading socket: Connection reset by peer" - break - else - client.close - client = make_client(URI.parse(host), region) - end - end - - chunk_start = chunk_end + 1 - chunk_end += HTTP_CHUNK_SIZE - first_chunk = false - end - end - client.close -end - get "/ggpht/*" do |env| url = env.request.path.lchop("/ggpht") diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index a6ab4015..70623cc0 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -185,6 +185,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) if !author raise InfoException.new("Deleted or invalid channel") end + author = author.content # Auto-generated channels diff --git a/src/invidious/helpers/macros.cr b/src/invidious/helpers/macros.cr index 5d426a8b..75df1612 100644 --- a/src/invidious/helpers/macros.cr +++ b/src/invidious/helpers/macros.cr @@ -56,3 +56,12 @@ end macro rendered(filename) render "src/invidious/views/#{{{filename}}}.ecr" end + +# Similar to Kemals halt method but works in a +# method. +macro haltf(env, status_code = 200, response = "") + {{env}}.response.status_code = {{status_code}} + {{env}}.response.print {{response}} + {{env}}.response.close + return +end diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr new file mode 100644 index 00000000..31e1a123 --- /dev/null +++ b/src/invidious/routes/api/manifest.cr @@ -0,0 +1,237 @@ +module Invidious::Routes::APIManifest + # /api/manifest/dash/id/:id + def self.get_dash_video_id(env) + env.response.headers.add("Access-Control-Allow-Origin", "*") + env.response.content_type = "application/dash+xml" + + local = env.params.query["local"]?.try &.== "true" + id = env.params.url["id"] + region = env.params.query["region"]? + + # Since some implementations create playlists based on resolution regardless of different codecs, + # we can opt to only add a source to a representation if it has a unique height within that representation + unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe } + + begin + video = get_video(id, PG_DB, region: region) + rescue ex : VideoRedirect + return env.redirect env.request.resource.gsub(id, ex.video_id) + rescue ex + haltf env, status_code: 403 + end + + if dashmpd = video.dash_manifest_url + manifest = YT_POOL.client &.get(URI.parse(dashmpd).request_target).body + + manifest = manifest.gsub(/[^<]+<\/BaseURL>/) do |baseurl| + url = baseurl.lchop("") + url = url.rchop("") + + if local + uri = URI.parse(url) + url = "#{uri.request_target}host/#{uri.host}/" + end + + "#{url}" + end + + return manifest + end + + adaptive_fmts = video.adaptive_fmts + + if local + adaptive_fmts.each do |fmt| + fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) + end + end + + audio_streams = video.audio_streams + video_streams = video.video_streams.sort_by { |stream| {stream["width"].as_i, stream["fps"].as_i} }.reverse + + manifest = XML.build(indent: " ", encoding: "UTF-8") do |xml| + xml.element("MPD", "xmlns": "urn:mpeg:dash:schema:mpd:2011", + "profiles": "urn:mpeg:dash:profile:full:2011", minBufferTime: "PT1.5S", type: "static", + mediaPresentationDuration: "PT#{video.length_seconds}S") do + xml.element("Period") do + i = 0 + + {"audio/mp4", "audio/webm"}.each do |mime_type| + mime_streams = audio_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type } + next if mime_streams.empty? + + xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true) do + mime_streams.each do |fmt| + codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"') + bandwidth = fmt["bitrate"].as_i + itag = fmt["itag"].as_i + url = fmt["url"].as_s + + xml.element("Representation", id: fmt["itag"], codecs: codecs, bandwidth: bandwidth) do + xml.element("AudioChannelConfiguration", schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011", + value: "2") + xml.element("BaseURL") { xml.text url } + xml.element("SegmentBase", indexRange: "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}") do + xml.element("Initialization", range: "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}") + end + end + end + end + + i += 1 + end + + potential_heights = {4320, 2160, 1440, 1080, 720, 480, 360, 240, 144} + + {"video/mp4", "video/webm"}.each do |mime_type| + mime_streams = video_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type } + next if mime_streams.empty? + + heights = [] of Int32 + xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, scanType: "progressive") do + mime_streams.each do |fmt| + codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"') + bandwidth = fmt["bitrate"].as_i + itag = fmt["itag"].as_i + url = fmt["url"].as_s + width = fmt["width"].as_i + height = fmt["height"].as_i + + # Resolutions reported by YouTube player (may not accurately reflect source) + height = potential_heights.min_by { |i| (height - i).abs } + next if unique_res && heights.includes? height + heights << height + + xml.element("Representation", id: itag, codecs: codecs, width: width, height: height, + startWithSAP: "1", maxPlayoutRate: "1", + bandwidth: bandwidth, frameRate: fmt["fps"]) do + xml.element("BaseURL") { xml.text url } + xml.element("SegmentBase", indexRange: "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}") do + xml.element("Initialization", range: "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}") + end + end + end + end + + i += 1 + end + end + end + end + + return manifest + end + + # /api/manifest/dash/id/videoplayback + def self.get_dash_video_playback(env) + env.response.headers.delete("Content-Type") + env.response.headers["Access-Control-Allow-Origin"] = "*" + env.redirect "/videoplayback?#{env.params.query}" + end + + # /api/manifest/dash/id/videoplayback/* + def self.get_dash_video_playback_greedy(env) + env.response.headers.delete("Content-Type") + env.response.headers["Access-Control-Allow-Origin"] = "*" + env.redirect env.request.path.lchop("/api/manifest/dash/id") + end + + # /api/manifest/dash/id/videoplayback && /api/manifest/dash/id/videoplayback/* + def self.options_dash_video_playback(env) + env.response.headers.delete("Content-Type") + env.response.headers["Access-Control-Allow-Origin"] = "*" + env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" + env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range" + end + + # /api/manifest/hls_playlist/* + def self.get_hls_playlist(env) + response = YT_POOL.client &.get(env.request.path) + + if response.status_code != 200 + haltf env, status_code: response.status_code + end + + local = env.params.query["local"]?.try &.== "true" + + env.response.content_type = "application/x-mpegURL" + env.response.headers.add("Access-Control-Allow-Origin", "*") + + manifest = response.body + + if local + manifest = manifest.gsub(/^https:\/\/r\d---.{11}\.c\.youtube\.com[^\n]*/m) do |match| + path = URI.parse(match).path + + path = path.lchop("/videoplayback/") + path = path.rchop("/") + + path = path.gsub(/mime\/\w+\/\w+/) do |mimetype| + mimetype = mimetype.split("/") + mimetype[0] + "/" + mimetype[1] + "%2F" + mimetype[2] + end + + path = path.split("/") + + raw_params = {} of String => Array(String) + path.each_slice(2) do |pair| + key, value = pair + value = URI.decode_www_form(value) + + if raw_params[key]? + raw_params[key] << value + else + raw_params[key] = [value] + end + end + + raw_params = HTTP::Params.new(raw_params) + if fvip = raw_params["hls_chunk_host"].match(/r(?\d+)---/) + raw_params["fvip"] = fvip["fvip"] + end + + raw_params["local"] = "true" + + "#{HOST_URL}/videoplayback?#{raw_params}" + end + end + + manifest + end + + # /api/manifest/hls_variant/* + def self.get_hls_variant(env) + response = YT_POOL.client &.get(env.request.path) + + if response.status_code != 200 + haltf env, status_code: response.status_code + end + + local = env.params.query["local"]?.try &.== "true" + + env.response.content_type = "application/x-mpegURL" + env.response.headers.add("Access-Control-Allow-Origin", "*") + + manifest = response.body + + if local + manifest = manifest.gsub("https://www.youtube.com", HOST_URL) + manifest = manifest.gsub("index.m3u8", "index.m3u8?local=true") + end + + manifest + end +end + +macro define_api_manifest_routes + Invidious::Routing.get "/api/manifest/dash/id/:id", Invidious::Routes::APIManifest, :get_dash_video_id + + Invidious::Routing.get "/api/manifest/dash/id/videoplayback", Invidious::Routes::APIManifest, :get_dash_video_playback + Invidious::Routing.get "/api/manifest/dash/id/videoplayback/*", Invidious::Routes::APIManifest, :get_dash_video_playback_greedy + + Invidious::Routing.options "/api/manifest/dash/id/videoplayback", Invidious::Routes::APIManifest, :options_dash_video_playback + Invidious::Routing.options "/api/manifest/dash/id/videoplayback/*", Invidious::Routes::APIManifest, :options_dash_video_playback + + Invidious::Routing.get "/api/manifest/hls_playlist/*", Invidious::Routes::APIManifest, :get_hls_playlist + Invidious::Routing.get "/api/manifest/hls_variant/*", Invidious::Routes::APIManifest, :get_hls_variant +end diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr new file mode 100644 index 00000000..a8b06bf7 --- /dev/null +++ b/src/invidious/routes/api/v1/channels.cr @@ -0,0 +1,243 @@ +module Invidious::Routes::APIv1 + def self.home(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + ucid = env.params.url["ucid"] + sort_by = env.params.query["sort_by"]?.try &.downcase + sort_by ||= "newest" + + begin + channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) + return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) + rescue ex + return error_json(500, ex) + end + + page = 1 + if channel.auto_generated + videos = [] of SearchVideo + count = 0 + else + begin + count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) + rescue ex + return error_json(500, ex) + end + end + + JSON.build do |json| + # TODO: Refactor into `to_json` for InvidiousChannel + json.object do + json.field "author", channel.author + json.field "authorId", channel.ucid + json.field "authorUrl", channel.author_url + + json.field "authorBanners" do + json.array do + if channel.banner + qualities = { + {width: 2560, height: 424}, + {width: 2120, height: 351}, + {width: 1060, height: 175}, + } + qualities.each do |quality| + json.object do + json.field "url", channel.banner.not_nil!.gsub("=w1060-", "=w#{quality[:width]}-") + json.field "width", quality[:width] + json.field "height", quality[:height] + end + end + + json.object do + json.field "url", channel.banner.not_nil!.split("=w1060-")[0] + json.field "width", 512 + json.field "height", 288 + end + end + end + end + + json.field "authorThumbnails" do + json.array do + qualities = {32, 48, 76, 100, 176, 512} + + qualities.each do |quality| + json.object do + json.field "url", channel.author_thumbnail.gsub(/=s\d+/, "=s#{quality}") + json.field "width", quality + json.field "height", quality + end + end + end + end + + json.field "subCount", channel.sub_count + json.field "totalViews", channel.total_views + json.field "joined", channel.joined.to_unix + + json.field "autoGenerated", channel.auto_generated + json.field "isFamilyFriendly", channel.is_family_friendly + json.field "description", html_to_content(channel.description_html) + json.field "descriptionHtml", channel.description_html + + json.field "allowedRegions", channel.allowed_regions + + json.field "latestVideos" do + json.array do + videos.each do |video| + video.to_json(locale, json) + end + end + end + + json.field "relatedChannels" do + json.array do + channel.related_channels.each do |related_channel| + json.object do + json.field "author", related_channel.author + json.field "authorId", related_channel.ucid + json.field "authorUrl", related_channel.author_url + + json.field "authorThumbnails" do + json.array do + qualities = {32, 48, 76, 100, 176, 512} + + qualities.each do |quality| + json.object do + json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}") + json.field "width", quality + json.field "height", quality + end + end + end + end + end + end + end + end + end + end + end + + def self.latest(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + ucid = env.params.url["ucid"] + + begin + videos = get_latest_videos(ucid) + rescue ex + return error_json(500, ex) + end + + JSON.build do |json| + json.array do + videos.each do |video| + video.to_json(locale, json) + end + end + end + end + + def self.videos(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + ucid = env.params.url["ucid"] + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + sort_by = env.params.query["sort"]?.try &.downcase + sort_by ||= env.params.query["sort_by"]?.try &.downcase + sort_by ||= "newest" + + begin + channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) + return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) + rescue ex + return error_json(500, ex) + end + + begin + count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) + rescue ex + return error_json(500, ex) + end + + JSON.build do |json| + json.array do + videos.each do |video| + video.to_json(locale, json) + end + end + end + end + + def self.playlists(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + ucid = env.params.url["ucid"] + continuation = env.params.query["continuation"]? + sort_by = env.params.query["sort"]?.try &.downcase || + env.params.query["sort_by"]?.try &.downcase || + "last" + + begin + channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) + return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) + rescue ex + return error_json(500, ex) + end + + items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) + + JSON.build do |json| + json.object do + json.field "playlists" do + json.array do + items.each do |item| + item.to_json(locale, json) if item.is_a?(SearchPlaylist) + end + end + end + + json.field "continuation", continuation + end + end + end + + def self.community(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + ucid = env.params.url["ucid"] + + thin_mode = env.params.query["thin_mode"]? + thin_mode = thin_mode == "true" + + format = env.params.query["format"]? + format ||= "json" + + continuation = env.params.query["continuation"]? + # sort_by = env.params.query["sort_by"]?.try &.downcase + + begin + fetch_channel_community(ucid, continuation, locale, format, thin_mode) + rescue ex + return error_json(500, ex) + end + end +end diff --git a/src/invidious/routes/api/v1/feeds.cr b/src/invidious/routes/api/v1/feeds.cr new file mode 100644 index 00000000..c24266c6 --- /dev/null +++ b/src/invidious/routes/api/v1/feeds.cr @@ -0,0 +1,46 @@ +module Invidious::Routes::APIv1 + def self.trending(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + region = env.params.query["region"]? + trending_type = env.params.query["type"]? + + begin + trending, plid = fetch_trending(trending_type, region, locale) + rescue ex + return error_json(500, ex) + end + + videos = JSON.build do |json| + json.array do + trending.each do |video| + video.to_json(locale, json) + end + end + end + + videos + end + + def self.popular(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + if !CONFIG.popular_enabled + error_message = {"error" => "Administrator has disabled this endpoint."}.to_json + env.response.status_code = 400 + return error_message + end + + JSON.build do |json| + json.array do + popular_videos.each do |video| + video.to_json(locale, json) + end + end + end + end +end diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr new file mode 100644 index 00000000..4bf8b8b0 --- /dev/null +++ b/src/invidious/routes/api/v1/misc.cr @@ -0,0 +1,13 @@ +module Invidious::Routes::APIv1 + # Stats API endpoint for Invidious + def self.stats(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + env.response.content_type = "application/json" + + if !CONFIG.statistics_enabled + return error_json(400, "Statistics are not enabled.") + end + + Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json + end +end diff --git a/src/invidious/routes/api/v1/routes.cr b/src/invidious/routes/api/v1/routes.cr new file mode 100644 index 00000000..5c61ed7c --- /dev/null +++ b/src/invidious/routes/api/v1/routes.cr @@ -0,0 +1,37 @@ +# There is far too many API routes to define in invidious.cr +# so we'll just do it here instead with a macro. +macro define_v1_api_routes(base_url = "/api/v1") + Invidious::Routing.get "#{{{base_url}}}/stats", Invidious::Routes::APIv1, :stats + + # Widgets + Invidious::Routing.get "#{{{base_url}}}/storyboards/:id", Invidious::Routes::APIv1, :storyboards + Invidious::Routing.get "#{{{base_url}}}/captions/:id", Invidious::Routes::APIv1, :captions + Invidious::Routing.get "#{{{base_url}}}/annotations/:id", Invidious::Routes::APIv1, :annotations + Invidious::Routing.get "#{{{base_url}}}/search/suggestions/:id", Invidious::Routes::APIv1, :search_suggestions + Invidious::Routing.get "#{{{base_url}}}/comments/:id", Invidious::Routes::APIv1, :comments + + # Feeds + Invidious::Routing.get "#{{{base_url}}}/trending", Invidious::Routes::APIv1, :trending + Invidious::Routing.get "#{{{base_url}}}/popular", Invidious::Routes::APIv1, :popular + + # Channels + Invidious::Routing.get "#{{{base_url}}}/channels/:ucid", Invidious::Routes::APIv1, :home + {% for route in { + {"home", "home"}, + {"videos", "videos"}, + {"latest", "latest"}, + {"playlists", "playlists"}, + {"comments", "community"}, # Why is the route for the community API `comments`?, + {"search", "channel_search"}, + } %} + + Invidious::Routing.get "#{{{base_url}}}/channels/#{{{route[0]}}}/:ucid", Invidious::Routes::APIv1, :{{route[1]}} + Invidious::Routing.get "#{{{base_url}}}/channels/:ucid/#{{{route[0]}}}", Invidious::Routes::APIv1, :{{route[1]}} + {% end %} + + # Search + Invidious::Routing.get "#{{{base_url}}}/search", Invidious::Routes::APIv1, :search + Invidious::Routing.get "#{{{base_url}}}/videos/:id", Invidious::Routes::APIv1, :videos + Invidious::Routing.get "#{{{base_url}}}/search", Invidious::Routes::APIv1, :search + +end diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr new file mode 100644 index 00000000..d1ed645d --- /dev/null +++ b/src/invidious/routes/api/v1/search.cr @@ -0,0 +1,101 @@ +module Invidious::Routes::APIv1 + def self.search(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + region = env.params.query["region"]? + + env.response.content_type = "application/json" + + query = env.params.query["q"]? + query ||= "" + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + sort_by = env.params.query["sort_by"]?.try &.downcase + sort_by ||= "relevance" + + date = env.params.query["date"]?.try &.downcase + date ||= "" + + duration = env.params.query["duration"]?.try &.downcase + duration ||= "" + + features = env.params.query["features"]?.try &.split(",").map { |feature| feature.downcase } + features ||= [] of String + + content_type = env.params.query["type"]?.try &.downcase + content_type ||= "video" + + begin + search_params = produce_search_params(page, sort_by, date, content_type, duration, features) + rescue ex + return error_json(400, ex) + end + + count, search_results = search(query, search_params, region).as(Tuple) + JSON.build do |json| + json.array do + search_results.each do |item| + item.to_json(locale, json) + end + end + end + end + + def self.channel_search(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + ucid = env.params.url["ucid"] + + query = env.params.query["q"]? + query ||= "" + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + count, search_results = channel_search(query, page, ucid) + JSON.build do |json| + json.array do + search_results.each do |item| + item.to_json(locale, json) + end + end + end + end + + def self.search_suggestions(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + region = env.params.query["region"]? + + env.response.content_type = "application/json" + + query = env.params.query["q"]? + query ||= "" + + begin + headers = HTTP::Headers{":authority" => "suggestqueries.google.com"} + response = YT_POOL.client &.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback", headers).body + + body = response[35..-2] + body = JSON.parse(body).as_a + suggestions = body[1].as_a[0..-2] + + JSON.build do |json| + json.object do + json.field "query", body[0].as_s + json.field "suggestions" do + json.array do + suggestions.each do |suggestion| + json.string suggestion[0].as_s + end + end + end + end + end + rescue ex + return error_json(500, ex) + end + end +end diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr new file mode 100644 index 00000000..7b7433f2 --- /dev/null +++ b/src/invidious/routes/api/v1/videos.cr @@ -0,0 +1,372 @@ +module Invidious::Routes::APIv1 + def self.videos(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + id = env.params.url["id"] + region = env.params.query["region"]? + + begin + video = get_video(id, PG_DB, region: region) + rescue ex : VideoRedirect + env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) + return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) + rescue ex + return error_json(500, ex) + end + + video.to_json(locale) + end + + def self.captions(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + id = env.params.url["id"] + region = env.params.query["region"]? + + # See https://github.com/ytdl-org/youtube-dl/blob/6ab30ff50bf6bd0585927cb73c7421bef184f87a/youtube_dl/extractor/youtube.py#L1354 + # It is possible to use `/api/timedtext?type=list&v=#{id}` and + # `/api/timedtext?type=track&v=#{id}&lang=#{lang_code}` directly, + # but this does not provide links for auto-generated captions. + # + # In future this should be investigated as an alternative, since it does not require + # getting video info. + + begin + video = get_video(id, PG_DB, region: region) + rescue ex : VideoRedirect + env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) + return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) + rescue ex + env.response.status_code = 500 + return + end + + captions = video.captions + + label = env.params.query["label"]? + lang = env.params.query["lang"]? + tlang = env.params.query["tlang"]? + + if !label && !lang + response = JSON.build do |json| + json.object do + json.field "captions" do + json.array do + captions.each do |caption| + json.object do + json.field "label", caption.name + json.field "languageCode", caption.languageCode + json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}" + end + end + end + end + end + end + + return response + end + + env.response.content_type = "text/vtt; charset=UTF-8" + + if lang + caption = captions.select { |caption| caption.languageCode == lang } + else + caption = captions.select { |caption| caption.name == label } + end + + if caption.empty? + env.response.status_code = 404 + return + else + caption = caption[0] + end + + url = URI.parse("#{caption.baseUrl}&tlang=#{tlang}").request_target + + # Auto-generated captions often have cues that aren't aligned properly with the video, + # as well as some other markup that makes it cumbersome, so we try to fix that here + if caption.name.includes? "auto-generated" + caption_xml = YT_POOL.client &.get(url).body + caption_xml = XML.parse(caption_xml) + + webvtt = String.build do |str| + str << <<-END_VTT + WEBVTT + Kind: captions + Language: #{tlang || caption.languageCode} + + + END_VTT + + caption_nodes = caption_xml.xpath_nodes("//transcript/text") + caption_nodes.each_with_index do |node, i| + start_time = node["start"].to_f.seconds + duration = node["dur"]?.try &.to_f.seconds + duration ||= start_time + + if caption_nodes.size > i + 1 + end_time = caption_nodes[i + 1]["start"].to_f.seconds + else + end_time = start_time + duration + end + + start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}" + end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}" + + text = HTML.unescape(node.content) + text = text.gsub(//, "") + text = text.gsub(/<\/font>/, "") + if md = text.match(/(?.*) : (?.*)/) + text = "#{md["text"]}" + end + + str << <<-END_CUE + #{start_time} --> #{end_time} + #{text} + + + END_CUE + end + end + else + webvtt = YT_POOL.client &.get("#{url}&format=vtt").body + end + + if title = env.params.query["title"]? + # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ + env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}" + end + + webvtt + end + + # Fetches YouTube storyboards + # + # Which are sprites containing x * y preview + # thumbnails for individual scenes in a video. + # See https://support.jwplayer.com/articles/how-to-add-preview-thumbnails + def self.storyboards(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + + id = env.params.url["id"] + region = env.params.query["region"]? + + begin + video = get_video(id, PG_DB, region: region) + rescue ex : VideoRedirect + env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) + return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) + rescue ex + env.response.status_code = 500 + return + end + + storyboards = video.storyboards + width = env.params.query["width"]? + height = env.params.query["height"]? + + if !width && !height + response = JSON.build do |json| + json.object do + json.field "storyboards" do + generate_storyboards(json, id, storyboards) + end + end + end + + return response + end + + env.response.content_type = "text/vtt" + + storyboard = storyboards.select { |storyboard| width == "#{storyboard[:width]}" || height == "#{storyboard[:height]}" } + + if storyboard.empty? + env.response.status_code = 404 + return + else + storyboard = storyboard[0] + end + + String.build do |str| + str << <<-END_VTT + WEBVTT + END_VTT + + start_time = 0.milliseconds + end_time = storyboard[:interval].milliseconds + + storyboard[:storyboard_count].times do |i| + url = storyboard[:url] + authority = /(i\d?).ytimg.com/.match(url).not_nil![1]? + url = url.gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "") + url = "#{HOST_URL}/sb/#{authority}/#{url}" + + storyboard[:storyboard_height].times do |j| + storyboard[:storyboard_width].times do |k| + str << <<-END_CUE + #{start_time}.000 --> #{end_time}.000 + #{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]} + + + END_CUE + + start_time += storyboard[:interval].milliseconds + end_time += storyboard[:interval].milliseconds + end + end + end + end + end + + def self.annotations(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "text/xml" + + id = env.params.url["id"] + source = env.params.query["source"]? + source ||= "archive" + + if !id.match(/[a-zA-Z0-9_-]{11}/) + env.response.status_code = 400 + return + end + + annotations = "" + + case source + when "archive" + if CONFIG.cache_annotations && (cached_annotation = PG_DB.query_one?("SELECT * FROM annotations WHERE id = $1", id, as: Annotation)) + annotations = cached_annotation.annotations + else + index = CHARS_SAFE.index(id[0]).not_nil!.to_s.rjust(2, '0') + + # IA doesn't handle leading hyphens, + # so we use https://archive.org/details/youtubeannotations_64 + if index == "62" + index = "64" + id = id.sub(/^-/, 'A') + end + + file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml") + + location = make_client(ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}")) + + if !location.headers["Location"]? + env.response.status_code = location.status_code + end + + response = make_client(URI.parse(location.headers["Location"]), &.get(location.headers["Location"])) + + if response.body.empty? + env.response.status_code = 404 + return + end + + if response.status_code != 200 + env.response.status_code = response.status_code + return + end + + annotations = response.body + + cache_annotation(PG_DB, id, annotations) + end + else # "youtube" + response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}") + + if response.status_code != 200 + env.response.status_code = response.status_code + return + end + + annotations = response.body + end + + etag = sha256(annotations)[0, 16] + if env.request.headers["If-None-Match"]?.try &.== etag + env.response.status_code = 304 + else + env.response.headers["ETag"] = etag + annotations + end + end + + def self.comments(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + region = env.params.query["region"]? + + env.response.content_type = "application/json" + + id = env.params.url["id"] + + source = env.params.query["source"]? + source ||= "youtube" + + thin_mode = env.params.query["thin_mode"]? + thin_mode = thin_mode == "true" + + format = env.params.query["format"]? + format ||= "json" + + action = env.params.query["action"]? + action ||= "action_get_comments" + + continuation = env.params.query["continuation"]? + sort_by = env.params.query["sort_by"]?.try &.downcase + + if source == "youtube" + sort_by ||= "top" + + begin + comments = fetch_youtube_comments(id, continuation, format, locale, thin_mode, region, sort_by: sort_by) + rescue ex + return error_json(500, ex) + end + + return comments + elsif source == "reddit" + sort_by ||= "confidence" + + begin + comments, reddit_thread = fetch_reddit_comments(id, sort_by: sort_by) + content_html = template_reddit_comments(comments, locale) + + content_html = fill_links(content_html, "https", "www.reddit.com") + content_html = replace_links(content_html) + rescue ex + comments = nil + reddit_thread = nil + content_html = "" + end + + if !reddit_thread || !comments + env.response.status_code = 404 + return + end + + if format == "json" + reddit_thread = JSON.parse(reddit_thread.to_json).as_h + reddit_thread["comments"] = JSON.parse(comments.to_json) + + return reddit_thread.to_json + else + response = { + "title" => reddit_thread.title, + "permalink" => reddit_thread.permalink, + "contentHtml" => content_html, + } + + return response.to_json + end + end + end +end diff --git a/src/invidious/routes/api_v1/channels.cr b/src/invidious/routes/api_v1/channels.cr deleted file mode 100644 index 03ebebfb..00000000 --- a/src/invidious/routes/api_v1/channels.cr +++ /dev/null @@ -1,244 +0,0 @@ -module Invidious::Routes::APIv1 - def self.home(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - ucid = env.params.url["ucid"] - sort_by = env.params.query["sort_by"]?.try &.downcase - sort_by ||= "newest" - - begin - channel = get_about_info(ucid, locale) - rescue ex : ChannelRedirect - env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) - return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) - rescue ex - return error_json(500, ex) - end - - page = 1 - if channel.auto_generated - videos = [] of SearchVideo - count = 0 - else - begin - count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) - rescue ex - return error_json(500, ex) - end - end - - JSON.build do |json| - # TODO: Refactor into `to_json` for InvidiousChannel - json.object do - json.field "author", channel.author - json.field "authorId", channel.ucid - json.field "authorUrl", channel.author_url - - json.field "authorBanners" do - json.array do - if channel.banner - qualities = { - {width: 2560, height: 424}, - {width: 2120, height: 351}, - {width: 1060, height: 175}, - } - qualities.each do |quality| - json.object do - json.field "url", channel.banner.not_nil!.gsub("=w1060-", "=w#{quality[:width]}-") - json.field "width", quality[:width] - json.field "height", quality[:height] - end - end - - json.object do - json.field "url", channel.banner.not_nil!.split("=w1060-")[0] - json.field "width", 512 - json.field "height", 288 - end - end - end - end - - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - - qualities.each do |quality| - json.object do - json.field "url", channel.author_thumbnail.gsub(/=s\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", quality - end - end - end - end - - json.field "subCount", channel.sub_count - json.field "totalViews", channel.total_views - json.field "joined", channel.joined.to_unix - json.field "paid", channel.paid - - json.field "autoGenerated", channel.auto_generated - json.field "isFamilyFriendly", channel.is_family_friendly - json.field "description", html_to_content(channel.description_html) - json.field "descriptionHtml", channel.description_html - - json.field "allowedRegions", channel.allowed_regions - - json.field "latestVideos" do - json.array do - videos.each do |video| - video.to_json(locale, json) - end - end - end - - json.field "relatedChannels" do - json.array do - channel.related_channels.each do |related_channel| - json.object do - json.field "author", related_channel.author - json.field "authorId", related_channel.ucid - json.field "authorUrl", related_channel.author_url - - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - - qualities.each do |quality| - json.object do - json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", quality - end - end - end - end - end - end - end - end - end - end - end - - def self.latest(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - ucid = env.params.url["ucid"] - - begin - videos = get_latest_videos(ucid) - rescue ex - return error_json(500, ex) - end - - JSON.build do |json| - json.array do - videos.each do |video| - video.to_json(locale, json) - end - end - end - end - - def self.videos(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - ucid = env.params.url["ucid"] - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - sort_by = env.params.query["sort"]?.try &.downcase - sort_by ||= env.params.query["sort_by"]?.try &.downcase - sort_by ||= "newest" - - begin - channel = get_about_info(ucid, locale) - rescue ex : ChannelRedirect - env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) - return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) - rescue ex - return error_json(500, ex) - end - - begin - count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) - rescue ex - return error_json(500, ex) - end - - JSON.build do |json| - json.array do - videos.each do |video| - video.to_json(locale, json) - end - end - end - end - - def self.playlists(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - ucid = env.params.url["ucid"] - continuation = env.params.query["continuation"]? - sort_by = env.params.query["sort"]?.try &.downcase || - env.params.query["sort_by"]?.try &.downcase || - "last" - - begin - channel = get_about_info(ucid, locale) - rescue ex : ChannelRedirect - env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) - return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) - rescue ex - return error_json(500, ex) - end - - items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) - - JSON.build do |json| - json.object do - json.field "playlists" do - json.array do - items.each do |item| - item.to_json(locale, json) if item.is_a?(SearchPlaylist) - end - end - end - - json.field "continuation", continuation - end - end - end - - def self.community(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - ucid = env.params.url["ucid"] - - thin_mode = env.params.query["thin_mode"]? - thin_mode = thin_mode == "true" - - format = env.params.query["format"]? - format ||= "json" - - continuation = env.params.query["continuation"]? - # sort_by = env.params.query["sort_by"]?.try &.downcase - - begin - fetch_channel_community(ucid, continuation, locale, format, thin_mode) - rescue ex - return error_json(500, ex) - end - end -end diff --git a/src/invidious/routes/api_v1/feeds.cr b/src/invidious/routes/api_v1/feeds.cr deleted file mode 100644 index c24266c6..00000000 --- a/src/invidious/routes/api_v1/feeds.cr +++ /dev/null @@ -1,46 +0,0 @@ -module Invidious::Routes::APIv1 - def self.trending(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - region = env.params.query["region"]? - trending_type = env.params.query["type"]? - - begin - trending, plid = fetch_trending(trending_type, region, locale) - rescue ex - return error_json(500, ex) - end - - videos = JSON.build do |json| - json.array do - trending.each do |video| - video.to_json(locale, json) - end - end - end - - videos - end - - def self.popular(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - if !CONFIG.popular_enabled - error_message = {"error" => "Administrator has disabled this endpoint."}.to_json - env.response.status_code = 400 - return error_message - end - - JSON.build do |json| - json.array do - popular_videos.each do |video| - video.to_json(locale, json) - end - end - end - end -end diff --git a/src/invidious/routes/api_v1/misc.cr b/src/invidious/routes/api_v1/misc.cr deleted file mode 100644 index 4bf8b8b0..00000000 --- a/src/invidious/routes/api_v1/misc.cr +++ /dev/null @@ -1,13 +0,0 @@ -module Invidious::Routes::APIv1 - # Stats API endpoint for Invidious - def self.stats(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - env.response.content_type = "application/json" - - if !CONFIG.statistics_enabled - return error_json(400, "Statistics are not enabled.") - end - - Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json - end -end diff --git a/src/invidious/routes/api_v1/routes.cr b/src/invidious/routes/api_v1/routes.cr deleted file mode 100644 index ec3d9dff..00000000 --- a/src/invidious/routes/api_v1/routes.cr +++ /dev/null @@ -1,30 +0,0 @@ -# There is far too many API routes to define in invidious.cr -# so we'll just do it here instead with a macro. -macro define_v1_api_routes(base_url = "/api/v1") - Invidious::Routing.get "#{{{base_url}}}/stats", Invidious::Routes::APIv1, :stats - - Invidious::Routing.get "#{{{base_url}}}/storyboards/:id", Invidious::Routes::APIv1, :storyboards - Invidious::Routing.get "#{{{base_url}}}/captions/:id", Invidious::Routes::APIv1, :captions - Invidious::Routing.get "#{{{base_url}}}/annotations/:id", Invidious::Routes::APIv1, :annotations - Invidious::Routing.get "#{{{base_url}}}/search/suggestions/:id", Invidious::Routes::APIv1, :search_suggestions - - Invidious::Routing.get "#{{{base_url}}}/comments/:id", Invidious::Routes::APIv1, :comments - Invidious::Routing.get "#{{{base_url}}}/trending", Invidious::Routes::APIv1, :trending - Invidious::Routing.get "#{{{base_url}}}/popular", Invidious::Routes::APIv1, :popular - - Invidious::Routing.get "#{{{base_url}}}/channels/:ucid", Invidious::Routes::APIv1, :home - - {% for route in { - {"home", "home"}, - {"videos", "videos"}, - {"latest", "latest"}, - {"playlists", "playlists"}, - {"comments", "community"}, # Why is the route for the community API `comments`?, - {"search", "channel_search"}, - } %} - - Invidious::Routing.get "#{{{base_url}}}/channels/#{{{route[0]}}}/:ucid", Invidious::Routes::APIv1, :{{route[1]}} - Invidious::Routing.get "#{{{base_url}}}/channels/:ucid/#{{{route[0]}}}", Invidious::Routes::APIv1, :{{route[1]}} - - {% end %} -end diff --git a/src/invidious/routes/api_v1/search.cr b/src/invidious/routes/api_v1/search.cr deleted file mode 100644 index 61fdadd8..00000000 --- a/src/invidious/routes/api_v1/search.cr +++ /dev/null @@ -1,24 +0,0 @@ -module Invidious::Routes::APIv1 - def self.channel_search(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - ucid = env.params.url["ucid"] - - query = env.params.query["q"]? - query ||= "" - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - count, search_results = channel_search(query, page, ucid) - JSON.build do |json| - json.array do - search_results.each do |item| - item.to_json(locale, json) - end - end - end - end -end diff --git a/src/invidious/routes/api_v1/video_playback.cr b/src/invidious/routes/api_v1/video_playback.cr deleted file mode 100644 index 16942b22..00000000 --- a/src/invidious/routes/api_v1/video_playback.cr +++ /dev/null @@ -1,2 +0,0 @@ -module Invidious::Routes::APIv1 -end diff --git a/src/invidious/routes/api_v1/widgets.cr b/src/invidious/routes/api_v1/widgets.cr deleted file mode 100644 index 0b1cf67e..00000000 --- a/src/invidious/routes/api_v1/widgets.cr +++ /dev/null @@ -1,386 +0,0 @@ -module Invidious::Routes::APIv1 - # Fetches YouTube storyboards - # - # Which are sprites containing x * y preview - # thumbnails for individual scenes in a video. - # See https://support.jwplayer.com/articles/how-to-add-preview-thumbnails - def self.storyboards(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - id = env.params.url["id"] - region = env.params.query["region"]? - - begin - video = get_video(id, PG_DB, region: region) - rescue ex : VideoRedirect - env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) - return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) - rescue ex - env.response.status_code = 500 - return - end - - storyboards = video.storyboards - width = env.params.query["width"]? - height = env.params.query["height"]? - - if !width && !height - response = JSON.build do |json| - json.object do - json.field "storyboards" do - generate_storyboards(json, id, storyboards) - end - end - end - - return response - end - - env.response.content_type = "text/vtt" - - storyboard = storyboards.select { |storyboard| width == "#{storyboard[:width]}" || height == "#{storyboard[:height]}" } - - if storyboard.empty? - env.response.status_code = 404 - return - else - storyboard = storyboard[0] - end - - String.build do |str| - str << <<-END_VTT - WEBVTT - END_VTT - - start_time = 0.milliseconds - end_time = storyboard[:interval].milliseconds - - storyboard[:storyboard_count].times do |i| - url = storyboard[:url] - authority = /(i\d?).ytimg.com/.match(url).not_nil![1]? - url = url.gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "") - url = "#{HOST_URL}/sb/#{authority}/#{url}" - - storyboard[:storyboard_height].times do |j| - storyboard[:storyboard_width].times do |k| - str << <<-END_CUE - #{start_time}.000 --> #{end_time}.000 - #{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]} - - - END_CUE - - start_time += storyboard[:interval].milliseconds - end_time += storyboard[:interval].milliseconds - end - end - end - end - end - - def self.captions(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/json" - - id = env.params.url["id"] - region = env.params.query["region"]? - - # See https://github.com/ytdl-org/youtube-dl/blob/6ab30ff50bf6bd0585927cb73c7421bef184f87a/youtube_dl/extractor/youtube.py#L1354 - # It is possible to use `/api/timedtext?type=list&v=#{id}` and - # `/api/timedtext?type=track&v=#{id}&lang=#{lang_code}` directly, - # but this does not provide links for auto-generated captions. - # - # In future this should be investigated as an alternative, since it does not require - # getting video info. - - begin - video = get_video(id, PG_DB, region: region) - rescue ex : VideoRedirect - env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) - return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) - rescue ex - env.response.status_code = 500 - return - end - - captions = video.captions - - label = env.params.query["label"]? - lang = env.params.query["lang"]? - tlang = env.params.query["tlang"]? - - if !label && !lang - response = JSON.build do |json| - json.object do - json.field "captions" do - json.array do - captions.each do |caption| - json.object do - json.field "label", caption.name - json.field "languageCode", caption.languageCode - json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}" - end - end - end - end - end - end - - return response - end - - env.response.content_type = "text/vtt; charset=UTF-8" - - if lang - caption = captions.select { |caption| caption.languageCode == lang } - else - caption = captions.select { |caption| caption.name == label } - end - - if caption.empty? - env.response.status_code = 404 - return - else - caption = caption[0] - end - - url = URI.parse("#{caption.baseUrl}&tlang=#{tlang}").request_target - - # Auto-generated captions often have cues that aren't aligned properly with the video, - # as well as some other markup that makes it cumbersome, so we try to fix that here - if caption.name.includes? "auto-generated" - caption_xml = YT_POOL.client &.get(url).body - caption_xml = XML.parse(caption_xml) - - webvtt = String.build do |str| - str << <<-END_VTT - WEBVTT - Kind: captions - Language: #{tlang || caption.languageCode} - - - END_VTT - - caption_nodes = caption_xml.xpath_nodes("//transcript/text") - caption_nodes.each_with_index do |node, i| - start_time = node["start"].to_f.seconds - duration = node["dur"]?.try &.to_f.seconds - duration ||= start_time - - if caption_nodes.size > i + 1 - end_time = caption_nodes[i + 1]["start"].to_f.seconds - else - end_time = start_time + duration - end - - start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}" - end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}" - - text = HTML.unescape(node.content) - text = text.gsub(//, "") - text = text.gsub(/<\/font>/, "") - if md = text.match(/(?.*) : (?.*)/) - text = "#{md["text"]}" - end - - str << <<-END_CUE - #{start_time} --> #{end_time} - #{text} - - - END_CUE - end - end - else - webvtt = YT_POOL.client &.get("#{url}&format=vtt").body - end - - if title = env.params.query["title"]? - # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ - env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}" - end - - webvtt - end - - def self.annotations(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "text/xml" - - id = env.params.url["id"] - source = env.params.query["source"]? - source ||= "archive" - - if !id.match(/[a-zA-Z0-9_-]{11}/) - env.response.status_code = 400 - return - end - - annotations = "" - - case source - when "archive" - if CONFIG.cache_annotations && (cached_annotation = PG_DB.query_one?("SELECT * FROM annotations WHERE id = $1", id, as: Annotation)) - annotations = cached_annotation.annotations - else - index = CHARS_SAFE.index(id[0]).not_nil!.to_s.rjust(2, '0') - - # IA doesn't handle leading hyphens, - # so we use https://archive.org/details/youtubeannotations_64 - if index == "62" - index = "64" - id = id.sub(/^-/, 'A') - end - - file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml") - - location = make_client(ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}")) - - if !location.headers["Location"]? - env.response.status_code = location.status_code - end - - response = make_client(URI.parse(location.headers["Location"]), &.get(location.headers["Location"])) - - if response.body.empty? - env.response.status_code = 404 - return - end - - if response.status_code != 200 - env.response.status_code = response.status_code - return - end - - annotations = response.body - - cache_annotation(PG_DB, id, annotations) - end - else # "youtube" - response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}") - - if response.status_code != 200 - env.response.status_code = response.status_code - return - end - - annotations = response.body - end - - etag = sha256(annotations)[0, 16] - if env.request.headers["If-None-Match"]?.try &.== etag - env.response.status_code = 304 - else - env.response.headers["ETag"] = etag - annotations - end - end - - def self.search_suggestions(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - region = env.params.query["region"]? - - env.response.content_type = "application/json" - - query = env.params.query["q"]? - query ||= "" - - begin - headers = HTTP::Headers{":authority" => "suggestqueries.google.com"} - response = YT_POOL.client &.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback", headers).body - - body = response[35..-2] - body = JSON.parse(body).as_a - suggestions = body[1].as_a[0..-2] - - JSON.build do |json| - json.object do - json.field "query", body[0].as_s - json.field "suggestions" do - json.array do - suggestions.each do |suggestion| - json.string suggestion[0].as_s - end - end - end - end - end - rescue ex - return error_json(500, ex) - end - end - - def self.comments(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - region = env.params.query["region"]? - - env.response.content_type = "application/json" - - id = env.params.url["id"] - - source = env.params.query["source"]? - source ||= "youtube" - - thin_mode = env.params.query["thin_mode"]? - thin_mode = thin_mode == "true" - - format = env.params.query["format"]? - format ||= "json" - - action = env.params.query["action"]? - action ||= "action_get_comments" - - continuation = env.params.query["continuation"]? - sort_by = env.params.query["sort_by"]?.try &.downcase - - if source == "youtube" - sort_by ||= "top" - - begin - comments = fetch_youtube_comments(id, continuation, format, locale, thin_mode, region, sort_by: sort_by) - rescue ex - return error_json(500, ex) - end - - return comments - elsif source == "reddit" - sort_by ||= "confidence" - - begin - comments, reddit_thread = fetch_reddit_comments(id, sort_by: sort_by) - content_html = template_reddit_comments(comments, locale) - - content_html = fill_links(content_html, "https", "www.reddit.com") - content_html = replace_links(content_html) - rescue ex - comments = nil - reddit_thread = nil - content_html = "" - end - - if !reddit_thread || !comments - env.response.status_code = 404 - return - end - - if format == "json" - reddit_thread = JSON.parse(reddit_thread.to_json).as_h - reddit_thread["comments"] = JSON.parse(comments.to_json) - - return reddit_thread.to_json - else - response = { - "title" => reddit_thread.title, - "permalink" => reddit_thread.permalink, - "contentHtml" => content_html, - } - - return response.to_json - end - end - end -end diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr new file mode 100644 index 00000000..0fe2853d --- /dev/null +++ b/src/invidious/routes/video_playback.cr @@ -0,0 +1,290 @@ +module Invidious::Routes::VideoPlayback + # /videoplayback + def self.get_video_playback(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + query_params = env.params.query + + fvip = query_params["fvip"]? || "3" + mns = query_params["mn"]?.try &.split(",") + mns ||= [] of String + + if query_params["region"]? + region = query_params["region"] + query_params.delete("region") + end + + if query_params["host"]? && !query_params["host"].empty? + host = "https://#{query_params["host"]}" + query_params.delete("host") + else + host = "https://r#{fvip}---#{mns.pop}.googlevideo.com" + end + + url = "/videoplayback?#{query_params.to_s}" + + headers = HTTP::Headers.new + REQUEST_HEADERS_WHITELIST.each do |header| + if env.request.headers[header]? + headers[header] = env.request.headers[header] + end + end + + client = make_client(URI.parse(host), region) + response = HTTP::Client::Response.new(500) + error = "" + 5.times do + begin + response = client.head(url, headers) + + if response.headers["Location"]? + location = URI.parse(response.headers["Location"]) + env.response.headers["Access-Control-Allow-Origin"] = "*" + + new_host = "#{location.scheme}://#{location.host}" + if new_host != host + host = new_host + client.close + client = make_client(URI.parse(new_host), region) + end + + url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" + else + break + end + rescue Socket::Addrinfo::Error + if !mns.empty? + mn = mns.pop + end + fvip = "3" + + host = "https://r#{fvip}---#{mn}.googlevideo.com" + client = make_client(URI.parse(host), region) + rescue ex + error = ex.message + end + end + + if response.status_code >= 400 + env.response.content_type = "text/plain" + haltf env, response.status_code + end + + if url.includes? "&file=seg.ts" + if CONFIG.disabled?("livestreams") + return error_template(403, "Administrator has disabled this endpoint.") + end + + begin + client.get(url, headers) do |response| + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value + end + end + + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if location = response.headers["Location"]? + location = URI.parse(location) + location = "#{location.request_target}&host=#{location.host}" + + if region + location += "®ion=#{region}" + end + + return env.redirect location + end + + IO.copy(response.body_io, env.response) + end + rescue ex + end + else + if query_params["title"]? && CONFIG.disabled?("downloads") || + CONFIG.disabled?("dash") + return error_template(403, "Administrator has disabled this endpoint.") + end + + content_length = nil + first_chunk = true + range_start, range_end = parse_range(env.request.headers["Range"]?) + chunk_start = range_start + chunk_end = range_end + + if !chunk_end || chunk_end - chunk_start > HTTP_CHUNK_SIZE + chunk_end = chunk_start + HTTP_CHUNK_SIZE - 1 + end + + # TODO: Record bytes written so we can restart after a chunk fails + while true + if !range_end && content_length + range_end = content_length + end + + if range_end && chunk_start > range_end + break + end + + if range_end && chunk_end > range_end + chunk_end = range_end + end + + headers["Range"] = "bytes=#{chunk_start}-#{chunk_end}" + + begin + client.get(url, headers) do |response| + if first_chunk + if !env.request.headers["Range"]? && response.status_code == 206 + env.response.status_code = 200 + else + env.response.status_code = response.status_code + end + + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) && key.downcase != "content-range" + env.response.headers[key] = value + end + end + + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if location = response.headers["Location"]? + location = URI.parse(location) + location = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" + + env.redirect location + break + end + + if title = query_params["title"]? + # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ + env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}" + end + + if !response.headers.includes_word?("Transfer-Encoding", "chunked") + content_length = response.headers["Content-Range"].split("/")[-1].to_i64 + if env.request.headers["Range"]? + env.response.headers["Content-Range"] = "bytes #{range_start}-#{range_end || (content_length - 1)}/#{content_length}" + env.response.content_length = ((range_end.try &.+ 1) || content_length) - range_start + else + env.response.content_length = content_length + end + end + end + + proxy_file(response, env) + end + rescue ex + if ex.message != "Error reading socket: Connection reset by peer" + break + else + client.close + client = make_client(URI.parse(host), region) + end + end + + chunk_start = chunk_end + 1 + chunk_end += HTTP_CHUNK_SIZE + first_chunk = false + end + end + client.close + end + + # /videoplayback/* + def self.get_video_playback_greedy(env) + path = env.request.path + + path = path.lchop("/videoplayback/") + path = path.rchop("/") + + path = path.gsub(/mime\/\w+\/\w+/) do |mimetype| + mimetype = mimetype.split("/") + mimetype[0] + "/" + mimetype[1] + "%2F" + mimetype[2] + end + + path = path.split("/") + + raw_params = {} of String => Array(String) + path.each_slice(2) do |pair| + key, value = pair + value = URI.decode_www_form(value) + + if raw_params[key]? + raw_params[key] << value + else + raw_params[key] = [value] + end + end + + query_params = HTTP::Params.new(raw_params) + + env.response.headers["Access-Control-Allow-Origin"] = "*" + return env.redirect "/videoplayback?#{query_params}" + end + + # /videoplayback/* && /videoplayback/* + def self.options_video_playback(env) + env.response.headers.delete("Content-Type") + env.response.headers["Access-Control-Allow-Origin"] = "*" + env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" + env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range" + end + + # /latest_version + # + # YouTube /videoplayback links expire after 6 hours, + # so we have a mechanism here to redirect to the latest version + def self.latest_version(env) + if env.params.query["download_widget"]? + download_widget = JSON.parse(env.params.query["download_widget"]) + + id = download_widget["id"].as_s + title = download_widget["title"].as_s + + if label = download_widget["label"]? + return env.redirect "/api/v1/captions/#{id}?label=#{label}&title=#{title}" + else + itag = download_widget["itag"].as_s.to_i + local = "true" + end + end + + id ||= env.params.query["id"]? + itag ||= env.params.query["itag"]?.try &.to_i + + region = env.params.query["region"]? + + local ||= env.params.query["local"]? + local ||= "false" + local = local == "true" + + if !id || !itag + haltf env, status_code: 400, response: "TESTING" + end + + video = get_video(id, PG_DB, region: region) + + fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag } + url = fmt.try &.["url"]?.try &.as_s + + if !url + haltf env, status_code: 404 + end + + url = URI.parse(url).request_target.not_nil! if local + url = "#{url}&title=#{title}" if title + + return env.redirect url + end +end + +macro define_video_playback_routes + Invidious::Routing.get "/videoplayback", Invidious::Routes::VideoPlayback, :get_video_playback + Invidious::Routing.get "/videoplayback/*", Invidious::Routes::VideoPlayback, :get_video_playback_greedy + + Invidious::Routing.options "/videoplayback", Invidious::Routes::VideoPlayback, :options_video_playback + Invidious::Routing.options "/videoplayback/*", Invidious::Routes::VideoPlayback, :options_video_playback + + Invidious::Routing.get "/latest_version", Invidious::Routes::VideoPlayback, :latest_version +end -- cgit v1.2.3