summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
authorPerflyst <mail@perflyst.de>2020-11-12 17:06:38 +0100
committerGitHub <noreply@github.com>2020-11-12 17:06:38 +0100
commitbb7d8735cbad7916b354412f965d48d886d2365e (patch)
tree4def28c6e8de906565d73b0c337d47b6a416e35a /src
parent1fc9506442ae18c7c7f0a684a59714e679678a54 (diff)
parent557b0d76abe978cd8044a48f89313ef805954713 (diff)
downloadinvidious-bb7d8735cbad7916b354412f965d48d886d2365e.tar.gz
invidious-bb7d8735cbad7916b354412f965d48d886d2365e.tar.bz2
invidious-bb7d8735cbad7916b354412f965d48d886d2365e.zip
Merge branch 'master' into patch-1
Diffstat (limited to 'src')
-rw-r--r--src/invidious.cr1262
-rw-r--r--src/invidious/channels.cr624
-rw-r--r--src/invidious/comments.cr106
-rw-r--r--src/invidious/helpers/handlers.cr32
-rw-r--r--src/invidious/helpers/helpers.cr860
-rw-r--r--src/invidious/helpers/i18n.cr2
-rw-r--r--src/invidious/helpers/jobs.cr370
-rw-r--r--src/invidious/helpers/macros.cr80
-rw-r--r--src/invidious/helpers/patch_mapping.cr166
-rw-r--r--src/invidious/helpers/signatures.cr56
-rw-r--r--src/invidious/helpers/static_file_handler.cr4
-rw-r--r--src/invidious/helpers/tokens.cr40
-rw-r--r--src/invidious/helpers/utils.cr60
-rw-r--r--src/invidious/jobs.cr13
-rw-r--r--src/invidious/jobs/base_job.cr3
-rw-r--r--src/invidious/jobs/bypass_captcha_job.cr131
-rw-r--r--src/invidious/jobs/notification_job.cr24
-rw-r--r--src/invidious/jobs/pull_popular_videos_job.cr27
-rw-r--r--src/invidious/jobs/refresh_channels_job.cr59
-rw-r--r--src/invidious/jobs/refresh_feeds_job.cr77
-rw-r--r--src/invidious/jobs/statistics_refresh_job.cr59
-rw-r--r--src/invidious/jobs/subscribe_to_feeds_job.cr52
-rw-r--r--src/invidious/jobs/update_decrypt_function_job.cr19
-rw-r--r--src/invidious/mixes.cr61
-rw-r--r--src/invidious/playlists.cr376
-rw-r--r--src/invidious/routes/base_route.cr9
-rw-r--r--src/invidious/routes/embed/index.cr27
-rw-r--r--src/invidious/routes/embed/show.cr174
-rw-r--r--src/invidious/routes/home.cr34
-rw-r--r--src/invidious/routes/licenses.cr6
-rw-r--r--src/invidious/routes/privacy.cr6
-rw-r--r--src/invidious/routes/watch.cr186
-rw-r--r--src/invidious/routing.cr8
-rw-r--r--src/invidious/search.cr204
-rw-r--r--src/invidious/trending.cr52
-rw-r--r--src/invidious/users.cr337
-rw-r--r--src/invidious/videos.cr1042
-rw-r--r--src/invidious/views/add_playlist_items.ecr12
-rw-r--r--src/invidious/views/channel.ecr6
-rw-r--r--src/invidious/views/community.ecr20
-rw-r--r--src/invidious/views/components/feed_menu.ecr28
-rw-r--r--src/invidious/views/components/item.ecr25
-rw-r--r--src/invidious/views/components/player.ecr35
-rw-r--r--src/invidious/views/components/player_sources.ecr2
-rw-r--r--src/invidious/views/components/subscribe_widget.ecr20
-rw-r--r--src/invidious/views/embed.ecr37
-rw-r--r--src/invidious/views/history.ecr14
-rw-r--r--src/invidious/views/login.ecr78
-rw-r--r--src/invidious/views/playlist.ecr18
-rw-r--r--src/invidious/views/playlists.ecr2
-rw-r--r--src/invidious/views/preferences.ecr15
-rw-r--r--src/invidious/views/search.ecr18
-rw-r--r--src/invidious/views/subscription_manager.ecr33
-rw-r--r--src/invidious/views/subscriptions.ecr10
-rw-r--r--src/invidious/views/template.ecr19
-rw-r--r--src/invidious/views/token_manager.ecr33
-rw-r--r--src/invidious/views/top.ecr20
-rw-r--r--src/invidious/views/view_all_playlists.ecr18
-rw-r--r--src/invidious/views/watch.ecr148
59 files changed, 3281 insertions, 3978 deletions
diff --git a/src/invidious.cr b/src/invidious.cr
index 8340ebab..284b238c 100644
--- a/src/invidious.cr
+++ b/src/invidious.cr
@@ -23,13 +23,18 @@ require "pg"
require "sqlite3"
require "xml"
require "yaml"
-require "zip"
+require "compress/zip"
require "protodec/utils"
require "./invidious/helpers/*"
require "./invidious/*"
+require "./invidious/routes/**"
+require "./invidious/jobs/**"
-CONFIG = Config.from_yaml(File.read("config/config.yml"))
-HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32)
+ENV_CONFIG_NAME = "INVIDIOUS_CONFIG"
+
+CONFIG_STR = ENV.has_key?(ENV_CONFIG_NAME) ? ENV.fetch(ENV_CONFIG_NAME) : File.read("config/config.yml")
+CONFIG = Config.from_yaml(CONFIG_STR)
+HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32)
PG_URL = URI.new(
scheme: "postgres",
@@ -45,9 +50,9 @@ ARCHIVE_URL = URI.parse("https://archive.org")
LOGIN_URL = URI.parse("https://accounts.google.com")
PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com")
REDDIT_URL = URI.parse("https://www.reddit.com")
-TEXTCAPTCHA_URL = URI.parse("http://textcaptcha.com")
+TEXTCAPTCHA_URL = URI.parse("https://textcaptcha.com")
YT_URL = URI.parse("https://www.youtube.com")
-YT_IMG_URL = URI.parse("https://i.ytimg.com")
+HOST_URL = make_host_url(CONFIG, Kemal.config)
CHARS_SAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
@@ -57,7 +62,7 @@ REQUEST_HEADERS_WHITELIST = {"accept", "accept-encoding", "cache-control", "con
RESPONSE_HEADERS_BLACKLIST = {"access-control-allow-origin", "alt-svc", "server"}
HTTP_CHUNK_SIZE = 10485760 # ~10MB
-CURRENT_BRANCH = {{ "#{`git branch | sed -n '/\* /s///p'`.strip}" }}
+CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}
CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }}
CURRENT_VERSION = {{ "#{`git describe --tags --abbrev=0`.strip}" }}
@@ -81,23 +86,25 @@ LOCALES = {
"es" => load_locale("es"),
"eu" => load_locale("eu"),
"fr" => load_locale("fr"),
+ "hu" => load_locale("hu-HU"),
"is" => load_locale("is"),
"it" => load_locale("it"),
"ja" => load_locale("ja"),
"nb-NO" => load_locale("nb-NO"),
"nl" => load_locale("nl"),
- "pt-BR" => load_locale("pt-BR"),
"pl" => load_locale("pl"),
+ "pt-BR" => load_locale("pt-BR"),
+ "pt-PT" => load_locale("pt-PT"),
"ro" => load_locale("ro"),
"ru" => load_locale("ru"),
+ "sv" => load_locale("sv-SE"),
"tr" => load_locale("tr"),
"uk" => load_locale("uk"),
"zh-CN" => load_locale("zh-CN"),
"zh-TW" => load_locale("zh-TW"),
}
-YT_POOL = QUICPool.new(YT_URL, capacity: CONFIG.pool_size, timeout: 0.05)
-YT_IMG_POOL = QUICPool.new(YT_IMG_URL, capacity: CONFIG.pool_size, timeout: 0.05)
+YT_POOL = QUICPool.new(YT_URL, capacity: CONFIG.pool_size, timeout: 0.1)
config = CONFIG
logger = Invidious::LogHandler.new
@@ -152,105 +159,47 @@ end
# Start jobs
-refresh_channels(PG_DB, logger, config)
-refresh_feeds(PG_DB, logger, config)
-subscribe_to_feeds(PG_DB, logger, HMAC_KEY, config)
+Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB, logger, config)
+Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB, logger, config)
+Invidious::Jobs.register Invidious::Jobs::SubscribeToFeedsJob.new(PG_DB, logger, config, HMAC_KEY)
+Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB)
+Invidious::Jobs.register Invidious::Jobs::UpdateDecryptFunctionJob.new
-statistics = {
- "error" => "Statistics are not availabile.",
-}
if config.statistics_enabled
- spawn do
- statistics = {
- "version" => "2.0",
- "software" => SOFTWARE,
- "openRegistrations" => config.registration_enabled,
- "usage" => {
- "users" => {
- "total" => PG_DB.query_one("SELECT count(*) FROM users", as: Int64),
- "activeHalfyear" => PG_DB.query_one("SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '6 months'", as: Int64),
- "activeMonth" => PG_DB.query_one("SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '1 month'", as: Int64),
- },
- },
- "metadata" => {
- "updatedAt" => Time.utc.to_unix,
- "lastChannelRefreshedAt" => PG_DB.query_one?("SELECT updated FROM channels ORDER BY updated DESC LIMIT 1", as: Time).try &.to_unix || 0_i64,
- },
- }
-
- loop do
- sleep 1.minute
- Fiber.yield
-
- statistics["usage"].as(Hash)["users"].as(Hash)["total"] = PG_DB.query_one("SELECT count(*) FROM users", as: Int64)
- statistics["usage"].as(Hash)["users"].as(Hash)["activeHalfyear"] = PG_DB.query_one("SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '6 months'", as: Int64)
- statistics["usage"].as(Hash)["users"].as(Hash)["activeMonth"] = PG_DB.query_one("SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '1 month'", as: Int64)
- statistics["metadata"].as(Hash(String, Int64))["updatedAt"] = Time.utc.to_unix
- statistics["metadata"].as(Hash(String, Int64))["lastChannelRefreshedAt"] = PG_DB.query_one?("SELECT updated FROM channels ORDER BY updated DESC LIMIT 1", as: Time).try &.to_unix || 0_i64
- end
- end
-end
-
-top_videos = [] of Video
-if config.top_enabled
- spawn do
- pull_top_videos(config, PG_DB) do |videos|
- top_videos = videos
- end
- end
+ Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, config, SOFTWARE)
end
-popular_videos = [] of ChannelVideo
-spawn do
- pull_popular_videos(PG_DB) do |videos|
- popular_videos = videos
- end
+if config.captcha_key
+ Invidious::Jobs.register Invidious::Jobs::BypassCaptchaJob.new(logger, config)
end
-decrypt_function = [] of {name: String, value: Int32}
-spawn do
- update_decrypt_function do |function|
- decrypt_function = function
- end
-end
+connection_channel = Channel({Bool, Channel(PQ::Notification)}).new(32)
+Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(connection_channel, PG_URL)
-if CONFIG.captcha_key
- spawn do
- bypass_captcha(CONFIG.captcha_key, logger) do |cookies|
- cookies.each do |cookie|
- config.cookies << cookie
- end
+Invidious::Jobs.start_all
- # Persist cookies between runs
- CONFIG.cookies = config.cookies
- File.write("config/config.yml", config.to_yaml)
- end
- end
+def popular_videos
+ Invidious::Jobs::PullPopularVideosJob::POPULAR_VIDEOS.get
end
-connection_channel = Channel({Bool, Channel(PQ::Notification)}).new(32)
-spawn do
- connections = [] of Channel(PQ::Notification)
-
- PG.connect_listen(PG_URL, "notifications") { |event| connections.each { |connection| connection.send(event) } }
-
- loop do
- action, connection = connection_channel.receive
+DECRYPT_FUNCTION = Invidious::Jobs::UpdateDecryptFunctionJob::DECRYPT_FUNCTION
- case action
- when true
- connections << connection
- when false
- connections.delete(connection)
- end
+before_all do |env|
+ preferences = begin
+ Preferences.from_json(env.request.cookies["PREFS"]?.try &.value || "{}")
+ rescue
+ Preferences.from_json("{}")
end
-end
-before_all do |env|
- host_url = make_host_url(config, Kemal.config)
+ env.set "preferences", preferences
env.response.headers["X-XSS-Protection"] = "1; mode=block"
env.response.headers["X-Content-Type-Options"] = "nosniff"
- env.response.headers["Content-Security-Policy"] = "default-src blob: data: 'self' #{host_url} 'unsafe-inline' 'unsafe-eval'; media-src blob: 'self' #{host_url} https://*.googlevideo.com:443"
+ extra_media_csp = ""
+ if CONFIG.disabled?("local") || !preferences.local
+ extra_media_csp += " https://*.googlevideo.com:443"
+ end
+ # TODO: Remove style-src's 'unsafe-inline', requires to remove all inline styles (<style> [..] </style>, style=" [..] ")
+ env.response.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' data:; connect-src 'self'; manifest-src 'self'; media-src 'self' blob:#{extra_media_csp}"
env.response.headers["Referrer-Policy"] = "same-origin"
if (Kemal.config.ssl || config.https_only) && config.hsts
@@ -268,12 +217,6 @@ before_all do |env|
"/latest_version",
}.any? { |r| env.request.resource.starts_with? r }
- begin
- preferences = Preferences.from_json(env.request.cookies["PREFS"]?.try &.value || "{}")
- rescue
- preferences = Preferences.from_json("{}")
- end
-
if env.request.cookies.has_key? "SID"
sid = env.request.cookies["SID"].value
@@ -295,6 +238,7 @@ before_all do |env|
}, HMAC_KEY, PG_DB, 1.week)
preferences = user.preferences
+ env.set "preferences", preferences
env.set "sid", sid
env.set "csrf_token", csrf_token
@@ -316,6 +260,7 @@ before_all do |env|
}, HMAC_KEY, PG_DB, 1.week)
preferences = user.preferences
+ env.set "preferences", preferences
env.set "sid", sid
env.set "csrf_token", csrf_token
@@ -333,7 +278,6 @@ before_all do |env|
preferences.dark_mode = dark_mode
preferences.thin_mode = thin_mode
preferences.locale = locale
- env.set "preferences", preferences
current_page = env.request.path
if env.request.query
@@ -349,473 +293,12 @@ before_all do |env|
env.set "current_page", URI.encode_www_form(current_page)
end
-get "/" do |env|
- preferences = env.get("preferences").as(Preferences)
- locale = LOCALES[preferences.locale]?
- user = env.get? "user"
-
- case preferences.default_home
- when ""
- templated "empty"
- when "Popular"
- templated "popular"
- when "Top"
- if config.top_enabled
- templated "top"
- else
- templated "empty"
- end
- when "Trending"
- env.redirect "/feed/trending"
- when "Subscriptions"
- if user
- env.redirect "/feed/subscriptions"
- else
- templated "popular"
- end
- when "Playlists"
- if user
- env.redirect "/view_all_playlists"
- else
- templated "popular"
- end
- end
-end
-
-get "/privacy" do |env|
- locale = LOCALES[env.get("preferences").as(Preferences).locale]?
- templated "privacy"
-end
-
-get "/licenses" do |env|
- locale = LOCALES[env.get("preferences").as(Preferences).locale]?
- rendered "licenses"
-end
-
-# Videos
-
-get "/watch" do |env|
- locale = LOCALES[env.get("preferences").as(Preferences).locale]?
- region = env.params.query["region"]?
-
- if env.params.query.to_s.includes?("%20") || env.params.query.to_s.includes?("+")
- url = "/watch?" + env.params.query.to_s.gsub("%20", "").delete("+")
- next env.redirect url
- end
-
- if env.params.query["v"]?
- id = env.params.query["v"]
-
- if env.params.query["v"].empty?
- error_message = "Invalid parameters."
- env.response.status_code = 400
- next templated "error"
- end
-
- if id.size > 11
- url = "/watch?v=#{id[0, 11]}"
- env.params.query.delete_all("v")
- if env.params.query.size > 0
- url += "&#{env.params.query}"
- end
-
- next env.redirect url
- end
- else
- next env.redirect "/"
- end
-
- plid = env.params.query["list"]?
- continuation = process_continuation(PG_DB, env.params.query, plid, id)
-
- nojs = env.params.query["nojs"]?
-
- nojs ||= "0"
- nojs = nojs == "1"
-
- preferences = env.get("preferences").as(Preferences)
-
- user = env.get?("user").try &.as(User)
- if user
- subscriptions = user.subscriptions
- watched = user.watched
- notifications = user.notifications
- end
- subscriptions ||= [] of String
-
- params = process_video_params(env.params.query, preferences)
- env.params.query.delete_all("listen")
-
- begin
- video = get_video(id, PG_DB, region: params.region)
- rescue ex : VideoRedirect
- next env.redirect env.request.resource.gsub(id, ex.video_id)
- rescue ex
- error_message = ex.message
- env.response.status_code = 500
- logger.puts("#{id} : #{ex.message}")
- next templated "error"
- end
-
- if preferences.annotations_subscribed &&
- subscriptions.includes?(video.ucid) &&
- (env.params.query["iv_load_policy"]? || "1") == "1"
- params.annotations = true
- end
- env.params.query.delete_all("iv_load_policy")
-
- if watched && !watched.includes? id
- PG_DB.exec("UPDATE users SET watched = watched || $1 WHERE email = $2", [id], user.as(User).email)
- end
-
- if notifications && notifications.includes? id
- PG_DB.exec("UPDATE users SET notifications = array_remove(notifications, $1) WHERE email = $2", id, user.as(User).email)
- env.get("user").as(User).notifications.delete(id)
- notifications.delete(id)
- end
-
- if nojs
- if preferences
- source = preferences.comments[0]
- if source.empty?
- source = preferences.comments[1]
- end
-
- if source == "youtube"
- begin
- comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
- rescue ex
- if preferences.comments[1] == "reddit"
- comments, reddit_thread = fetch_reddit_comments(id)
- comment_html = template_reddit_comments(comments, locale)
-
- comment_html = fill_links(comment_html, "https", "www.reddit.com")
- comment_html = replace_links(comment_html)
- end
- end
- elsif source == "reddit"
- begin
- comments, reddit_thread = fetch_reddit_comments(id)
- comment_html = template_reddit_comments(comments, locale)
-
- comment_html = fill_links(comment_html, "https", "www.reddit.com")
- comment_html = replace_links(comment_html)
- rescue ex
- if preferences.comments[1] == "youtube"
- comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
- end
- end
- end
- else
- comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
- end
-
- comment_html ||= ""
- end
-
- fmt_stream = video.fmt_stream(decrypt_function)
- adaptive_fmts = video.adaptive_fmts(decrypt_function)
-
- if params.local
- fmt_stream.each { |fmt| fmt["url"] = URI.parse(fmt["url"]).full_path }
- adaptive_fmts.each { |fmt| fmt["url"] = URI.parse(fmt["url"]).full_path }
- end
-
- video_streams = video.video_streams(adaptive_fmts)
- audio_streams = video.audio_streams(adaptive_fmts)
-
- # Older videos may not have audio sources available.
- # We redirect here so they're not unplayable
- if audio_streams.empty? && !video.live_now
- if params.quality == "dash"
- env.params.query.delete_all("quality")
- env.params.query["quality"] = "medium"
- next env.redirect "/watch?#{env.params.query}"
- elsif params.listen
- env.params.query.delete_all("listen")
- env.params.query["listen"] = "0"
- next env.redirect "/watch?#{env.params.query}"
- end
- end
-
- captions = video.captions
-
- preferred_captions = captions.select { |caption|
- params.preferred_captions.includes?(caption.name.simpleText) ||
- params.preferred_captions.includes?(caption.languageCode.split("-")[0])
- }
- preferred_captions.sort_by! { |caption|
- (params.preferred_captions.index(caption.name.simpleText) ||
- params.preferred_captions.index(caption.languageCode.split("-")[0])).not_nil!
- }
- captions = captions - preferred_captions
-
- aspect_ratio = "16:9"
-
- video.description_html = fill_links(video.description_html, "https", "www.youtube.com")
- video.description_html = replace_links(video.description_html)
-
- host_url = make_host_url(config, Kemal.config)
-
- if video.player_response["streamingData"]?.try &.["hlsManifestUrl"]?
- hlsvp = video.player_response["streamingData"]["hlsManifestUrl"].as_s
- hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", host_url)
- end
-
- thumbnail = "/vi/#{video.id}/maxres.jpg"
-
- if params.raw
- if params.listen
- url = audio_streams[0]["url"]
-
- audio_streams.each do |fmt|
- if fmt["bitrate"] == params.quality.rchop("k")
- url = fmt["url"]
- end
- end
- else
- url = fmt_stream[0]["url"]
-
- fmt_stream.each do |fmt|
- if fmt["label"].split(" - ")[0] == params.quality
- url = fmt["url"]
- end
- end
- end
-
- next env.redirect url
- end
-
- rvs = [] of Hash(String, String)
- video.info["rvs"]?.try &.split(",").each do |rv|
- rvs << HTTP::Params.parse(rv).to_h
- end
-
- rating = video.info["avg_rating"].to_f64
- if video.views > 0
- engagement = ((video.dislikes.to_f + video.likes.to_f)/video.views * 100)
- else
- engagement = 0
- end
-
- playability_status = video.player_response["playabilityStatus"]?
- if playability_status && playability_status["status"] == "LIVE_STREAM_OFFLINE" && !video.premiere_timestamp
- reason = playability_status["reason"]?.try &.as_s
- end
- reason ||= ""
-
- templated "watch"
-end
-
-get "/embed/" do |env|
- locale = LOCALES[env.get("preferences").as(Preferences).locale]?
-
- if plid = env.params.query["list"]?
- begin
- playlist = get_playlist(PG_DB, plid, locale: locale)
- offset = env.params.query["index"]?.try &.to_i? || 0
- videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale)
- rescue ex
- error_message = ex.message
- env.response.status_code = 500
- next templated "error"
- end
-
- url = "/embed/#{videos[0].id}?#{env.params.query}"
-
- if env.params.query.size > 0
- url += "?#{env.params.query}"
- end
- else
- url = "/"
- end
-
- env.redirect url
-end
-
-get "/embed/:id" do |env|
- locale = LOCALES[env.get("preferences").as(Preferences).locale]?
- id = env.params.url["id"]
-
- plid = env.params.query["list"]?
- continuation = process_continuation(PG_DB, env.params.query, plid, id)
-
- if md = env.params.query["playlist"]?
- .try &.match(/[a-zA-Z0-9_-]{11}(,[a-zA-Z0-9_-]{11})*/)
- video_series = md[0].split(",")
- env.params.query.delete("playlist")
- end
-
- preferences = env.get("preferences").as(Preferences)
-
- if id.includes?("%20") || id.includes?("+") || env.params.query.to_s.includes?("%20") || env.params.query.to_s.includes?("+")
- id = env.params.url["id"].gsub("%20", "").delete("+")
-
- url = "/embed/#{id}"
-
- if env.params.query.size > 0
- url += "?#{env.params.query.to_s.gsub("%20", "").delete("+")}"
- end
-
- next env.redirect url
- end
-
- # YouTube embed supports `videoseries` with either `list=PLID`
- # or `playlist=VIDEO_ID,VIDEO_ID`
- case id
- when "videoseries"
- url = ""
-
- if plid
- begin
- playlist = get_playlist(PG_DB, plid, locale: locale)
- offset = env.params.query["index"]?.try &.to_i? || 0
- videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale)
- rescue ex
- error_message = ex.message
- env.response.status_code = 500
- next templated "error"
- end
-
- url = "/embed/#{videos[0].id}"
- elsif video_series
- url = "/embed/#{video_series.shift}"
- env.params.query["playlist"] = video_series.join(",")
- else
- next env.redirect "/"
- end
-
- if env.params.query.size > 0
- url += "?#{env.params.query}"
- end
-
- next env.redirect url
- when "live_stream"
- response = YT_POOL.client &.get("/embed/live_stream?channel=#{env.params.query["channel"]? || ""}")
- video_id = response.body.match(/"video_id":"(?<video_id>[a-zA-Z0-9_-]{11})"/).try &.["video_id"]
-
- env.params.query.delete_all("channel")
-
- if !video_id || video_id == "live_stream"
- error_message = "Video is unavailable."
- next templated "error"
- end
-
- url = "/embed/#{video_id}"
-
- if env.params.query.size > 0
- url += "?#{env.params.query}"
- end
-
- next env.redirect url
- when id.size > 11
- url = "/embed/#{id[0, 11]}"
-
- if env.params.query.size > 0
- url += "?#{env.params.query}"
- end
-
- next env.redirect url
- end
-
- params = process_video_params(env.params.query, preferences)
-
- user = env.get?("user").try &.as(User)
- if user
- subscriptions = user.subscriptions
- watched = user.watched
- notifications = user.notifications
- end
- subscriptions ||= [] of String
-
- begin
- video = get_video(id, PG_DB, region: params.region)
- rescue ex : VideoRedirect
- next env.redirect env.request.resource.gsub(id, ex.video_id)
- rescue ex
- error_message = ex.message
- env.response.status_code = 500
- next templated "error"
- end
-
- if preferences.annotations_subscribed &&
- subscriptions.includes?(video.ucid) &&
- (env.params.query["iv_load_policy"]? || "1") == "1"
- params.annotations = true
- end
-
- # if watched && !watched.includes? id
- # PG_DB.exec("UPDATE users SET watched = watched || $1 WHERE email = $2", [id], user.as(User).email)
- # end
-
- if notifications && notifications.includes? id
- PG_DB.exec("UPDATE users SET notifications = array_remove(notifications, $1) WHERE email = $2", id, user.as(User).email)
- env.get("user").as(User).notifications.delete(id)
- notifications.delete(id)
- end
-
- fmt_stream = video.fmt_stream(decrypt_function)
- adaptive_fmts = video.adaptive_fmts(decrypt_function)
-
- if params.local
- fmt_stream.each { |fmt| fmt["url"] = URI.parse(fmt["url"]).full_path }
- adaptive_fmts.each { |fmt| fmt["url"] = URI.parse(fmt["url"]).full_path }
- end
-
- video_streams = video.video_streams(adaptive_fmts)
- audio_streams = video.audio_streams(adaptive_fmts)
-
- if audio_streams.empty? && !video.live_now
- if params.quality == "dash"
- env.params.query.delete_all("quality")
- next env.redirect "/embed/#{id}?#{env.params.query}"
- elsif params.listen
- env.params.query.delete_all("listen")
- env.params.query["listen"] = "0"
- next env.redirect "/embed/#{id}?#{env.params.query}"
- end
- end
-
- captions = video.captions
-
- preferred_captions = captions.select { |caption|
- params.preferred_captions.includes?(caption.name.simpleText) ||
- params.preferred_captions.includes?(caption.languageCode.split("-")[0])
- }
- preferred_captions.sort_by! { |caption|
- (params.preferred_captions.index(caption.name.simpleText) ||
- params.preferred_captions.index(caption.languageCode.split("-")[0])).not_nil!
- }
- captions = captions - preferred_captions
-
- aspect_ratio = nil
-
- video.description_html = fill_links(video.description_html, "https", "www.youtube.com")
- video.description_html = replace_links(video.description_html)
-
- host_url = make_host_url(config, Kemal.config)
-
- if video.player_response["streamingData"]?.try &.["hlsManifestUrl"]?
- hlsvp = video.player_response["streamingData"]["hlsManifestUrl"].as_s
- hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", host_url)
- end
-
- thumbnail = "/vi/#{video.id}/maxres.jpg"
-
- if params.raw
- url = fmt_stream[0]["url"]
-
- fmt_stream.each do |fmt|
- if fmt["label"].split(" - ")[0] == params.quality
- url = fmt["url"]
- end
- end
-
- next env.redirect url
- end
-
- rendered "embed"
-end
+Invidious::Routing.get "/", Invidious::Routes::Home
+Invidious::Routing.get "/privacy", Invidious::Routes::Privacy
+Invidious::Routing.get "/licenses", Invidious::Routes::Licenses
+Invidious::Routing.get "/watch", Invidious::Routes::Watch
+Invidious::Routing.get "/embed/", Invidious::Routes::Embed::Index
+Invidious::Routing.get "/embed/:id", Invidious::Routes::Embed::Show
# Playlists
@@ -835,8 +318,14 @@ get "/view_all_playlists" do |env|
user = user.as(User)
- items = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 ORDER BY created", user.email, as: InvidiousPlaylist)
- items.map! do |item|
+ items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist)
+ items_created.map! do |item|
+ item.author = ""
+ item
+ end
+
+ items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist)
+ items_saved.map! do |item|
item.author = ""
item
end
@@ -907,6 +396,25 @@ post "/create_playlist" do |env|
env.redirect "/playlist?list=#{playlist.id}"
end
+get "/subscribe_playlist" do |env|
+ locale = LOCALES[env.get("preferences").as(Preferences).locale]?
+
+ user = env.get? "user"
+ referer = get_referer(env)
+
+ if !user
+ next env.redirect "/"
+ end
+
+ user = user.as(User)
+
+ playlist_id = env.params.query["list"]
+ playlist = get_playlist(PG_DB, playlist_id, locale)
+ subscribe_playlist(PG_DB, user, playlist)
+
+ env.redirect "/playlist?list=#{playlist.id}"
+end
+
get "/delete_playlist" do |env|
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
@@ -922,10 +430,6 @@ get "/delete_playlist" do |env|
sid = sid.as(String)
plid = env.params.query["list"]?
- if !plid || !plid.starts_with?("IV")
- next env.redirect referer
- end
-
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
if !playlist || playlist.author != user.email
next env.redirect referer
@@ -1224,29 +728,33 @@ post "/playlist_ajax" do |env|
end
end
- playlist_video = PlaylistVideo.new(
- title: video.title,
- id: video.id,
- author: video.author,
- ucid: video.ucid,
+ playlist_video = PlaylistVideo.new({
+ title: video.title,
+ id: video.id,
+ author: video.author,
+ ucid: video.ucid,
length_seconds: video.length_seconds,
- published: video.published,
- plid: playlist_id,
- live_now: video.live_now,
- index: Random::Secure.rand(0_i64..Int64::MAX)
- )
+ published: video.published,
+ plid: playlist_id,
+ live_now: video.live_now,
+ index: Random::Secure.rand(0_i64..Int64::MAX),
+ })
video_array = playlist_video.to_a
args = arg_array(video_array)
PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array)
- PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = video_count + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, playlist_id)
+ PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, playlist_id)
when "action_remove_video"
index = env.params.query["set_video_id"]
PG_DB.exec("DELETE FROM playlist_videos * WHERE index = $1", index)
- PG_DB.exec("UPDATE playlists SET index = array_remove(index, $1), video_count = video_count - 1, updated = $2 WHERE id = $3", index, Time.utc, playlist_id)
+ PG_DB.exec("UPDATE playlists SET index = array_remove(index, $1), video_count = cardinality(index) - 1, updated = $2 WHERE id = $3", index, Time.utc, playlist_id)
when "action_move_video_before"
# TODO: Playlist stub
+ else
+ error_message = {"error" => "Unsupported action #{action}"}.to_json
+ env.response.status_code = 400
+ next error_message
end
if redirect
@@ -1261,9 +769,9 @@ get "/playlist" do |env|
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
user = env.get?("user").try &.as(User)
- plid = env.params.query["list"]?
referer = get_referer(env)
+ plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
if !plid
next env.redirect "/"
end
@@ -1330,16 +838,14 @@ get "/opensearch.xml" do |env|
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
env.response.content_type = "application/opensearchdescription+xml"
- host = make_host_url(config, Kemal.config)
-
XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("OpenSearchDescription", xmlns: "http://a9.com/-/spec/opensearch/1.1/") do
xml.element("ShortName") { xml.text "Invidious" }
xml.element("LongName") { xml.text "Invidious Search" }
xml.element("Description") { xml.text "Search for videos, channels, and playlists on Invidious" }
xml.element("InputEncoding") { xml.text "UTF-8" }
- xml.element("Image", width: 48, height: 48, type: "image/x-icon") { xml.text "#{host}/favicon.ico" }
- xml.element("Url", type: "text/html", method: "get", template: "#{host}/search?q={searchTerms}")
+ xml.element("Image", width: 48, height: 48, type: "image/x-icon") { xml.text "#{HOST_URL}/favicon.ico" }
+ xml.element("Url", type: "text/html", method: "get", template: "#{HOST_URL}/search?q={searchTerms}")
end
end
end
@@ -1448,7 +954,6 @@ post "/login" do |env|
traceback = IO::Memory.new
# See https://github.com/ytdl-org/youtube-dl/blob/2019.04.07/youtube_dl/extractor/youtube.py#L82
- # TODO: Convert to QUIC
begin
client = QUIC::Client.new(LOGIN_URL)
headers = HTTP::Headers.new
@@ -1541,7 +1046,7 @@ post "/login" do |env|
case prompt_type
when "TWO_STEP_VERIFICATION"
prompt_type = 2
- when "LOGIN_CHALLENGE"
+ else # "LOGIN_CHALLENGE"
prompt_type = 4
end
@@ -1834,7 +1339,7 @@ post "/login" do |env|
env.response.status_code = 400
next templated "error"
end
- when "text"
+ else # "text"
answer = Digest::MD5.hexdigest(answer.downcase.strip)
found_valid_captcha = false
@@ -1859,8 +1364,8 @@ post "/login" do |env|
sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
user, sid = create_user(sid, email, password)
user_array = user.to_a
+ user_array[4] = user_array[4].to_json # User preferences
- user_array[4] = user_array[4].to_json
args = arg_array(user_array)
PG_DB.exec("INSERT INTO users VALUES (#{args})", args: user_array)
@@ -2087,10 +1592,6 @@ post "/preferences" do |env|
end
config.default_user_preferences.feed_menu = admin_feed_menu
- top_enabled = env.params.body["top_enabled"]?.try &.as(String)
- top_enabled ||= "off"
- config.top_enabled = top_enabled == "on"
-
captcha_enabled = env.params.body["captcha_enabled"]?.try &.as(String)
captcha_enabled ||= "off"
config.captcha_enabled = captcha_enabled == "on"
@@ -2241,10 +1742,14 @@ post "/watch_ajax" do |env|
case action
when "action_mark_watched"
if !user.watched.includes? id
- PG_DB.exec("UPDATE users SET watched = watched || $1 WHERE email = $2", [id], user.email)
+ PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.email)
end
when "action_mark_unwatched"
PG_DB.exec("UPDATE users SET watched = array_remove(watched, $1) WHERE email = $2", id, user.email)
+ else
+ error_message = {"error" => "Unsupported action #{action}"}.to_json
+ env.response.status_code = 400
+ next error_message
end
if redirect
@@ -2308,8 +1813,7 @@ get "/modify_notifications" do |env|
end
headers = cookies.add_request_headers(headers)
- match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[A-Za-z0-9\_\-\=]+)"/)
- if match
+ if match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[^"]+)"/)
session_token = match["session_token"]
else
next env.redirect referer
@@ -2399,6 +1903,10 @@ post "/subscription_ajax" do |env|
end
when "action_remove_subscriptions"
PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_remove(subscriptions, $1) WHERE email = $2", channel_id, email)
+ else
+ error_message = {"error" => "Unsupported action #{action}"}.to_json
+ env.response.status_code = 400
+ next error_message
end
if redirect
@@ -2444,20 +1952,39 @@ get "/subscription_manager" do |env|
end
subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousChannel)
-
subscriptions.sort_by! { |channel| channel.author.downcase }
if action_takeout
- host_url = make_host_url(config, Kemal.config)
-
if format == "json"
env.response.content_type = "application/json"
env.response.headers["content-disposition"] = "attachment"
- next {
- "subscriptions" => user.subscriptions,
- "watch_history" => user.watched,
- "preferences" => user.preferences,
- }.to_json
+ playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist)
+
+ next JSON.build do |json|
+ json.object do
+ json.field "subscriptions", user.subscriptions
+ json.field "watch_history", user.watched
+ json.field "preferences", user.preferences
+ json.field "playlists" do
+ json.array do
+ playlists.each do |playlist|
+ json.object do
+ json.field "title", playlist.title
+ json.field "description", html_to_content(playlist.description_html)
+ json.field "privacy", playlist.privacy.to_s
+ json.field "videos" do
+ json.array do
+ PG_DB.query_all("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 500", playlist.id, playlist.index, as: String).each do |video_id|
+ json.string video_id
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
else
env.response.content_type = "application/xml"
env.response.headers["content-disposition"] = "attachment"
@@ -2475,7 +2002,7 @@ get "/subscription_manager" do |env|
if format == "newpipe"
xmlUrl = "https://www.youtube.com/feeds/videos.xml?channel_id=#{channel.id}"
else
- xmlUrl = "#{host_url}/feed/channel/#{channel.id}"
+ xmlUrl = "#{HOST_URL}/feed/channel/#{channel.id}"
end
xml.element("outline", text: channel.author, title: channel.author,
@@ -2517,42 +2044,13 @@ post "/data_control" do |env|
if user
user = user.as(User)
- spawn do
- # Since import can take a while, if we're not done after 20 seconds
- # push out content to prevent timeout.
-
- # Interesting to note is that Chrome will try to render before the content has finished loading,
- # which is why we include a loading icon. Firefox and its derivatives will not see this page,
- # instead redirecting immediately once the connection has closed.
-
- # https://stackoverflow.com/q/2091239 is helpful but not directly applicable here.
-
- sleep 20.seconds
- env.response.puts %(<meta http-equiv="refresh" content="0; url=#{referer}">)
- env.response.puts %(<link rel="stylesheet" href="/css/ionicons.min.css?v=#{ASSET_COMMIT}">)
- env.response.puts %(<link rel="stylesheet" href="/css/default.css?v=#{ASSET_COMMIT}">)
- if env.get("preferences").as(Preferences).dark_mode == "dark"
- env.response.puts %(<link rel="stylesheet" href="/css/darktheme.css?v=#{ASSET_COMMIT}">)
- else
- env.response.puts %(<link rel="stylesheet" href="/css/lighttheme.css?v=#{ASSET_COMMIT}">)
- end
- env.response.puts %(<h3><div class="loading"><i class="icon ion-ios-refresh"></i></div></h3>)
- env.response.flush
-
- loop do
- env.response.puts %(<!-- keepalive #{Time.utc.to_unix} -->)
- env.response.flush
-
- sleep (20 + rand(11)).seconds
- end
- end
+ # TODO: Find a way to prevent browser timeout
HTTP::FormData.parse(env.request) do |part|
body = part.body.gets_to_end
- if body.empty?
- next
- end
+ next if body.empty?
+ # TODO: Unify into single import based on content-type
case part.name
when "import_invidious"
body = JSON.parse(body)
@@ -2573,9 +2071,55 @@ post "/data_control" do |env|
end
if body["preferences"]?
- user.preferences = Preferences.from_json(body["preferences"].to_json, user.preferences)
+ user.preferences = Preferences.from_json(body["preferences"].to_json)
PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", user.preferences.to_json, user.email)
end
+
+ if playlists = body["playlists"]?.try &.as_a?
+ playlists.each do |item|
+ title = item["title"]?.try &.as_s?.try &.delete("<>")
+ description = item["description"]?.try &.as_s?.try &.delete("\r")
+ privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy }
+
+ next if !title
+ next if !description
+ next if !privacy
+
+ playlist = create_playlist(PG_DB, title, privacy, user)
+ PG_DB.exec("UPDATE playlists SET description = $1 WHERE id = $2", description, playlist.id)
+
+ videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|
+ raise "Playlist cannot have more than 500 videos" if idx > 500
+
+ video_id = video_id.try &.as_s?
+ next if !video_id
+
+ begin
+ video = get_video(video_id, PG_DB)
+ rescue ex
+ next
+ end
+
+ playlist_video = PlaylistVideo.new({
+ title: video.title,
+ id: video.id,
+ author: video.author,
+ ucid: video.ucid,
+ length_seconds: video.length_seconds,
+ published: video.published,
+ plid: playlist.id,
+ live_now: video.live_now,
+ index: Random::Secure.rand(0_i64..Int64::MAX),
+ })
+
+ video_array = playlist_video.to_a
+ args = arg_array(video_array)
+
+ PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array)
+ PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, playlist.id)
+ end
+ end
+ end
when "import_youtube"
subscriptions = XML.parse(body)
user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel|
@@ -2615,7 +2159,7 @@ post "/data_control" do |env|
PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
when "import_newpipe"
- Zip::Reader.open(IO::Memory.new(body)) do |file|
+ Compress::Zip::Reader.open(IO::Memory.new(body)) do |file|
file.each_entry do |entry|
if entry.filename == "newpipe.db"
tempfile = File.tempfile(".db")
@@ -2639,6 +2183,7 @@ post "/data_control" do |env|
end
end
end
+ else nil # Ignore
end
end
end
@@ -2980,6 +2525,10 @@ post "/token_ajax" do |env|
case action
when .starts_with? "action_revoke_token"
PG_DB.exec("DELETE FROM session_ids * WHERE id = $1 AND email = $2", session, user.email)
+ else
+ error_message = {"error" => "Unsupported action #{action}"}.to_json
+ env.response.status_code = 400
+ next error_message
end
if redirect
@@ -2994,12 +2543,7 @@ end
get "/feed/top" do |env|
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
-
- if config.top_enabled
- templated "top"
- else
- env.redirect "/"
- end
+ env.redirect "/"
end
get "/feed/popular" do |env|
@@ -3122,12 +2666,10 @@ get "/feed/channel/:ucid" do |env|
next error_message
end
- rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}").body
- rss = XML.parse_html(rss)
+ response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}")
+ rss = XML.parse_html(response.body)
- videos = [] of SearchVideo
-
- rss.xpath_nodes("//feed/entry").each do |entry|
+ videos = rss.xpath_nodes("//feed/entry").map do |entry|
video_id = entry.xpath_node("videoid").not_nil!.content
title = entry.xpath_node("title").not_nil!.content
@@ -3139,41 +2681,39 @@ get "/feed/channel/:ucid" do |env|
description_html = entry.xpath_node("group/description").not_nil!.to_s
views = entry.xpath_node("group/community/statistics").not_nil!.["views"].to_i64
- videos << SearchVideo.new(
- title: title,
- id: video_id,
- author: author,
- ucid: ucid,
- published: published,
- views: views,
- description_html: description_html,
- length_seconds: 0,
- live_now: false,
- paid: false,
- premium: false,
- premiere_timestamp: nil
- )
+ SearchVideo.new({
+ title: title,
+ id: video_id,
+ author: author,
+ ucid: ucid,
+ published: published,
+ views: views,
+ description_html: description_html,
+ length_seconds: 0,
+ live_now: false,
+ paid: false,
+ premium: false,
+ premiere_timestamp: nil,
+ })
end
- host_url = make_host_url(config, Kemal.config)
-
XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015",
"xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom",
"xml:lang": "en-US") do
- xml.element("link", rel: "self", href: "#{host_url}#{env.request.resource}")
+ xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}")
xml.element("id") { xml.text "yt:channel:#{channel.ucid}" }
xml.element("yt:channelId") { xml.text channel.ucid }
xml.element("title") { xml.text channel.author }
- xml.element("link", rel: "alternate", href: "#{host_url}/channel/#{channel.ucid}")
+ xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{channel.ucid}")
xml.element("author") do
xml.element("name") { xml.text channel.author }
- xml.element("uri") { xml.text "#{host_url}/channel/#{channel.ucid}" }
+ xml.element("uri") { xml.text "#{HOST_URL}/channel/#{channel.ucid}" }
end
videos.each do |video|
- video.to_xml(host_url, channel.auto_generated, params, xml)
+ video.to_xml(channel.auto_generated, params, xml)
end
end
end
@@ -3207,19 +2747,18 @@ get "/feed/private" do |env|
params = HTTP::Params.parse(env.params.query["params"]? || "")
videos, notifications = get_subscription_feed(PG_DB, user, max_results, page)
- host_url = make_host_url(config, Kemal.config)
XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015",
"xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom",
"xml:lang": "en-US") do
- xml.element("link", "type": "text/html", rel: "alternate", href: "#{host_url}/feed/subscriptions")
+ xml.element("link", "type": "text/html", rel: "alternate", href: "#{HOST_URL}/feed/subscriptions")
xml.element("link", "type": "application/atom+xml", rel: "self",
- href: "#{host_url}#{env.request.resource}")
+ href: "#{HOST_URL}#{env.request.resource}")
xml.element("title") { xml.text translate(locale, "Invidious Private Feed for `x`", user.email) }
(notifications + videos).each do |video|
- video.to_xml(locale, host_url, params, xml)
+ video.to_xml(locale, params, xml)
end
end
end
@@ -3233,8 +2772,6 @@ get "/feed/playlist/:plid" do |env|
plid = env.params.url["plid"]
params = HTTP::Params.parse(env.params.query["params"]? || "")
-
- host_url = make_host_url(config, Kemal.config)
path = env.request.path
if plid.starts_with? "IV"
@@ -3245,18 +2782,18 @@ get "/feed/playlist/:plid" do |env|
xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015",
"xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom",
"xml:lang": "en-US") do
- xml.element("link", rel: "self", href: "#{host_url}#{env.request.resource}")
+ xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}")
xml.element("id") { xml.text "iv:playlist:#{plid}" }
xml.element("iv:playlistId") { xml.text plid }
xml.element("title") { xml.text playlist.title }
- xml.element("link", rel: "alternate", href: "#{host_url}/playlist?list=#{plid}")
+ xml.element("link", rel: "alternate", href: "#{HOST_URL}/playlist?list=#{plid}")
xml.element("author") do
xml.element("name") { xml.text playlist.author }
end
videos.each do |video|
- video.to_xml(host_url, false, xml)
+ video.to_xml(false, xml)
end
end
end
@@ -3275,7 +2812,8 @@ get "/feed/playlist/:plid" do |env|
when "url", "href"
full_path = URI.parse(node[attribute.name]).full_path
query_string_opt = full_path.starts_with?("/watch?v=") ? "&#{params}" : ""
- node[attribute.name] = "#{host_url}#{full_path}#{query_string_opt}"
+ node[attribute.name] = "#{HOST_URL}#{full_path}#{query_string_opt}"
+ else nil # Skip
end
end
end
@@ -3283,7 +2821,7 @@ get "/feed/playlist/:plid" do |env|
document = document.to_xml(options: XML::SaveOptions::NO_DECL)
document.scan(/<uri>(?<url>[^<]+)<\/uri>/).each do |match|
- content = "#{host_url}#{URI.parse(match["url"]).full_path}"
+ content = "#{HOST_URL}#{URI.parse(match["url"]).full_path}"
document = document.gsub(match[0], "<uri>#{content}</uri>")
end
@@ -3386,39 +2924,26 @@ post "/feed/webhook/:token" do |env|
}.to_json
PG_DB.exec("NOTIFY notifications, E'#{payload}'")
- video = ChannelVideo.new(
- id: id,
- title: video.title,
- published: published,
- updated: updated,
- ucid: video.ucid,
- author: author,
- length_seconds: video.length_seconds,
- live_now: video.live_now,
+ video = ChannelVideo.new({
+ id: id,
+ title: video.title,
+ published: published,
+ updated: updated,
+ ucid: video.ucid,
+ author: author,
+ length_seconds: video.length_seconds,
+ live_now: video.live_now,
premiere_timestamp: video.premiere_timestamp,
- views: video.views,
- )
+ views: video.views,
+ })
- emails = PG_DB.query_all("UPDATE users SET notifications = notifications || $1 \
- WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications) RETURNING email",
- video.id, video.published, video.ucid, as: String)
-
- video_array = video.to_a
- args = arg_array(video_array)
-
- PG_DB.exec("INSERT INTO channel_videos VALUES (#{args}) \
+ was_insert = PG_DB.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \
ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \
updated = $4, ucid = $5, author = $6, length_seconds = $7, \
- live_now = $8, premiere_timestamp = $9, views = $10", args: video_array)
-
- # Update all users affected by insert
- if emails.empty?
- values = "'{}'"
- else
- values = "VALUES #{emails.map { |email| %((E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}')) }.join(",")}"
- end
+ live_now = $8, premiere_timestamp = $9, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool)
- PG_DB.exec("UPDATE users SET feed_needs_update = true WHERE email = ANY(#{values})")
+ PG_DB.exec("UPDATE users SET notifications = array_append(notifications, $1), \
+ feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert
end
end
@@ -3471,14 +2996,12 @@ get "/c/:user" do |env|
user = env.params.url["user"]
response = YT_POOL.client &.get("/c/#{user}")
- document = XML.parse_html(response.body)
+ html = XML.parse_html(response.body)
- anchor = document.xpath_node(%q(//a[contains(@class,"branded-page-header-title-link")]))
- if !anchor
- next env.redirect "/"
- end
+ ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1]
+ next env.redirect "/" if !ucid
- env.redirect anchor["href"]
+ env.redirect "/channel/#{ucid}"
end
# Legacy endpoint for /user/:username
@@ -3568,14 +3091,14 @@ get "/channel/:ucid" do |env|
item.author
end
end
- items = items.select { |item| item.is_a?(SearchPlaylist) }.map { |item| item.as(SearchPlaylist) }
+ items = items.select(&.is_a?(SearchPlaylist)).map(&.as(SearchPlaylist))
items.each { |item| item.author = "" }
else
sort_options = {"newest", "oldest", "popular"}
sort_by ||= "newest"
- items, count = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
- items.select! { |item| !item.paid }
+ count, items = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
+ items.reject! &.paid
env.set "search", "channel:#{channel.ucid} "
end
@@ -3670,7 +3193,7 @@ get "/channel/:ucid/community" do |env|
end
begin
- items = JSON.parse(fetch_channel_community(ucid, continuation, locale, config, Kemal.config, "json", thin_mode))
+ items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode))
rescue ex
env.response.status_code = 500
error_message = ex.message
@@ -3691,12 +3214,7 @@ get "/api/v1/stats" do |env|
next error_message
end
- if statistics["error"]?
- env.response.status_code = 500
- next statistics.to_json
- end
-
- statistics.to_json
+ Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json
end
# YouTube provides "storyboards", which are sprites containing x * y
@@ -3723,7 +3241,6 @@ get "/api/v1/storyboards/:id" do |env|
end
storyboards = video.storyboards
-
width = env.params.query["width"]?
height = env.params.query["height"]?
@@ -3731,7 +3248,7 @@ get "/api/v1/storyboards/:id" do |env|
response = JSON.build do |json|
json.object do
json.field "storyboards" do
- generate_storyboards(json, id, storyboards, config, Kemal.config)
+ generate_storyboards(json, id, storyboards)
end
end
end
@@ -3761,8 +3278,10 @@ get "/api/v1/storyboards/:id" do |env|
end_time = storyboard[:interval].milliseconds
storyboard[:storyboard_count].times do |i|
- host_url = make_host_url(config, Kemal.config)
- url = storyboard[:url].gsub("$M", i).gsub("https://i9.ytimg.com", host_url)
+ url = storyboard[:url]
+ authority = /(i\d?).ytimg.com/.match(url).not_nil![1]?
+ url = storyboard[:url].gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "")
+ url = "#{HOST_URL}/sb/#{authority}/#{url}"
storyboard[:storyboard_height].times do |j|
storyboard[:storyboard_width].times do |k|
@@ -4044,7 +3563,7 @@ get "/api/v1/annotations/:id" do |env|
cache_annotation(PG_DB, id, annotations)
end
- when "youtube"
+ else # "youtube"
response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}")
if response.status_code != 200
@@ -4085,7 +3604,7 @@ get "/api/v1/videos/:id" do |env|
next error_message
end
- video.to_json(locale, config, Kemal.config, decrypt_function)
+ video.to_json(locale)
end
get "/api/v1/trending" do |env|
@@ -4107,7 +3626,7 @@ get "/api/v1/trending" do |env|
videos = JSON.build do |json|
json.array do
trending.each do |video|
- video.to_json(locale, config, Kemal.config, json)
+ video.to_json(locale, json)
end
end
end
@@ -4123,7 +3642,7 @@ get "/api/v1/popular" do |env|
JSON.build do |json|
json.array do
popular_videos.each do |video|
- video.to_json(locale, config, Kemal.config, json)
+ video.to_json(locale, json)
end
end
end
@@ -4133,41 +3652,7 @@ get "/api/v1/top" do |env|
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
env.response.content_type = "application/json"
-
- if !config.top_enabled
- error_message = {"error" => "Administrator has disabled this endpoint."}.to_json
- env.response.status_code = 400
- next error_message
- end
-
- JSON.build do |json|
- json.array do
- top_videos.each do |video|
- # Top videos have much more information than provided below (adaptiveFormats, etc)
- # but can be very out of date, so we only provide a subset here
-
- json.object do
- json.field "title", video.title
- json.field "videoId", video.id
- json.field "videoThumbnails" do
- generate_thumbnails(json, video.id, config, Kemal.config)
- end
-
- json.field "lengthSeconds", video.length_seconds
- json.field "viewCount", video.views
-
- json.field "author", video.author
- json.field "authorId", video.ucid
- json.field "authorUrl", "/channel/#{video.ucid}"
- json.field "published", video.published.to_unix
- json.field "publishedText", translate(locale, "`x` ago", recode_date(video.published, locale))
-
- json.field "description", html_to_content(video.description_html)
- json.field "descriptionHtml", video.description_html
- end
- end
- end
- end
+ "[]"
end
get "/api/v1/channels/:ucid" do |env|
@@ -4198,7 +3683,7 @@ get "/api/v1/channels/:ucid" do |env|
count = 0
else
begin
- videos, count = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
+ count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
rescue ex
error_message = {"error" => ex.message}.to_json
env.response.status_code = 500
@@ -4244,7 +3729,7 @@ get "/api/v1/channels/:ucid" do |env|
qualities.each do |quality|
json.object do
- json.field "url", channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
+ json.field "url", channel.author_thumbnail.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
@@ -4267,7 +3752,7 @@ get "/api/v1/channels/:ucid" do |env|
json.field "latestVideos" do
json.array do
videos.each do |video|
- video.to_json(locale, config, Kemal.config, json)
+ video.to_json(locale, json)
end
end
end
@@ -4328,7 +3813,7 @@ end
end
begin
- videos, count = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
+ count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
rescue ex
error_message = {"error" => ex.message}.to_json
env.response.status_code = 500
@@ -4338,7 +3823,7 @@ end
JSON.build do |json|
json.array do
videos.each do |video|
- video.to_json(locale, config, Kemal.config, json)
+ video.to_json(locale, json)
end
end
end
@@ -4364,7 +3849,7 @@ end
JSON.build do |json|
json.array do
videos.each do |video|
- video.to_json(locale, config, Kemal.config, json)
+ video.to_json(locale, json)
end
end
end
@@ -4379,9 +3864,9 @@ end
ucid = env.params.url["ucid"]
continuation = env.params.query["continuation"]?
- sort_by = env.params.query["sort"]?.try &.downcase
- sort_by ||= env.params.query["sort_by"]?.try &.downcase
- sort_by ||= "last"
+ sort_by = env.params.query["sort"]?.try &.downcase ||
+ env.params.query["sort_by"]?.try &.downcase ||
+ "last"
begin
channel = get_about_info(ucid, locale)
@@ -4403,9 +3888,7 @@ end
json.field "playlists" do
json.array do
items.each do |item|
- if item.is_a?(SearchPlaylist)
- item.to_json(locale, config, Kemal.config, json)
- end
+ item.to_json(locale, json) if item.is_a?(SearchPlaylist)
end
end
end
@@ -4434,7 +3917,7 @@ end
# sort_by = env.params.query["sort_by"]?.try &.downcase
begin
- fetch_channel_community(ucid, continuation, locale, config, Kemal.config, format, thin_mode)
+ fetch_channel_community(ucid, continuation, locale, format, thin_mode)
rescue ex
env.response.status_code = 400
error_message = {"error" => ex.message}.to_json
@@ -4460,7 +3943,7 @@ get "/api/v1/channels/search/:ucid" do |env|
JSON.build do |json|
json.array do
search_results.each do |item|
- item.to_json(locale, config, Kemal.config, json)
+ item.to_json(locale, json)
end
end
end
@@ -4505,7 +3988,7 @@ get "/api/v1/search" do |env|
JSON.build do |json|
json.array do
search_results.each do |item|
- item.to_json(locale, config, Kemal.config, json)
+ item.to_json(locale, json)
end
end
end
@@ -4521,10 +4004,8 @@ get "/api/v1/search/suggestions" do |env|
query ||= ""
begin
- client = QUIC::Client.new("suggestqueries.google.com")
- client.family = CONFIG.force_resolve || Socket::Family::INET
- client.family = Socket::Family::INET if client.family == Socket::Family::UNSPEC
- response = client.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback").body
+ headers = HTTP::Headers{":authority" => "suggestqueries.google.com"}
+ response = YT_POOL.client &.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback", headers).body
body = response[35..-2]
body = JSON.parse(body).as_a
@@ -4584,7 +4065,7 @@ end
next error_message
end
- response = playlist.to_json(offset, locale, config, Kemal.config, continuation: continuation)
+ response = playlist.to_json(offset, locale, continuation: continuation)
if format == "html"
response = JSON.parse(response)
@@ -4648,7 +4129,7 @@ get "/api/v1/mixes/:rdid" do |env|
json.field "videoThumbnails" do
json.array do
- generate_thumbnails(json, video.id, config, Kemal.config)
+ generate_thumbnails(json, video.id)
end
end
@@ -4683,7 +4164,7 @@ get "/api/v1/auth/notifications" do |env|
topics = env.params.query["topics"]?.try &.split(",").uniq.first(1000)
topics ||= [] of String
- create_notification_stream(env, config, Kemal.config, decrypt_function, topics, connection_channel)
+ create_notification_stream(env, topics, connection_channel)
end
post "/api/v1/auth/notifications" do |env|
@@ -4692,7 +4173,7 @@ post "/api/v1/auth/notifications" do |env|
topics = env.params.body["topics"]?.try &.split(",").uniq.first(1000)
topics ||= [] of String
- create_notification_stream(env, config, Kemal.config, decrypt_function, topics, connection_channel)
+ create_notification_stream(env, topics, connection_channel)
end
get "/api/v1/auth/preferences" do |env|
@@ -4706,7 +4187,7 @@ post "/api/v1/auth/preferences" do |env|
user = env.get("user").as(User)
begin
- preferences = Preferences.from_json(env.request.body || "{}", user.preferences)
+ preferences = Preferences.from_json(env.request.body || "{}")
rescue
preferences = user.preferences
end
@@ -4736,7 +4217,7 @@ get "/api/v1/auth/feed" do |env|
json.field "notifications" do
json.array do
notifications.each do |video|
- video.to_json(locale, config, Kemal.config, json)
+ video.to_json(locale, json)
end
end
end
@@ -4744,7 +4225,7 @@ get "/api/v1/auth/feed" do |env|
json.field "videos" do
json.array do
videos.each do |video|
- video.to_json(locale, config, Kemal.config, json)
+ video.to_json(locale, json)
end
end
end
@@ -4816,7 +4297,7 @@ get "/api/v1/auth/playlists" do |env|
JSON.build do |json|
json.array do
playlists.each do |playlist|
- playlist.to_json(0, locale, config, Kemal.config, json)
+ playlist.to_json(0, locale, json)
end
end
end
@@ -4847,10 +4328,8 @@ post "/api/v1/auth/playlists" do |env|
next error_message
end
- host_url = make_host_url(config, Kemal.config)
-
playlist = create_playlist(PG_DB, title, privacy, user)
- env.response.headers["Location"] = "#{host_url}/api/v1/auth/playlists/#{playlist.id}"
+ env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{playlist.id}"
env.response.status_code = 201
{
"title" => title,
@@ -4962,29 +4441,27 @@ post "/api/v1/auth/playlists/:plid/videos" do |env|
next error_message
end
- playlist_video = PlaylistVideo.new(
- title: video.title,
- id: video.id,
- author: video.author,
- ucid: video.ucid,
+ playlist_video = PlaylistVideo.new({
+ title: video.title,
+ id: video.id,
+ author: video.author,
+ ucid: video.ucid,
length_seconds: video.length_seconds,
- published: video.published,
- plid: plid,
- live_now: video.live_now,
- index: Random::Secure.rand(0_i64..Int64::MAX)
- )
+ published: video.published,
+ plid: plid,
+ live_now: video.live_now,
+ index: Random::Secure.rand(0_i64..Int64::MAX),
+ })
video_array = playlist_video.to_a
args = arg_array(video_array)
PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array)
- PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = video_count + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, plid)
+ PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, plid)
- host_url = make_host_url(config, Kemal.config)
-
- env.response.headers["Location"] = "#{host_url}/api/v1/auth/playlists/#{plid}/videos/#{playlist_video.index.to_u64.to_s(16).upcase}"
+ env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{plid}/videos/#{playlist_video.index.to_u64.to_s(16).upcase}"
env.response.status_code = 201
- playlist_video.to_json(locale, config, Kemal.config, index: playlist.index.size)
+ playlist_video.to_json(locale, index: playlist.index.size)
end
delete "/api/v1/auth/playlists/:plid/videos/:index" do |env|
@@ -5014,7 +4491,7 @@ delete "/api/v1/auth/playlists/:plid/videos/:index" do |env|
end
PG_DB.exec("DELETE FROM playlist_videos * WHERE index = $1", index)
- PG_DB.exec("UPDATE playlists SET index = array_remove(index, $1), video_count = video_count - 1, updated = $2 WHERE id = $3", index, Time.utc, plid)
+ PG_DB.exec("UPDATE playlists SET index = array_remove(index, $1), video_count = cardinality(index) - 1, updated = $2 WHERE id = $3", index, Time.utc, plid)
env.response.status_code = 204
end
@@ -5162,7 +4639,7 @@ get "/api/manifest/dash/id/:id" do |env|
next
end
- if dashmpd = video.player_response["streamingData"]?.try &.["dashManifestUrl"]?.try &.as_s
+ if dashmpd = video.dash_manifest_url
manifest = YT_POOL.client &.get(URI.parse(dashmpd).full_path).body
manifest = manifest.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl|
@@ -5179,16 +4656,16 @@ get "/api/manifest/dash/id/:id" do |env|
next manifest
end
- adaptive_fmts = video.adaptive_fmts(decrypt_function)
+ adaptive_fmts = video.adaptive_fmts
if local
adaptive_fmts.each do |fmt|
- fmt["url"] = URI.parse(fmt["url"]).full_path
+ fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path)
end
end
- audio_streams = video.audio_streams(adaptive_fmts)
- video_streams = video.video_streams(adaptive_fmts).sort_by { |stream| {stream["size"].split("x")[0].to_i, stream["fps"].to_i} }.reverse
+ audio_streams = video.audio_streams
+ video_streams = video.video_streams.sort_by { |stream| {stream["width"].as_i, stream["fps"].as_i} }.reverse
XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("MPD", "xmlns": "urn:mpeg:dash:schema:mpd:2011",
@@ -5198,24 +4675,22 @@ get "/api/manifest/dash/id/:id" do |env|
i = 0
{"audio/mp4", "audio/webm"}.each do |mime_type|
- mime_streams = audio_streams.select { |stream| stream["type"].starts_with? mime_type }
- if mime_streams.empty?
- next
- end
+ mime_streams = audio_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type }
+ next if mime_streams.empty?
xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true) do
mime_streams.each do |fmt|
- codecs = fmt["type"].split("codecs=")[1].strip('"')
- bandwidth = fmt["bitrate"].to_i * 1000
- itag = fmt["itag"]
- url = fmt["url"]
+ codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"')
+ bandwidth = fmt["bitrate"].as_i
+ itag = fmt["itag"].as_i
+ url = fmt["url"].as_s
xml.element("Representation", id: fmt["itag"], codecs: codecs, bandwidth: bandwidth) do
xml.element("AudioChannelConfiguration", schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011",
value: "2")
xml.element("BaseURL") { xml.text url }
- xml.element("SegmentBase", indexRange: fmt["index"]) do
- xml.element("Initialization", range: fmt["init"])
+ xml.element("SegmentBase", indexRange: "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}") do
+ xml.element("Initialization", range: "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}")
end
end
end
@@ -5224,21 +4699,24 @@ get "/api/manifest/dash/id/:id" do |env|
i += 1
end
+ potential_heights = {4320, 2160, 1440, 1080, 720, 480, 360, 240, 144}
+
{"video/mp4", "video/webm"}.each do |mime_type|
- mime_streams = video_streams.select { |stream| stream["type"].starts_with? mime_type }
+ mime_streams = video_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type }
next if mime_streams.empty?
heights = [] of Int32
xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, scanType: "progressive") do
mime_streams.each do |fmt|
- codecs = fmt["type"].split("codecs=")[1].strip('"')
- bandwidth = fmt["bitrate"]
- itag = fmt["itag"]
- url = fmt["url"]
- width, height = fmt["size"].split("x").map { |i| i.to_i }
+ codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"')
+ bandwidth = fmt["bitrate"].as_i
+ itag = fmt["itag"].as_i
+ url = fmt["url"].as_s
+ width = fmt["width"].as_i
+ height = fmt["height"].as_i
# Resolutions reported by YouTube player (may not accurately reflect source)
- height = [4320, 2160, 1440, 1080, 720, 480, 360, 240, 144].sort_by { |i| (height - i).abs }[0]
+ height = potential_heights.min_by { |i| (height - i).abs }
next if unique_res && heights.includes? height
heights << height
@@ -5246,8 +4724,8 @@ get "/api/manifest/dash/id/:id" do |env|
startWithSAP: "1", maxPlayoutRate: "1",
bandwidth: bandwidth, frameRate: fmt["fps"]) do
xml.element("BaseURL") { xml.text url }
- xml.element("SegmentBase", indexRange: fmt["index"]) do
- xml.element("Initialization", range: fmt["init"])
+ xml.element("SegmentBase", indexRange: "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}") do
+ xml.element("Initialization", range: "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}")
end
end
end
@@ -5261,10 +4739,10 @@ get "/api/manifest/dash/id/:id" do |env|
end
get "/api/manifest/hls_variant/*" do |env|
- manifest = YT_POOL.client &.get(env.request.path)
+ response = YT_POOL.client &.get(env.request.path)
- if manifest.status_code != 200
- env.response.status_code = manifest.status_code
+ if response.status_code != 200
+ env.response.status_code = response.status_code
next
end
@@ -5273,12 +4751,10 @@ get "/api/manifest/hls_variant/*" do |env|
env.response.content_type = "application/x-mpegURL"
env.response.headers.add("Access-Control-Allow-Origin", "*")
- host_url = make_host_url(config, Kemal.config)
-
- manifest = manifest.body
+ manifest = response.body
if local
- manifest = manifest.gsub("https://www.youtube.com", host_url)
+ manifest = manifest.gsub("https://www.youtube.com", HOST_URL)
manifest = manifest.gsub("index.m3u8", "index.m3u8?local=true")
end
@@ -5286,10 +4762,10 @@ get "/api/manifest/hls_variant/*" do |env|
end
get "/api/manifest/hls_playlist/*" do |env|
- manifest = YT_POOL.client &.get(env.request.path)
+ response = YT_POOL.client &.get(env.request.path)
- if manifest.status_code != 200
- env.response.status_code = manifest.status_code
+ if response.status_code != 200
+ env.response.status_code = response.status_code
next
end
@@ -5298,9 +4774,7 @@ get "/api/manifest/hls_playlist/*" do |env|
env.response.content_type = "application/x-mpegURL"
env.response.headers.add("Access-Control-Allow-Origin", "*")
- host_url = make_host_url(config, Kemal.config)
-
- manifest = manifest.body
+ manifest = response.body
if local
manifest = manifest.gsub(/^https:\/\/r\d---.{11}\.c\.youtube\.com[^\n]*/m) do |match|
@@ -5335,7 +4809,7 @@ get "/api/manifest/hls_playlist/*" do |env|
raw_params["local"] = "true"
- "#{host_url}/videoplayback?#{raw_params}"
+ "#{HOST_URL}/videoplayback?#{raw_params}"
end
end
@@ -5361,7 +4835,7 @@ get "/latest_version" do |env|
end
id ||= env.params.query["id"]?
- itag ||= env.params.query["itag"]?
+ itag ||= env.params.query["itag"]?.try &.to_i
region = env.params.query["region"]?
@@ -5376,26 +4850,16 @@ get "/latest_version" do |env|
video = get_video(id, PG_DB, region: region)
- fmt_stream = video.fmt_stream(decrypt_function)
- adaptive_fmts = video.adaptive_fmts(decrypt_function)
+ fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag }
+ url = fmt.try &.["url"]?.try &.as_s
- urls = (fmt_stream + adaptive_fmts).select { |fmt| fmt["itag"] == itag }
- if urls.empty?
+ if !url
env.response.status_code = 404
next
- elsif urls.size > 1
- env.response.status_code = 409
- next
- end
-
- url = urls[0]["url"]
- if local
- url = URI.parse(url).full_path.not_nil!
end
- if title
- url += "&title=#{title}"
- end
+ url = URI.parse(url).full_path.not_nil! if local
+ url = "#{url}&title=#{title}" if title
env.redirect url
end
@@ -5488,8 +4952,8 @@ get "/videoplayback" do |env|
end
client = make_client(URI.parse(host), region)
-
response = HTTP::Client::Response.new(500)
+ error = ""
5.times do
begin
response = client.head(url, headers)
@@ -5514,12 +4978,14 @@ get "/videoplayback" do |env|
host = "https://r#{fvip}---#{mn}.googlevideo.com"
client = make_client(URI.parse(host), region)
rescue ex
+ error = ex.message
end
end
if response.status_code >= 400
env.response.status_code = response.status_code
- next
+ env.response.content_type = "text/plain"
+ next error
end
if url.includes? "&file=seg.ts"
@@ -5650,11 +5116,9 @@ get "/videoplayback" do |env|
end
get "/ggpht/*" do |env|
- host = "https://yt3.ggpht.com"
- client = make_client(URI.parse(host))
url = env.request.path.lchop("/ggpht")
- headers = HTTP::Headers.new
+ headers = HTTP::Headers{":authority" => "yt3.ggpht.com"}
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
@@ -5662,7 +5126,7 @@ get "/ggpht/*" do |env|
end
begin
- client.get(url, headers) do |response|
+ YT_POOL.client &.get(url, headers) do |response|
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
@@ -5683,28 +5147,24 @@ get "/ggpht/*" do |env|
end
end
-options "/sb/:id/:storyboard/:index" do |env|
- env.response.headers.delete("Content-Type")
+options "/sb/:authority/:id/:storyboard/:index" do |env|
env.response.headers["Access-Control-Allow-Origin"] = "*"
env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS"
env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range"
end
-get "/sb/:id/:storyboard/:index" do |env|
+get "/sb/:authority/:id/:storyboard/:index" do |env|
+ authority = env.params.url["authority"]
id = env.params.url["id"]
storyboard = env.params.url["storyboard"]
index = env.params.url["index"]
- if storyboard.starts_with? "storyboard_live"
- host = "https://i.ytimg.com"
- else
- host = "https://i9.ytimg.com"
- end
- client = make_client(URI.parse(host))
-
url = "/sb/#{id}/#{storyboard}/#{index}?#{env.params.query}"
headers = HTTP::Headers.new
+
+ headers[":authority"] = "#{authority}.ytimg.com"
+
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
@@ -5712,7 +5172,7 @@ get "/sb/:id/:storyboard/:index" do |env|
end
begin
- client.get(url, headers) do |response|
+ YT_POOL.client &.get(url, headers) do |response|
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
@@ -5720,6 +5180,7 @@ get "/sb/:id/:storyboard/:index" do |env|
end
end
+ env.response.headers["Connection"] = "close"
env.response.headers["Access-Control-Allow-Origin"] = "*"
if response.status_code >= 300
@@ -5737,11 +5198,9 @@ get "/s_p/:id/:name" do |env|
id = env.params.url["id"]
name = env.params.url["name"]
- host = "https://i9.ytimg.com"
- client = make_client(URI.parse(host))
url = env.request.resource
- headers = HTTP::Headers.new
+ headers = HTTP::Headers{":authority" => "i9.ytimg.com"}
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
@@ -5749,7 +5208,7 @@ get "/s_p/:id/:name" do |env|
end
begin
- client.get(url, headers) do |response|
+ YT_POOL.client &.get(url, headers) do |response|
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
@@ -5804,9 +5263,11 @@ get "/vi/:id/:name" do |env|
id = env.params.url["id"]
name = env.params.url["name"]
+ headers = HTTP::Headers{":authority" => "i.ytimg.com"}
+
if name == "maxres.jpg"
- build_thumbnails(id, config, Kemal.config).each do |thumb|
- if YT_IMG_POOL.client &.head("/vi/#{id}/#{thumb[:url]}.jpg").status_code == 200
+ build_thumbnails(id).each do |thumb|
+ if YT_POOL.client &.head("/vi/#{id}/#{thumb[:url]}.jpg", headers).status_code == 200
name = thumb[:url] + ".jpg"
break
end
@@ -5814,7 +5275,6 @@ get "/vi/:id/:name" do |env|
end
url = "/vi/#{id}/#{name}"
- headers = HTTP::Headers.new
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
@@ -5822,7 +5282,7 @@ get "/vi/:id/:name" do |env|
end
begin
- YT_IMG_POOL.client &.get(url, headers) do |response|
+ YT_POOL.client &.get(url, headers) do |response|
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
@@ -5844,8 +5304,8 @@ get "/vi/:id/:name" do |env|
end
get "/Captcha" do |env|
- client = make_client(LOGIN_URL)
- response = client.get(env.request.resource)
+ headers = HTTP::Headers{":authority" => "accounts.google.com"}
+ response = YT_POOL.client &.get(env.request.resource, headers)
env.response.headers["Content-Type"] = response.headers["Content-Type"]
response.body
end
@@ -5910,7 +5370,7 @@ end
error 500 do |env|
error_message = <<-END_HTML
Looks like you've found a bug in Invidious. Feel free to open a new issue
- <a href="https://github.com/omarroth/invidious/issues">here</a>
+ <a href="https://github.com/iv-org/invidious/issues">here</a>
or send an email to
<a href="mailto:#{CONFIG.admin_email}">#{CONFIG.admin_email}</a>.
END_HTML
diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr
index 7cd1bef1..656b9953 100644
--- a/src/invidious/channels.cr
+++ b/src/invidious/channels.cr
@@ -1,22 +1,35 @@
struct InvidiousChannel
- db_mapping({
- id: String,
- author: String,
- updated: Time,
- deleted: Bool,
- subscribed: Time?,
- })
+ include DB::Serializable
+
+ property id : String
+ property author : String
+ property updated : Time
+ property deleted : Bool
+ property subscribed : Time?
end
struct ChannelVideo
- def to_json(locale, config, kemal_config, json : JSON::Builder)
+ include DB::Serializable
+
+ property id : String
+ property title : String
+ property published : Time
+ property updated : Time
+ property ucid : String
+ property author : String
+ property length_seconds : Int32 = 0
+ property live_now : Bool = false
+ property premiere_timestamp : Time? = nil
+ property views : Int64? = nil
+
+ def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "shortVideo"
json.field "title", self.title
json.field "videoId", self.id
json.field "videoThumbnails" do
- generate_thumbnails(json, self.id, config, Kemal.config)
+ generate_thumbnails(json, self.id)
end
json.field "lengthSeconds", self.length_seconds
@@ -31,17 +44,17 @@ struct ChannelVideo
end
end
- def to_json(locale, config, kemal_config, json : JSON::Builder | Nil = nil)
+ def to_json(locale, json : JSON::Builder | Nil = nil)
if json
- to_json(locale, config, kemal_config, json)
+ to_json(locale, json)
else
JSON.build do |json|
- to_json(locale, config, kemal_config, json)
+ to_json(locale, json)
end
end
end
- def to_xml(locale, host_url, query_params, xml : XML::Builder)
+ def to_xml(locale, query_params, xml : XML::Builder)
query_params["v"] = self.id
xml.element("entry") do
@@ -49,17 +62,17 @@ struct ChannelVideo
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
- xml.element("link", rel: "alternate", href: "#{host_url}/watch?#{query_params}")
+ xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}")
xml.element("author") do
xml.element("name") { xml.text self.author }
- xml.element("uri") { xml.text "#{host_url}/channel/#{self.ucid}" }
+ xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
- xml.element("a", href: "#{host_url}/watch?#{query_params}") do
- xml.element("img", src: "#{host_url}/vi/#{self.id}/mqdefault.jpg")
+ xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do
+ xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
end
end
@@ -69,64 +82,59 @@ struct ChannelVideo
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
- xml.element("media:thumbnail", url: "#{host_url}/vi/#{self.id}/mqdefault.jpg",
+ xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
end
end
end
- def to_xml(locale, config, kemal_config, xml : XML::Builder | Nil = nil)
+ def to_xml(locale, xml : XML::Builder | Nil = nil)
if xml
- to_xml(locale, config, kemal_config, xml)
+ to_xml(locale, xml)
else
XML.build do |xml|
- to_xml(locale, config, kemal_config, xml)
+ to_xml(locale, xml)
end
end
end
- db_mapping({
- id: String,
- title: String,
- published: Time,
- updated: Time,
- ucid: String,
- author: String,
- length_seconds: {type: Int32, default: 0},
- live_now: {type: Bool, default: false},
- premiere_timestamp: {type: Time?, default: nil},
- views: {type: Int64?, default: nil},
- })
+ def to_tuple
+ {% begin %}
+ {
+ {{*@type.instance_vars.map { |var| var.name }}}
+ }
+ {% end %}
+ end
end
struct AboutRelatedChannel
- db_mapping({
- ucid: String,
- author: String,
- author_url: String,
- author_thumbnail: String,
- })
+ include DB::Serializable
+
+ property ucid : String
+ property author : String
+ property author_url : String
+ property author_thumbnail : String
end
# TODO: Refactor into either SearchChannel or InvidiousChannel
struct AboutChannel
- db_mapping({
- ucid: String,
- author: String,
- auto_generated: Bool,
- author_url: String,
- author_thumbnail: String,
- banner: String?,
- description_html: String,
- paid: Bool,
- total_views: Int64,
- sub_count: Int32,
- joined: Time,
- is_family_friendly: Bool,
- allowed_regions: Array(String),
- related_channels: Array(AboutRelatedChannel),
- tabs: Array(String),
- })
+ include DB::Serializable
+
+ property ucid : String
+ property author : String
+ property auto_generated : Bool
+ property author_url : String
+ property author_thumbnail : String
+ property banner : String?
+ property description_html : String
+ property paid : Bool
+ property total_views : Int64
+ property sub_count : Int32
+ property joined : Time
+ property is_family_friendly : Bool
+ property allowed_regions : Array(String)
+ property related_channels : Array(AboutRelatedChannel)
+ property tabs : Array(String)
end
class ChannelRedirect < Exception
@@ -213,33 +221,20 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
page = 1
- url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated)
- response = YT_POOL.client &.get(url)
+ response = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
+ videos = [] of SearchVideo
begin
- json = JSON.parse(response.body)
+ initial_data = JSON.parse(response.body).as_a.find &.["response"]?
+ raise "Could not extract JSON" if !initial_data
+ videos = extract_videos(initial_data.as_h, author, ucid)
rescue ex
if response.body.includes?("To continue with your YouTube experience, please fill out the form below.") ||
response.body.includes?("https://www.google.com/sorry/index")
raise "Could not extract channel info. Instance is likely blocked."
end
-
- raise "Could not extract JSON"
end
- if json["content_html"]? && !json["content_html"].as_s.empty?
- document = XML.parse_html(json["content_html"].as_s)
- nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
-
- if auto_generated
- videos = extract_videos(nodeset)
- else
- videos = extract_videos(nodeset, ucid, author)
- end
- end
-
- videos ||= [] of ChannelVideo
-
rss.xpath_nodes("//feed/entry").each do |entry|
video_id = entry.xpath_node("videoid").not_nil!.content
title = entry.xpath_node("title").not_nil!.content
@@ -260,41 +255,28 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
premiere_timestamp = channel_video.try &.premiere_timestamp
- video = ChannelVideo.new(
- id: video_id,
- title: title,
- published: published,
- updated: Time.utc,
- ucid: ucid,
- author: author,
- length_seconds: length_seconds,
- live_now: live_now,
+ video = ChannelVideo.new({
+ id: video_id,
+ title: title,
+ published: published,
+ updated: Time.utc,
+ ucid: ucid,
+ author: author,
+ length_seconds: length_seconds,
+ live_now: live_now,
premiere_timestamp: premiere_timestamp,
- views: views,
- )
-
- emails = db.query_all("UPDATE users SET notifications = notifications || $1 \
- WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications) RETURNING email",
- video.id, video.published, ucid, as: String)
-
- video_array = video.to_a
- args = arg_array(video_array)
+ views: views,
+ })
# We don't include the 'premiere_timestamp' here because channel pages don't include them,
# meaning the above timestamp is always null
- db.exec("INSERT INTO channel_videos VALUES (#{args}) \
+ was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \
ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \
updated = $4, ucid = $5, author = $6, length_seconds = $7, \
- live_now = $8, views = $10", args: video_array)
-
- # Update all users affected by insert
- if emails.empty?
- values = "'{}'"
- else
- values = "VALUES #{emails.map { |email| %((E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}')) }.join(",")}"
- end
+ live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool)
- db.exec("UPDATE users SET feed_needs_update = true WHERE email = ANY(#{values})")
+ db.exec("UPDATE users SET notifications = array_append(notifications, $1), \
+ feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert
end
if pull_all_videos
@@ -303,38 +285,24 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
ids = [] of String
loop do
- url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated)
- response = YT_POOL.client &.get(url)
- json = JSON.parse(response.body)
-
- if json["content_html"]? && !json["content_html"].as_s.empty?
- document = XML.parse_html(json["content_html"].as_s)
- nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
- else
- break
- end
-
- nodeset = nodeset.not_nil!
-
- if auto_generated
- videos = extract_videos(nodeset)
- else
- videos = extract_videos(nodeset, ucid, author)
- end
-
- count = nodeset.size
- videos = videos.map { |video| ChannelVideo.new(
- id: video.id,
- title: video.title,
- published: video.published,
- updated: Time.utc,
- ucid: video.ucid,
- author: video.author,
- length_seconds: video.length_seconds,
- live_now: video.live_now,
+ response = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
+ initial_data = JSON.parse(response.body).as_a.find &.["response"]?
+ raise "Could not extract JSON" if !initial_data
+ videos = extract_videos(initial_data.as_h, author, ucid)
+
+ count = videos.size
+ videos = videos.map { |video| ChannelVideo.new({
+ id: video.id,
+ title: video.title,
+ published: video.published,
+ updated: Time.utc,
+ ucid: video.ucid,
+ author: video.author,
+ length_seconds: video.length_seconds,
+ live_now: video.live_now,
premiere_timestamp: video.premiere_timestamp,
- views: video.views
- ) }
+ views: video.views,
+ }) }
videos.each do |video|
ids << video.id
@@ -342,42 +310,28 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
# We are notified of Red videos elsewhere (PubSub), which includes a correct published date,
# so since they don't provide a published date here we can safely ignore them.
if Time.utc - video.published > 1.minute
- emails = db.query_all("UPDATE users SET notifications = notifications || $1 \
- WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications) RETURNING email",
- video.id, video.published, video.ucid, as: String)
-
- video_array = video.to_a
- args = arg_array(video_array)
-
- # We don't update the 'premire_timestamp' here because channel pages don't include them
- db.exec("INSERT INTO channel_videos VALUES (#{args}) \
+ was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \
ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \
updated = $4, ucid = $5, author = $6, length_seconds = $7, \
- live_now = $8, views = $10", args: video_array)
+ live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool)
- # Update all users affected by insert
- if emails.empty?
- values = "'{}'"
- else
- values = "VALUES #{emails.map { |email| %((E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}')) }.join(",")}"
- end
-
- db.exec("UPDATE users SET feed_needs_update = true WHERE email = ANY(#{values})")
+ db.exec("UPDATE users SET notifications = array_append(notifications, $1), \
+ feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert
end
end
- if count < 25
- break
- end
-
+ break if count < 25
page += 1
end
-
- # When a video is deleted from a channel, we find and remove it here
- db.exec("DELETE FROM channel_videos * WHERE NOT id = ANY ('{#{ids.map { |id| %("#{id}") }.join(",")}}') AND ucid = $1", ucid)
end
- channel = InvidiousChannel.new(ucid, author, Time.utc, false, nil)
+ channel = InvidiousChannel.new({
+ id: ucid,
+ author: author,
+ updated: Time.utc,
+ deleted: false,
+ subscribed: nil,
+ })
return channel
end
@@ -387,23 +341,11 @@ def fetch_channel_playlists(ucid, author, auto_generated, continuation, sort_by)
url = produce_channel_playlists_url(ucid, continuation, sort_by, auto_generated)
response = YT_POOL.client &.get(url)
- json = JSON.parse(response.body)
-
- if json["load_more_widget_html"].as_s.empty?
- continuation = nil
- else
- continuation = XML.parse_html(json["load_more_widget_html"].as_s)
- continuation = continuation.xpath_node(%q(//button[@data-uix-load-more-href]))
-
- if continuation
- continuation = extract_channel_playlists_cursor(continuation["data-uix-load-more-href"], auto_generated)
- end
- end
- html = XML.parse_html(json["content_html"].as_s)
- nodeset = html.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
+ continuation = response.body.match(/"continuation":"(?<continuation>[^"]+)"/).try &.["continuation"]?
+ initial_data = JSON.parse(response.body).as_a.find(&.["response"]?).try &.as_h
else
- url = "/channel/#{ucid}/playlists?disable_polymer=1&flow=list&view=1"
+ url = "/channel/#{ucid}/playlists?flow=list&view=1"
case sort_by
when "last", "last_added"
@@ -412,55 +354,58 @@ def fetch_channel_playlists(ucid, author, auto_generated, continuation, sort_by)
url += "&sort=da"
when "newest", "newest_created"
url += "&sort=dd"
+ else nil # Ignore
end
response = YT_POOL.client &.get(url)
- html = XML.parse_html(response.body)
-
- continuation = html.xpath_node(%q(//button[@data-uix-load-more-href]))
- if continuation
- continuation = extract_channel_playlists_cursor(continuation["data-uix-load-more-href"], auto_generated)
- end
-
- nodeset = html.xpath_nodes(%q(//ul[@id="browse-items-primary"]/li[contains(@class, "feed-item-container")]))
+ continuation = response.body.match(/"continuation":"(?<continuation>[^"]+)"/).try &.["continuation"]?
+ initial_data = extract_initial_data(response.body)
end
- if auto_generated
- items = extract_shelf_items(nodeset, ucid, author)
- else
- items = extract_items(nodeset, ucid, author)
- end
+ return [] of SearchItem, nil if !initial_data
+ items = extract_items(initial_data)
+ continuation = extract_channel_playlists_cursor(continuation, auto_generated) if continuation
return items, continuation
end
-def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest")
+def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false)
object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:base64" => {
- "2:string" => "videos",
- "6:varint": 2_i64,
- "7:varint": 1_i64,
- "12:varint": 1_i64,
- "13:string": "",
- "23:varint": 0_i64,
+ "2:string" => "videos",
+ "6:varint" => 2_i64,
+ "7:varint" => 1_i64,
+ "12:varint" => 1_i64,
+ "13:string" => "",
+ "23:varint" => 0_i64,
},
},
}
- if auto_generated
- seed = Time.unix(1525757349)
- until seed >= Time.utc
- seed += 1.month
- end
- timestamp = seed - (page - 1).months
+ if !v2
+ if auto_generated
+ seed = Time.unix(1525757349)
+ until seed >= Time.utc
+ seed += 1.month
+ end
+ timestamp = seed - (page - 1).months
- object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x36_i64
- object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{timestamp.to_unix}"
+ object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x36_i64
+ object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{timestamp.to_unix}"
+ else
+ object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0_i64
+ object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{page}"
+ end
else
object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0_i64
- object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{page}"
+
+ object["80226972:embedded"]["3:base64"].as(Hash)["61:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json({
+ "1:string" => Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json({
+ "1:varint" => 30_i64 * (page - 1),
+ }))),
+ })))
end
case sort_by
@@ -469,6 +414,7 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "
object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 0x01_i64
when "oldest"
object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 0x02_i64
+ else nil # Ignore
end
object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"])))
@@ -487,12 +433,12 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated
"80226972:embedded" => {
"2:string" => ucid,
"3:base64" => {
- "2:string" => "playlists",
- "6:varint": 2_i64,
- "7:varint": 1_i64,
- "12:varint": 1_i64,
- "13:string": "",
- "23:varint": 0_i64,
+ "2:string" => "playlists",
+ "6:varint" => 2_i64,
+ "7:varint" => 1_i64,
+ "12:varint" => 1_i64,
+ "13:string" => "",
+ "23:varint" => 0_i64,
},
},
}
@@ -513,6 +459,7 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated
object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 3_i64
when "last", "last_added"
object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 4_i64
+ else nil # Ignore
end
end
@@ -527,9 +474,8 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated
return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
end
-def extract_channel_playlists_cursor(url, auto_generated)
- cursor = URI.parse(url).query_params
- .try { |i| URI.decode_www_form(i["continuation"]) }
+def extract_channel_playlists_cursor(cursor, auto_generated)
+ cursor = URI.decode_www_form(cursor)
.try { |i| Base64.decode(i) }
.try { |i| IO::Memory.new(i) }
.try { |i| Protodec::Any.parse(i) }
@@ -554,13 +500,13 @@ def extract_channel_playlists_cursor(url, auto_generated)
end
# TODO: Add "sort_by"
-def fetch_channel_community(ucid, continuation, locale, config, kemal_config, format, thin_mode)
+def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
response = YT_POOL.client &.get("/channel/#{ucid}/community?gl=US&hl=en")
- if response.status_code == 404
+ if response.status_code != 200
response = YT_POOL.client &.get("/user/#{ucid}/community?gl=US&hl=en")
end
- if response.status_code == 404
+ if response.status_code != 200
error_message = translate(locale, "This channel does not exist.")
raise error_message
end
@@ -581,16 +527,8 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config, fo
headers = HTTP::Headers.new
headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"]
- headers["content-type"] = "application/x-www-form-urlencoded"
- headers["x-client-data"] = "CIi2yQEIpbbJAQipncoBCNedygEIqKPKAQ=="
- headers["x-spf-previous"] = ""
- headers["x-spf-referer"] = ""
-
- headers["x-youtube-client-name"] = "1"
- headers["x-youtube-client-version"] = "2.20180719"
-
- session_token = response.body.match(/"XSRF_TOKEN":"(?<session_token>[A-Za-z0-9\_\-\=]+)"/).try &.["session_token"]? || ""
+ session_token = response.body.match(/"XSRF_TOKEN":"(?<session_token>[^"]+)"/).try &.["session_token"]? || ""
post_req = {
session_token: session_token,
}
@@ -628,17 +566,9 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config, fo
post = post["backstagePostThreadRenderer"]?.try &.["post"]["backstagePostRenderer"]? ||
post["commentThreadRenderer"]?.try &.["comment"]["commentRenderer"]?
- if !post
- next
- end
-
- if !post["contentText"]?
- content_html = ""
- else
- content_html = post["contentText"]["simpleText"]?.try &.as_s.rchop('\ufeff').try { |block| HTML.escape(block) }.to_s ||
- content_to_comment_html(post["contentText"]["runs"].as_a).try &.to_s || ""
- end
+ next if !post
+ content_html = post["contentText"]?.try { |t| parse_content(t) } || ""
author = post["authorText"]?.try &.["simpleText"]? || ""
json.object do
@@ -707,7 +637,7 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config, fo
json.field "title", attachment["title"]["simpleText"].as_s
json.field "videoId", video_id
json.field "videoThumbnails" do
- generate_thumbnails(json, video_id, config, kemal_config)
+ generate_thumbnails(json, video_id)
end
json.field "lengthSeconds", decode_length_seconds(attachment["lengthText"]["simpleText"].as_s)
@@ -845,153 +775,173 @@ def extract_channel_community_cursor(continuation)
cursor
end
+INITDATA_PREQUERY = "window[\"ytInitialData\"] = {"
+
def get_about_info(ucid, locale)
- about = YT_POOL.client &.get("/channel/#{ucid}/about?disable_polymer=1&gl=US&hl=en")
- if about.status_code == 404
- about = YT_POOL.client &.get("/user/#{ucid}/about?disable_polymer=1&gl=US&hl=en")
+ about = YT_POOL.client &.get("/channel/#{ucid}/about?gl=US&hl=en")
+ if about.status_code != 200
+ about = YT_POOL.client &.get("/user/#{ucid}/about?gl=US&hl=en")
end
if md = about.headers["location"]?.try &.match(/\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/)
raise ChannelRedirect.new(channel_id: md["ucid"])
end
- about = XML.parse_html(about.body)
-
- if about.xpath_node(%q(//div[contains(@class, "channel-empty-message")]))
+ if about.status_code != 200
error_message = translate(locale, "This channel does not exist.")
raise error_message
end
- if about.xpath_node(%q(//span[contains(@class,"qualified-channel-title-text")]/a)).try &.content.empty?
+ initdata_pre = about.body.index(INITDATA_PREQUERY)
+ initdata_post = initdata_pre.nil? ? nil : about.body.index("};", initdata_pre)
+ if initdata_post.nil?
+ about = XML.parse_html(about.body)
error_message = about.xpath_node(%q(//div[@class="yt-alert-content"])).try &.content.strip
error_message ||= translate(locale, "Could not get channel info.")
raise error_message
end
+ initdata_pre = initdata_pre.not_nil! + INITDATA_PREQUERY.size - 1
- author = about.xpath_node(%q(//span[contains(@class,"qualified-channel-title-text")]/a)).not_nil!.content
- author_url = about.xpath_node(%q(//span[contains(@class,"qualified-channel-title-text")]/a)).not_nil!["href"]
- author_thumbnail = about.xpath_node(%q(//img[@class="channel-header-profile-image"])).not_nil!["src"]
+ initdata = JSON.parse(about.body[initdata_pre, initdata_post - initdata_pre + 1])
+ about = XML.parse_html(about.body)
+
+ if about.xpath_node(%q(//div[contains(@class, "channel-empty-message")]))
+ error_message = translate(locale, "This channel does not exist.")
+ raise error_message
+ end
+
+ author = about.xpath_node(%q(//meta[@name="title"])).not_nil!["content"]
+ author_url = about.xpath_node(%q(//link[@rel="canonical"])).not_nil!["href"]
+ author_thumbnail = about.xpath_node(%q(//link[@rel="image_src"])).not_nil!["href"]
ucid = about.xpath_node(%q(//meta[@itemprop="channelId"])).not_nil!["content"]
- banner = about.xpath_node(%q(//div[@id="gh-banner"]/style)).not_nil!.content
- banner = "https:" + banner.match(/background-image: url\((?<url>[^)]+)\)/).not_nil!["url"]
+ # Raises a KeyError on failure.
+ banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]?
+ banner = banners.try &.[-1]?.try &.["url"].as_s?
- if banner.includes? "channels/c4/default_banner"
- banner = nil
- end
+ # if banner.includes? "channels/c4/default_banner"
+ # banner = nil
+ # end
- description_html = about.xpath_node(%q(//div[contains(@class,"about-description")])).try &.to_s ||
- %(<div class="about-description branded-page-box-padding"><pre></pre></div>)
+ description = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]?.try &.as_s? || ""
+ description_html = HTML.escape(description).gsub("\n", "<br>")
paid = about.xpath_node(%q(//meta[@itemprop="paid"])).not_nil!["content"] == "True"
is_family_friendly = about.xpath_node(%q(//meta[@itemprop="isFamilyFriendly"])).not_nil!["content"] == "True"
allowed_regions = about.xpath_node(%q(//meta[@itemprop="regionsAllowed"])).not_nil!["content"].split(",")
- related_channels = about.xpath_nodes(%q(//div[contains(@class, "branded-page-related-channels")]/ul/li))
- related_channels = related_channels.map do |node|
- related_id = node["data-external-id"]?
- related_id ||= ""
-
- anchor = node.xpath_node(%q(.//h3[contains(@class, "yt-lockup-title")]/a))
- related_title = anchor.try &.["title"]
- related_title ||= ""
+ related_channels = initdata["contents"]["twoColumnBrowseResultsRenderer"]
+ .["secondaryContents"]?.try &.["browseSecondaryContentsRenderer"]["contents"][0]?
+ .try &.["verticalChannelSectionRenderer"]?.try &.["items"]?.try &.as_a.map do |node|
+ renderer = node["miniChannelRenderer"]?
+ related_id = renderer.try &.["channelId"]?.try &.as_s?
+ related_id ||= ""
- related_author_url = anchor.try &.["href"]
- related_author_url ||= ""
+ related_title = renderer.try &.["title"]?.try &.["simpleText"]?.try &.as_s?
+ related_title ||= ""
- related_author_thumbnail = node.xpath_node(%q(.//img)).try &.["data-thumb"]
- related_author_thumbnail ||= ""
+ related_author_url = renderer.try &.["navigationEndpoint"]?.try &.["commandMetadata"]?.try &.["webCommandMetadata"]?
+ .try &.["url"]?.try &.as_s?
+ related_author_url ||= ""
- AboutRelatedChannel.new(
- ucid: related_id,
- author: related_title,
- author_url: related_author_url,
- author_thumbnail: related_author_thumbnail,
- )
- end
-
- joined = about.xpath_node(%q(//span[contains(., "Joined")]))
- .try &.content.try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
+ related_author_thumbnails = renderer.try &.["thumbnail"]?.try &.["thumbnails"]?.try &.as_a?
+ related_author_thumbnails ||= [] of JSON::Any
- total_views = about.xpath_node(%q(//span[contains(., "views")]/b))
- .try &.content.try &.gsub(/\D/, "").to_i64? || 0_i64
+ related_author_thumbnail = ""
+ if related_author_thumbnails.size > 0
+ related_author_thumbnail = related_author_thumbnails[-1]["url"]?.try &.as_s?
+ related_author_thumbnail ||= ""
+ end
- sub_count = about.xpath_node(%q(.//span[contains(@class, "subscriber-count")]))
- .try &.["title"].try { |text| short_text_to_number(text) } || 0
+ AboutRelatedChannel.new({
+ ucid: related_id,
+ author: related_title,
+ author_url: related_author_url,
+ author_thumbnail: related_author_thumbnail,
+ })
+ end
+ related_channels ||= [] of AboutRelatedChannel
- # Auto-generated channels
- # https://support.google.com/youtube/answer/2579942
+ total_views = 0_i64
+ joined = Time.unix(0)
+ tabs = [] of String
auto_generated = false
- if about.xpath_node(%q(//ul[@class="about-custom-links"]/li/a[@title="Auto-generated by YouTube"])) ||
- about.xpath_node(%q(//span[@class="qualified-channel-title-badge"]/span[@title="Auto-generated by YouTube"]))
- auto_generated = true
- end
- tabs = about.xpath_nodes(%q(//ul[@id="channel-navigation-menu"]/li/a/span)).map { |node| node.content.downcase }
-
- AboutChannel.new(
- ucid: ucid,
- author: author,
- auto_generated: auto_generated,
- author_url: author_url,
- author_thumbnail: author_thumbnail,
- banner: banner,
- description_html: description_html,
- paid: paid,
- total_views: total_views,
- sub_count: sub_count,
- joined: joined,
+ tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?.try &.as_a?
+ if !tabs_json.nil?
+ # Retrieve information from the tabs array. The index we are looking for varies between channels.
+ tabs_json.each do |node|
+ # Try to find the about section which is located in only one of the tabs.
+ channel_about_meta = node["tabRenderer"]?.try &.["content"]?.try &.["sectionListRenderer"]?
+ .try &.["contents"]?.try &.[0]?.try &.["itemSectionRenderer"]?.try &.["contents"]?
+ .try &.[0]?.try &.["channelAboutFullMetadataRenderer"]?
+
+ if !channel_about_meta.nil?
+ total_views = channel_about_meta["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D/, "").to_i64? || 0_i64
+
+ # The joined text is split to several sub strings. The reduce joins those strings before parsing the date.
+ joined = channel_about_meta["joinedDateText"]?.try &.["runs"]?.try &.as_a.reduce("") { |acc, node| acc + node["text"].as_s }
+ .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
+
+ # Auto-generated channels
+ # https://support.google.com/youtube/answer/2579942
+ # For auto-generated channels, channel_about_meta only has ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"]
+ if (channel_about_meta["primaryLinks"]?.try &.size || 0) == 1 && (channel_about_meta["primaryLinks"][0]?) &&
+ (channel_about_meta["primaryLinks"][0]["title"]?.try &.["simpleText"]?.try &.as_s? || "") == "Auto-generated by YouTube"
+ auto_generated = true
+ end
+ end
+ end
+ tabs = tabs_json.reject { |node| node["tabRenderer"]?.nil? }.map { |node| node["tabRenderer"]["title"].as_s.downcase }
+ end
+
+ sub_count = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s?
+ .try { |text| short_text_to_number(text.split(" ")[0]) } || 0
+
+ AboutChannel.new({
+ ucid: ucid,
+ author: author,
+ auto_generated: auto_generated,
+ author_url: author_url,
+ author_thumbnail: author_thumbnail,
+ banner: banner,
+ description_html: description_html,
+ paid: paid,
+ total_views: total_views,
+ sub_count: sub_count,
+ joined: joined,
is_family_friendly: is_family_friendly,
- allowed_regions: allowed_regions,
- related_channels: related_channels,
- tabs: tabs
- )
+ allowed_regions: allowed_regions,
+ related_channels: related_channels,
+ tabs: tabs,
+ })
+end
+
+def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest")
+ url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated, sort_by: sort_by, v2: true)
+ return YT_POOL.client &.get(url)
end
def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")
- count = 0
videos = [] of SearchVideo
2.times do |i|
- url = produce_channel_videos_url(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)
- response = YT_POOL.client &.get(url)
- json = JSON.parse(response.body)
-
- if json["content_html"]? && !json["content_html"].as_s.empty?
- document = XML.parse_html(json["content_html"].as_s)
- nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
-
- if !json["load_more_widget_html"]?.try &.as_s.empty?
- count += 30
- end
-
- if auto_generated
- videos += extract_videos(nodeset)
- else
- videos += extract_videos(nodeset, ucid, author)
- end
- else
- break
- end
+ response = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)
+ initial_data = JSON.parse(response.body).as_a.find &.["response"]?
+ break if !initial_data
+ videos.concat extract_videos(initial_data.as_h, author, ucid)
end
- return videos, count
+ return videos.size, videos
end
def get_latest_videos(ucid)
- videos = [] of SearchVideo
-
- url = produce_channel_videos_url(ucid, 0)
- response = YT_POOL.client &.get(url)
- json = JSON.parse(response.body)
-
- if json["content_html"]? && !json["content_html"].as_s.empty?
- document = XML.parse_html(json["content_html"].as_s)
- nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
-
- videos = extract_videos(nodeset, ucid)
- end
+ response = get_channel_videos_response(ucid, 1)
+ initial_data = JSON.parse(response.body).as_a.find &.["response"]?
+ return [] of SearchVideo if !initial_data
+ author = initial_data["response"]?.try &.["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s
+ items = extract_videos(initial_data.as_h, author, ucid)
- return videos
+ return items
end
diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr
index 2d7bc1cf..407cef78 100644
--- a/src/invidious/comments.cr
+++ b/src/invidious/comments.cr
@@ -1,11 +1,23 @@
class RedditThing
- JSON.mapping({
- kind: String,
- data: RedditComment | RedditLink | RedditMore | RedditListing,
- })
+ include JSON::Serializable
+
+ property kind : String
+ property data : RedditComment | RedditLink | RedditMore | RedditListing
end
class RedditComment
+ include JSON::Serializable
+
+ property author : String
+ property body_html : String
+ property replies : RedditThing | String
+ property score : Int32
+ property depth : Int32
+ property permalink : String
+
+ @[JSON::Field(converter: RedditComment::TimeConverter)]
+ property created_utc : Time
+
module TimeConverter
def self.from_json(value : JSON::PullParser) : Time
Time.unix(value.read_float.to_i)
@@ -15,51 +27,38 @@ class RedditComment
json.number(value.to_unix)
end
end
-
- JSON.mapping({
- author: String,
- body_html: String,
- replies: RedditThing | String,
- score: Int32,
- depth: Int32,
- permalink: String,
- created_utc: {
- type: Time,
- converter: RedditComment::TimeConverter,
- },
- })
end
struct RedditLink
- JSON.mapping({
- author: String,
- score: Int32,
- subreddit: String,
- num_comments: Int32,
- id: String,
- permalink: String,
- title: String,
- })
+ include JSON::Serializable
+
+ property author : String
+ property score : Int32
+ property subreddit : String
+ property num_comments : Int32
+ property id : String
+ property permalink : String
+ property title : String
end
struct RedditMore
- JSON.mapping({
- children: Array(String),
- count: Int32,
- depth: Int32,
- })
+ include JSON::Serializable
+
+ property children : Array(String)
+ property count : Int32
+ property depth : Int32
end
class RedditListing
- JSON.mapping({
- children: Array(RedditThing),
- modhash: String,
- })
+ include JSON::Serializable
+
+ property children : Array(RedditThing)
+ property modhash : String
end
def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, sort_by = "top")
video = get_video(id, db, region: region)
- session_token = video.info["session_token"]?
+ session_token = video.session_token
case cursor
when nil, ""
@@ -85,17 +84,9 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so
session_token: session_token,
}
- headers = HTTP::Headers.new
-
- headers["content-type"] = "application/x-www-form-urlencoded"
- headers["cookie"] = video.info["cookie"]
-
- headers["x-client-data"] = "CIi2yQEIpbbJAQipncoBCNedygEIqKPKAQ=="
- headers["x-spf-previous"] = "https://www.youtube.com/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999"
- headers["x-spf-referer"] = "https://www.youtube.com/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999"
-
- headers["x-youtube-client-name"] = "1"
- headers["x-youtube-client-version"] = "2.20180719"
+ headers = HTTP::Headers{
+ "cookie" => video.cookie,
+ }
response = YT_POOL.client(region, &.post("/comment_service_ajax?action_get_comments=1&hl=en&gl=US", headers, form: post_req))
response = JSON.parse(response.body)
@@ -150,8 +141,7 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so
node_comment = node["commentRenderer"]
end
- content_html = node_comment["contentText"]["simpleText"]?.try &.as_s.rchop('\ufeff').try { |block| HTML.escape(block) }.to_s ||
- content_to_comment_html(node_comment["contentText"]["runs"].as_a).try &.to_s || ""
+ content_html = node_comment["contentText"]?.try { |t| parse_content(t) } || ""
author = node_comment["authorText"]?.try &.["simpleText"]? || ""
json.field "author", author
@@ -294,7 +284,7 @@ def template_youtube_comments(comments, locale, thin_mode)
<div class="pure-u-23-24">
<p>
<a href="javascript:void(0)" data-continuation="#{child["replies"]["continuation"]}"
- onclick="get_youtube_replies(this)">#{translate(locale, "View `x` replies", number_with_separator(child["replies"]["replyCount"]))}</a>
+ data-onclick="get_youtube_replies">#{translate(locale, "View `x` replies", number_with_separator(child["replies"]["replyCount"]))}</a>
</p>
</div>
</div>
@@ -347,7 +337,7 @@ def template_youtube_comments(comments, locale, thin_mode)
END_HTML
else
html << <<-END_HTML
- <iframe id='ivplayer' type='text/html' style='position:absolute;width:100%;height:100%;left:0;top:0' src='/embed/#{attachment["videoId"]?}?autoplay=0' frameborder='0'></iframe>
+ <iframe id='ivplayer' style='position:absolute;width:100%;height:100%;left:0;top:0' src='/embed/#{attachment["videoId"]?}?autoplay=0' style='border:none;'></iframe>
END_HTML
end
@@ -356,6 +346,7 @@ def template_youtube_comments(comments, locale, thin_mode)
</div>
</div>
END_HTML
+ else nil # Ignore
end
end
@@ -413,7 +404,7 @@ def template_youtube_comments(comments, locale, thin_mode)
<div class="pure-u-1">
<p>
<a href="javascript:void(0)" data-continuation="#{comments["continuation"]}"
- onclick="get_youtube_replies(this, true)">#{translate(locale, "Load more")}</a>
+ data-onclick="get_youtube_replies" data-load-more>#{translate(locale, "Load more")}</a>
</p>
</div>
</div>
@@ -451,7 +442,7 @@ def template_reddit_comments(root, locale)
html << <<-END_HTML
<p>
- <a href="javascript:void(0)" onclick="toggle_parent(this)">[ - ]</a>
+ <a href="javascript:void(0)" data-onclick="toggle_parent">[ - ]</a>
<b><a href="https://www.reddit.com/user/#{child.author}">#{child.author}</a></b>
#{translate(locale, "`x` points", number_with_separator(child.score))}
<span title="#{child.created_utc.to_s(translate(locale, "%a %B %-d %T %Y UTC"))}">#{translate(locale, "`x` ago", recode_date(child.created_utc, locale))}</span>
@@ -522,6 +513,11 @@ def fill_links(html, scheme, host)
return html.to_xml(options: XML::SaveOptions::NO_DECL)
end
+def parse_content(content : JSON::Any) : String
+ content["simpleText"]?.try &.as_s.rchop('\ufeff').try { |b| HTML.escape(b) }.to_s ||
+ content["runs"]?.try &.as_a.try { |r| content_to_comment_html(r).try &.to_s } || ""
+end
+
def content_to_comment_html(content)
comment_html = content.map do |run|
text = HTML.escape(run["text"].as_s)
@@ -556,7 +552,7 @@ def content_to_comment_html(content)
video_id = watch_endpoint["videoId"].as_s
if length_seconds
- text = %(<a href="javascript:void(0)" onclick="player.currentTime(#{length_seconds})">#{text}</a>)
+ text = %(<a href="javascript:void(0)" data-onclick="jump_to_time" data-jump-time="#{length_seconds}">#{text}</a>)
else
text = %(<a href="/watch?v=#{video_id}">#{text}</a>)
end
@@ -609,6 +605,8 @@ def produce_comment_continuation(video_id, cursor = "", sort_by = "top")
object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64
when "new", "newest"
object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 1_i64
+ else # top
+ object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64
end
continuation = object.try { |i| Protodec::Any.cast_json(object) }
diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr
index 456618cf..045b6701 100644
--- a/src/invidious/helpers/handlers.cr
+++ b/src/invidious/helpers/handlers.cr
@@ -61,7 +61,7 @@ class Kemal::ExceptionHandler
end
class FilteredCompressHandler < Kemal::Handler
- exclude ["/videoplayback", "/videoplayback/*", "/vi/*", "/ggpht/*", "/api/v1/auth/notifications"]
+ exclude ["/videoplayback", "/videoplayback/*", "/vi/*", "/sb/*", "/ggpht/*", "/api/v1/auth/notifications"]
exclude ["/api/v1/auth/notifications", "/data_control"], "POST"
def call(env)
@@ -74,10 +74,10 @@ class FilteredCompressHandler < Kemal::Handler
if request_headers.includes_word?("Accept-Encoding", "gzip")
env.response.headers["Content-Encoding"] = "gzip"
- env.response.output = Gzip::Writer.new(env.response.output, sync_close: true)
+ env.response.output = Compress::Gzip::Writer.new(env.response.output, sync_close: true)
elsif request_headers.includes_word?("Accept-Encoding", "deflate")
env.response.headers["Content-Encoding"] = "deflate"
- env.response.output = Flate::Writer.new(env.response.output, sync_close: true)
+ env.response.output = Compress::Deflate::Writer.new(env.response.output, sync_close: true)
end
call_next env
@@ -212,29 +212,3 @@ class DenyFrame < Kemal::Handler
call_next env
end
end
-
-# Temp fixes for https://github.com/crystal-lang/crystal/issues/7383
-class HTTP::UnknownLengthContent
- def read_byte
- ensure_send_continue
- if @io.is_a?(OpenSSL::SSL::Socket::Client)
- return if @io.as(OpenSSL::SSL::Socket::Client).@in_buffer_rem.empty?
- end
- @io.read_byte
- end
-end
-
-class HTTP::Client
- private def handle_response(response)
- if @socket.is_a?(OpenSSL::SSL::Socket::Client) && @host.ends_with?("googlevideo.com")
- close unless response.keep_alive? || @socket.as(OpenSSL::SSL::Socket::Client).@in_buffer_rem.empty?
-
- if @socket.as(OpenSSL::SSL::Socket::Client).@in_buffer_rem.empty?
- @socket = nil
- end
- else
- close unless response.keep_alive?
- end
- response
- end
-end
diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr
index 2341d3be..62c24f3e 100644
--- a/src/invidious/helpers/helpers.cr
+++ b/src/invidious/helpers/helpers.cr
@@ -1,217 +1,100 @@
require "./macros"
struct Nonce
- db_mapping({
- nonce: String,
- expire: Time,
- })
+ include DB::Serializable
+
+ property nonce : String
+ property expire : Time
end
struct SessionId
- db_mapping({
- id: String,
- email: String,
- issued: String,
- })
+ include DB::Serializable
+
+ property id : String
+ property email : String
+ property issued : String
end
struct Annotation
- db_mapping({
- id: String,
- annotations: String,
- })
+ include DB::Serializable
+
+ property id : String
+ property annotations : String
end
struct ConfigPreferences
- module StringToArray
- def self.to_json(value : Array(String), json : JSON::Builder)
- json.array do
- value.each do |element|
- json.string element
- end
- end
- end
-
- def self.from_json(value : JSON::PullParser) : Array(String)
- begin
- result = [] of String
- value.read_array do
- result << HTML.escape(value.read_string[0, 100])
- end
- rescue ex
- result = [HTML.escape(value.read_string[0, 100]), ""]
- end
-
- result
- end
-
- def self.to_yaml(value : Array(String), yaml : YAML::Nodes::Builder)
- yaml.sequence do
- value.each do |element|
- yaml.scalar element
- end
- end
- end
-
- def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(String)
- begin
- unless node.is_a?(YAML::Nodes::Sequence)
- node.raise "Expected sequence, not #{node.class}"
- end
-
- result = [] of String
- node.nodes.each do |item|
- unless item.is_a?(YAML::Nodes::Scalar)
- node.raise "Expected scalar, not #{item.class}"
- end
-
- result << HTML.escape(item.value[0, 100])
- end
- rescue ex
- if node.is_a?(YAML::Nodes::Scalar)
- result = [HTML.escape(node.value[0, 100]), ""]
- else
- result = ["", ""]
- end
- end
-
- result
- end
- end
-
- module BoolToString
- def self.to_json(value : String, json : JSON::Builder)
- json.string value
- end
-
- def self.from_json(value : JSON::PullParser) : String
- begin
- result = value.read_string
-
- if result.empty?
- CONFIG.default_user_preferences.dark_mode
- else
- result
- end
- rescue ex
- if value.read_bool
- "dark"
- else
- "light"
- end
- end
- end
-
- def self.to_yaml(value : String, yaml : YAML::Nodes::Builder)
- yaml.scalar value
- end
-
- def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String
- unless node.is_a?(YAML::Nodes::Scalar)
- node.raise "Expected scalar, not #{node.class}"
- end
-
- case node.value
- when "true"
- "dark"
- when "false"
- "light"
- when ""
- CONFIG.default_user_preferences.dark_mode
- else
- node.value
- end
- end
+ include YAML::Serializable
+
+ property annotations : Bool = false
+ property annotations_subscribed : Bool = false
+ property autoplay : Bool = false
+ property captions : Array(String) = ["", "", ""]
+ property comments : Array(String) = ["youtube", ""]
+ property continue : Bool = false
+ property continue_autoplay : Bool = true
+ property dark_mode : String = ""
+ property latest_only : Bool = false
+ property listen : Bool = false
+ property local : Bool = false
+ property locale : String = "en-US"
+ property max_results : Int32 = 40
+ property notifications_only : Bool = false
+ property player_style : String = "invidious"
+ property quality : String = "hd720"
+ property default_home : String = "Popular"
+ property feed_menu : Array(String) = ["Popular", "Trending", "Subscriptions", "Playlists"]
+ property related_videos : Bool = true
+ property sort : String = "published"
+ property speed : Float32 = 1.0_f32
+ property thin_mode : Bool = false
+ property unseen_only : Bool = false
+ property video_loop : Bool = false
+ property volume : Int32 = 100
+
+ def to_tuple
+ {% begin %}
+ {
+ {{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}}
+ }
+ {% end %}
end
-
- yaml_mapping({
- annotations: {type: Bool, default: false},
- annotations_subscribed: {type: Bool, default: false},
- autoplay: {type: Bool, default: false},
- captions: {type: Array(String), default: ["", "", ""], converter: StringToArray},
- comments: {type: Array(String), default: ["youtube", ""], converter: StringToArray},
- continue: {type: Bool, default: false},
- continue_autoplay: {type: Bool, default: true},
- dark_mode: {type: String, default: "", converter: BoolToString},
- latest_only: {type: Bool, default: false},
- listen: {type: Bool, default: false},
- local: {type: Bool, default: false},
- locale: {type: String, default: "en-US"},
- max_results: {type: Int32, default: 40},
- notifications_only: {type: Bool, default: false},
- player_style: {type: String, default: "invidious"},
- quality: {type: String, default: "hd720"},
- default_home: {type: String, default: "Popular"},
- feed_menu: {type: Array(String), default: ["Popular", "Trending", "Subscriptions", "Playlists"]},
- related_videos: {type: Bool, default: true},
- sort: {type: String, default: "published"},
- speed: {type: Float32, default: 1.0_f32},
- thin_mode: {type: Bool, default: false},
- unseen_only: {type: Bool, default: false},
- video_loop: {type: Bool, default: false},
- volume: {type: Int32, default: 100},
- })
end
struct Config
- module ConfigPreferencesConverter
- def self.to_yaml(value : Preferences, yaml : YAML::Nodes::Builder)
- value.to_yaml(yaml)
- end
-
- def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Preferences
- Preferences.new(*ConfigPreferences.new(ctx, node).to_tuple)
- end
- end
-
- module FamilyConverter
- def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder)
- case value
- when Socket::Family::UNSPEC
- yaml.scalar nil
- when Socket::Family::INET
- yaml.scalar "ipv4"
- when Socket::Family::INET6
- yaml.scalar "ipv6"
- end
- end
-
- def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family
- if node.is_a?(YAML::Nodes::Scalar)
- case node.value.downcase
- when "ipv4"
- Socket::Family::INET
- when "ipv6"
- Socket::Family::INET6
- else
- Socket::Family::UNSPEC
- end
- else
- node.raise "Expected scalar, not #{node.class}"
- end
- end
- end
-
- module StringToCookies
- def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder)
- (value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml)
- end
-
- def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies
- unless node.is_a?(YAML::Nodes::Scalar)
- node.raise "Expected scalar, not #{node.class}"
- end
-
- cookies = HTTP::Cookies.new
- node.value.split(";").each do |cookie|
- next if cookie.strip.empty?
- name, value = cookie.split("=", 2)
- cookies << HTTP::Cookie.new(name.strip, value.strip)
- end
-
- cookies
- end
- end
+ include YAML::Serializable
+
+ property channel_threads : Int32 # Number of threads to use for crawling videos from channels (for updating subscriptions)
+ property feed_threads : Int32 # Number of threads to use for updating feeds
+ property db : DBConfig # Database configuration
+ property full_refresh : Bool # Used for crawling channels: threads should check all videos uploaded by a channel
+ property https_only : Bool? # Used to tell Invidious it is behind a proxy, so links to resources should be https://
+ property hmac_key : String? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
+ property domain : String? # Domain to be used for links to resources on the site where an absolute URL is required
+ property use_pubsub_feeds : Bool | Int32 = false # Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
+ property captcha_enabled : Bool = true
+ property login_enabled : Bool = true
+ property registration_enabled : Bool = true
+ property statistics_enabled : Bool = false
+ property admins : Array(String) = [] of String
+ property external_port : Int32? = nil
+ property default_user_preferences : ConfigPreferences = ConfigPreferences.from_yaml("")
+ property dmca_content : Array(String) = [] of String # For compliance with DMCA, disables download widget using list of video IDs
+ property check_tables : Bool = false # Check table integrity, automatically try to add any missing columns, create tables, etc.
+ property cache_annotations : Bool = false # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards
+ property banner : String? = nil # Optional banner to be displayed along top of page for announcements, etc.
+ property hsts : Bool? = true # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely
+ property disable_proxy : Bool? | Array(String)? = false # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local'
+
+ @[YAML::Field(converter: Preferences::FamilyConverter)]
+ property force_resolve : Socket::Family = Socket::Family::UNSPEC # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
+ property port : Int32 = 3000 # Port to listen for connections (overrided by command line argument)
+ property host_binding : String = "0.0.0.0" # Host to bind (overrided by command line argument)
+ property pool_size : Int32 = 100 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
+ property admin_email : String = "omarroth@protonmail.com" # Email for bug reports
+
+ @[YAML::Field(converter: Preferences::StringToCookies)]
+ property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format
+ property captcha_key : String? = nil # Key for Anti-Captcha
def disabled?(option)
case disabled = CONFIG.disable_proxy
@@ -223,77 +106,20 @@ struct Config
else
return false
end
+ else
+ return false
end
end
-
- YAML.mapping({
- channel_threads: Int32, # Number of threads to use for crawling videos from channels (for updating subscriptions)
- feed_threads: Int32, # Number of threads to use for updating feeds
- db: DBConfig, # Database configuration
- full_refresh: Bool, # Used for crawling channels: threads should check all videos uploaded by a channel
- https_only: Bool?, # Used to tell Invidious it is behind a proxy, so links to resources should be https://
- hmac_key: String?, # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
- domain: String?, # Domain to be used for links to resources on the site where an absolute URL is required
- use_pubsub_feeds: {type: Bool | Int32, default: false}, # Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
- top_enabled: {type: Bool, default: true},
- captcha_enabled: {type: Bool, default: true},
- login_enabled: {type: Bool, default: true},
- registration_enabled: {type: Bool, default: true},
- statistics_enabled: {type: Bool, default: false},
- admins: {type: Array(String), default: [] of String},
- external_port: {type: Int32?, default: nil},
- default_user_preferences: {type: Preferences,
- default: Preferences.new(*ConfigPreferences.from_yaml("").to_tuple),
- converter: ConfigPreferencesConverter,
- },
- dmca_content: {type: Array(String), default: [] of String}, # For compliance with DMCA, disables download widget using list of video IDs
- check_tables: {type: Bool, default: false}, # Check table integrity, automatically try to add any missing columns, create tables, etc.
- cache_annotations: {type: Bool, default: false}, # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards
- banner: {type: String?, default: nil}, # Optional banner to be displayed along top of page for announcements, etc.
- hsts: {type: Bool?, default: true}, # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely
- disable_proxy: {type: Bool? | Array(String)?, default: false}, # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local'
- force_resolve: {type: Socket::Family, default: Socket::Family::UNSPEC, converter: FamilyConverter}, # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
- port: {type: Int32, default: 3000}, # Port to listen for connections (overrided by command line argument)
- host_binding: {type: String, default: "0.0.0.0"}, # Host to bind (overrided by command line argument)
- pool_size: {type: Int32, default: 100}, # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
- admin_email: {type: String, default: "omarroth@protonmail.com"}, # Email for bug reports
- cookies: {type: HTTP::Cookies, default: HTTP::Cookies.new, converter: StringToCookies}, # Saved cookies in "name1=value1; name2=value2..." format
- captcha_key: {type: String?, default: nil}, # Key for Anti-Captcha
- })
end
struct DBConfig
- yaml_mapping({
- user: String,
- password: String,
- host: String,
- port: Int32,
- dbname: String,
- })
-end
+ include YAML::Serializable
-def rank_videos(db, n)
- top = [] of {Float64, String}
-
- db.query("SELECT id, wilson_score, published FROM videos WHERE views > 5000 ORDER BY published DESC LIMIT 1000") do |rs|
- rs.each do
- id = rs.read(String)
- wilson_score = rs.read(Float64)
- published = rs.read(Time)
-
- # Exponential decay, older videos tend to rank lower
- temperature = wilson_score * Math.exp(-0.000005*((Time.utc - published).total_minutes))
- top << {temperature, id}
- end
- end
-
- top.sort!
-
- # Make hottest come first
- top.reverse!
- top = top.map { |a, b| b }
-
- return top[0..n - 1]
+ property user : String
+ property password : String
+ property host : String
+ property port : Int32
+ property dbname : String
end
def login_req(f_req)
@@ -334,293 +160,179 @@ def html_to_content(description_html : String)
return description
end
-def extract_videos(nodeset, ucid = nil, author_name = nil)
- videos = extract_items(nodeset, ucid, author_name)
- videos.select { |item| item.is_a?(SearchVideo) }.map { |video| video.as(SearchVideo) }
+def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
+ extract_items(initial_data, author_fallback, author_id_fallback).select(&.is_a?(SearchVideo)).map(&.as(SearchVideo))
end
-def extract_items(nodeset, ucid = nil, author_name = nil)
- # TODO: Make this a 'common', so it makes more sense to be used here
- items = [] of SearchItem
-
- nodeset.each do |node|
- anchor = node.xpath_node(%q(.//h3[contains(@class, "yt-lockup-title")]/a))
- if !anchor
- next
- end
- title = anchor.content.strip
- id = anchor["href"]
-
- if anchor["href"].starts_with? "https://www.googleadservices.com"
- next
- end
-
- author_id = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-byline")]/a)).try &.["href"].split("/")[-1] || ucid || ""
- author = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-byline")]/a)).try &.content.strip || author_name || ""
- description_html = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-description")])).try &.to_s || ""
-
- tile = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-tile")]))
- if !tile
- next
- end
-
- case tile["class"]
- when .includes? "yt-lockup-playlist"
- plid = HTTP::Params.parse(URI.parse(id).query.not_nil!)["list"]
-
- anchor = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-meta")]/a))
-
- if !anchor
- anchor = node.xpath_node(%q(.//ul[@class="yt-lockup-meta-info"]/li/a))
- end
-
- video_count = node.xpath_node(%q(.//span[@class="formatted-video-count-label"]/b)) ||
- node.xpath_node(%q(.//span[@class="formatted-video-count-label"]))
- if video_count
- video_count = video_count.content
-
- if video_count == "50+"
- author = "YouTube"
- author_id = "UC-9-kyTW8ZkZNDHQJ6FgpwQ"
- end
-
- video_count = video_count.gsub(/\D/, "").to_i?
- end
- video_count ||= 0
-
- videos = [] of SearchPlaylistVideo
- node.xpath_nodes(%q(.//*[contains(@class, "yt-lockup-playlist-items")]/li)).each do |video|
- anchor = video.xpath_node(%q(.//a))
- if anchor
- video_title = anchor.content.strip
- id = HTTP::Params.parse(URI.parse(anchor["href"]).query.not_nil!)["v"]
- end
- video_title ||= ""
- id ||= ""
-
- anchor = video.xpath_node(%q(.//span/span))
- if anchor
- length_seconds = decode_length_seconds(anchor.content)
- end
- length_seconds ||= 0
-
- videos << SearchPlaylistVideo.new(
- video_title,
- id,
- length_seconds
- )
- end
-
- playlist_thumbnail = node.xpath_node(%q(.//span/img)).try &.["data-thumb"]?
- playlist_thumbnail ||= node.xpath_node(%q(.//span/img)).try &.["src"]
-
- items << SearchPlaylist.new(
- title: title,
- id: plid,
- author: author,
- ucid: author_id,
- video_count: video_count,
- videos: videos,
- thumbnail: playlist_thumbnail
- )
- when .includes? "yt-lockup-channel"
- author = title.strip
-
- ucid = node.xpath_node(%q(.//button[contains(@class, "yt-uix-subscription-button")])).try &.["data-channel-external-id"]?
- ucid ||= id.split("/")[-1]
-
- author_thumbnail = node.xpath_node(%q(.//div/span/img)).try &.["data-thumb"]?
- author_thumbnail ||= node.xpath_node(%q(.//div/span/img)).try &.["src"]
- if author_thumbnail
- author_thumbnail = URI.parse(author_thumbnail)
- author_thumbnail.scheme = "https"
- author_thumbnail = author_thumbnail.to_s
- end
-
- author_thumbnail ||= ""
-
- subscriber_count = node.xpath_node(%q(.//span[contains(@class, "subscriber-count")]))
- .try &.["title"].try { |text| short_text_to_number(text) } || 0
-
- video_count = node.xpath_node(%q(.//ul[@class="yt-lockup-meta-info"]/li)).try &.content.split(" ")[0].gsub(/\D/, "").to_i?
-
- items << SearchChannel.new(
- author: author,
- ucid: ucid,
- author_thumbnail: author_thumbnail,
- subscriber_count: subscriber_count,
- video_count: video_count || 0,
- description_html: description_html,
- auto_generated: video_count ? false : true,
- )
- else
- id = id.lchop("/watch?v=")
-
- metadata = node.xpath_node(%q(.//div[contains(@class,"yt-lockup-meta")]/ul))
-
- published = metadata.try &.xpath_node(%q(.//li[contains(text(), " ago")])).try { |node| decode_date(node.content.sub(/^[a-zA-Z]+ /, "")) }
- published ||= metadata.try &.xpath_node(%q(.//span[@data-timestamp])).try { |node| Time.unix(node["data-timestamp"].to_i64) }
- published ||= Time.utc
-
- view_count = metadata.try &.xpath_node(%q(.//li[contains(text(), " views")])).try &.content.gsub(/\D/, "").to_i64?
- view_count ||= 0_i64
-
- length_seconds = node.xpath_node(%q(.//span[@class="video-time"])).try { |node| decode_length_seconds(node.content) }
- length_seconds ||= -1
-
- live_now = node.xpath_node(%q(.//span[contains(@class, "yt-badge-live")])) ? true : false
- premium = node.xpath_node(%q(.//span[text()="Premium"])) ? true : false
-
- if !premium || node.xpath_node(%q(.//span[contains(text(), "Free episode")]))
- paid = false
- else
+def extract_item(item : JSON::Any, author_fallback : String? = nil, author_id_fallback : String? = nil)
+ if i = (item["videoRenderer"]? || item["gridVideoRenderer"]?)
+ video_id = i["videoId"].as_s
+ title = i["title"].try { |t| t["simpleText"]?.try &.as_s || t["runs"]?.try &.as_a.map(&.["text"].as_s).join("") } || ""
+
+ author_info = i["ownerText"]?.try &.["runs"].as_a[0]?
+ author = author_info.try &.["text"].as_s || author_fallback || ""
+ author_id = author_info.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]["browseId"].as_s || author_id_fallback || ""
+
+ published = i["publishedTimeText"]?.try &.["simpleText"]?.try { |t| decode_date(t.as_s) } || Time.local
+ view_count = i["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64
+ description_html = i["descriptionSnippet"]?.try { |t| parse_content(t) } || ""
+ length_seconds = i["lengthText"]?.try &.["simpleText"]?.try &.as_s.try { |t| decode_length_seconds(t) } ||
+ i["thumbnailOverlays"]?.try &.as_a.find(&.["thumbnailOverlayTimeStatusRenderer"]?).try &.["thumbnailOverlayTimeStatusRenderer"]?
+ .try &.["text"]?.try &.["simpleText"]?.try &.as_s.try { |t| decode_length_seconds(t) } || 0
+
+ live_now = false
+ paid = false
+ premium = false
+
+ premiere_timestamp = i["upcomingEventData"]?.try &.["startTime"]?.try { |t| Time.unix(t.as_s.to_i64) }
+
+ i["badges"]?.try &.as_a.each do |badge|
+ b = badge["metadataBadgeRenderer"]
+ case b["label"].as_s
+ when "LIVE NOW"
+ live_now = true
+ when "New", "4K", "CC"
+ # TODO
+ when "Premium"
paid = true
- end
- premiere_timestamp = node.xpath_node(%q(.//ul[@class="yt-lockup-meta-info"]/li/span[@class="localized-date"])).try &.["data-timestamp"]?.try &.to_i64
- if premiere_timestamp
- premiere_timestamp = Time.unix(premiere_timestamp)
+ # TODO: Potentially available as i["topStandaloneBadge"]["metadataBadgeRenderer"]
+ premium = true
+ else nil # Ignore
end
-
- items << SearchVideo.new(
- title: title,
- id: id,
- author: author,
- ucid: author_id,
- published: published,
- views: view_count,
- description_html: description_html,
- length_seconds: length_seconds,
- live_now: live_now,
- paid: paid,
- premium: premium,
- premiere_timestamp: premiere_timestamp
- )
end
- end
- return items
+ SearchVideo.new({
+ title: title,
+ id: video_id,
+ author: author,
+ ucid: author_id,
+ published: published,
+ views: view_count,
+ description_html: description_html,
+ length_seconds: length_seconds,
+ live_now: live_now,
+ paid: paid,
+ premium: premium,
+ premiere_timestamp: premiere_timestamp,
+ })
+ elsif i = item["channelRenderer"]?
+ author = i["title"]["simpleText"]?.try &.as_s || author_fallback || ""
+ author_id = i["channelId"]?.try &.as_s || author_id_fallback || ""
+
+ author_thumbnail = i["thumbnail"]["thumbnails"]?.try &.as_a[0]?.try { |u| "https:#{u["url"]}" } || ""
+ subscriber_count = i["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s.try { |s| short_text_to_number(s.split(" ")[0]) } || 0
+
+ auto_generated = false
+ auto_generated = true if !i["videoCountText"]?
+ video_count = i["videoCountText"]?.try &.["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0
+ description_html = i["descriptionSnippet"]?.try { |t| parse_content(t) } || ""
+
+ SearchChannel.new({
+ author: author,
+ ucid: author_id,
+ author_thumbnail: author_thumbnail,
+ subscriber_count: subscriber_count,
+ video_count: video_count,
+ description_html: description_html,
+ auto_generated: auto_generated,
+ })
+ elsif i = item["gridPlaylistRenderer"]?
+ title = i["title"]["runs"].as_a[0]?.try &.["text"].as_s || ""
+ plid = i["playlistId"]?.try &.as_s || ""
+
+ video_count = i["videoCountText"]["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0
+ playlist_thumbnail = i["thumbnail"]["thumbnails"][0]?.try &.["url"]?.try &.as_s || ""
+
+ SearchPlaylist.new({
+ title: title,
+ id: plid,
+ author: author_fallback || "",
+ ucid: author_id_fallback || "",
+ video_count: video_count,
+ videos: [] of SearchPlaylistVideo,
+ thumbnail: playlist_thumbnail,
+ })
+ elsif i = item["playlistRenderer"]?
+ title = i["title"]["simpleText"]?.try &.as_s || ""
+ plid = i["playlistId"]?.try &.as_s || ""
+
+ video_count = i["videoCount"]?.try &.as_s.to_i || 0
+ playlist_thumbnail = i["thumbnails"].as_a[0]?.try &.["thumbnails"]?.try &.as_a[0]?.try &.["url"].as_s || ""
+
+ author_info = i["shortBylineText"]?.try &.["runs"].as_a[0]?
+ author = author_info.try &.["text"].as_s || author_fallback || ""
+ author_id = author_info.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]["browseId"].as_s || author_id_fallback || ""
+
+ videos = i["videos"]?.try &.as_a.map do |v|
+ v = v["childVideoRenderer"]
+ v_title = v["title"]["simpleText"]?.try &.as_s || ""
+ v_id = v["videoId"]?.try &.as_s || ""
+ v_length_seconds = v["lengthText"]?.try &.["simpleText"]?.try { |t| decode_length_seconds(t.as_s) } || 0
+ SearchPlaylistVideo.new({
+ title: v_title,
+ id: v_id,
+ length_seconds: v_length_seconds,
+ })
+ end || [] of SearchPlaylistVideo
+
+ # TODO: i["publishedTimeText"]?
+
+ SearchPlaylist.new({
+ title: title,
+ id: plid,
+ author: author,
+ ucid: author_id,
+ video_count: video_count,
+ videos: videos,
+ thumbnail: playlist_thumbnail,
+ })
+ elsif i = item["radioRenderer"]? # Mix
+ # TODO
+ elsif i = item["showRenderer"]? # Show
+ # TODO
+ elsif i = item["shelfRenderer"]?
+ elsif i = item["horizontalCardListRenderer"]?
+ elsif i = item["searchPyvRenderer"]? # Ad
+ end
end
-def extract_shelf_items(nodeset, ucid = nil, author_name = nil)
- items = [] of SearchPlaylist
-
- nodeset.each do |shelf|
- shelf_anchor = shelf.xpath_node(%q(.//h2[contains(@class, "branded-page-module-title")]))
- next if !shelf_anchor
-
- title = shelf_anchor.xpath_node(%q(.//span[contains(@class, "branded-page-module-title-text")])).try &.content.strip
- title ||= ""
-
- id = shelf_anchor.xpath_node(%q(.//a)).try &.["href"]
- next if !id
-
- shelf_is_playlist = false
- videos = [] of SearchPlaylistVideo
-
- shelf.xpath_nodes(%q(.//ul[contains(@class, "yt-uix-shelfslider-list") or contains(@class, "expanded-shelf-content-list")]/li)).each do |child_node|
- type = child_node.xpath_node(%q(./div))
- if !type
- next
- end
-
- case type["class"]
- when .includes? "yt-lockup-video"
- shelf_is_playlist = true
-
- anchor = child_node.xpath_node(%q(.//h3[contains(@class, "yt-lockup-title")]/a))
- if anchor
- video_title = anchor.content.strip
- video_id = HTTP::Params.parse(URI.parse(anchor["href"]).query.not_nil!)["v"]
- end
- video_title ||= ""
- video_id ||= ""
-
- anchor = child_node.xpath_node(%q(.//span[@class="video-time"]))
- if anchor
- length_seconds = decode_length_seconds(anchor.content)
- end
- length_seconds ||= 0
-
- videos << SearchPlaylistVideo.new(
- video_title,
- video_id,
- length_seconds
- )
- when .includes? "yt-lockup-playlist"
- anchor = child_node.xpath_node(%q(.//h3[contains(@class, "yt-lockup-title")]/a))
- if anchor
- playlist_title = anchor.content.strip
- params = HTTP::Params.parse(URI.parse(anchor["href"]).query.not_nil!)
- plid = params["list"]
- end
- playlist_title ||= ""
- plid ||= ""
-
- playlist_thumbnail = child_node.xpath_node(%q(.//span/img)).try &.["data-thumb"]?
- playlist_thumbnail ||= child_node.xpath_node(%q(.//span/img)).try &.["src"]
-
- video_count = child_node.xpath_node(%q(.//span[@class="formatted-video-count-label"]/b)) ||
- child_node.xpath_node(%q(.//span[@class="formatted-video-count-label"]))
- if video_count
- video_count = video_count.content.gsub(/\D/, "").to_i?
- end
- video_count ||= 50
-
- videos = [] of SearchPlaylistVideo
- child_node.xpath_nodes(%q(.//*[contains(@class, "yt-lockup-playlist-items")]/li)).each do |video|
- anchor = video.xpath_node(%q(.//a))
- if anchor
- video_title = anchor.content.strip
- id = HTTP::Params.parse(URI.parse(anchor["href"]).query.not_nil!)["v"]
- end
- video_title ||= ""
- id ||= ""
-
- anchor = video.xpath_node(%q(.//span/span))
- if anchor
- length_seconds = decode_length_seconds(anchor.content)
- end
- length_seconds ||= 0
-
- videos << SearchPlaylistVideo.new(
- video_title,
- id,
- length_seconds
- )
- end
-
- items << SearchPlaylist.new(
- title: playlist_title,
- id: plid,
- author: author_name,
- ucid: ucid,
- video_count: video_count,
- videos: videos,
- thumbnail: playlist_thumbnail
- )
- end
- end
+def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
+ items = [] of SearchItem
- if shelf_is_playlist
- plid = HTTP::Params.parse(URI.parse(id).query.not_nil!)["list"]
-
- items << SearchPlaylist.new(
- title: title,
- id: plid,
- author: author_name,
- ucid: ucid,
- video_count: videos.size,
- videos: videos,
- thumbnail: "https://i.ytimg.com/vi/#{videos[0].id}/mqdefault.jpg"
- )
- end
+ channel_v2_response = initial_data
+ .try &.["response"]?
+ .try &.["continuationContents"]?
+ .try &.["gridContinuation"]?
+ .try &.["items"]?
+
+ if channel_v2_response
+ channel_v2_response.try &.as_a.each { |item|
+ extract_item(item, author_fallback, author_id_fallback)
+ .try { |t| items << t }
+ }
+ else
+ initial_data.try { |t| t["contents"]? || t["response"]? }
+ .try { |t| t["twoColumnBrowseResultsRenderer"]?.try &.["tabs"].as_a.select(&.["tabRenderer"]?.try &.["selected"].as_bool)[0]?.try &.["tabRenderer"]["content"] ||
+ t["twoColumnSearchResultsRenderer"]?.try &.["primaryContents"] ||
+ t["continuationContents"]? }
+ .try { |t| t["sectionListRenderer"]? || t["sectionListContinuation"]? }
+ .try &.["contents"].as_a
+ .each { |c| c.try &.["itemSectionRenderer"]?.try &.["contents"].as_a
+ .try { |t| t[0]?.try &.["shelfRenderer"]?.try &.["content"]["expandedShelfContentsRenderer"]?.try &.["items"].as_a ||
+ t[0]?.try &.["gridRenderer"]?.try &.["items"].as_a || t }
+ .each { |item|
+ extract_item(item, author_fallback, author_id_fallback)
+ .try { |t| items << t }
+ } }
end
- return items
+ items
end
def check_enum(db, logger, enum_name, struct_type = nil)
+ return # TODO
+
if !db.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool)
logger.puts("CREATE TYPE #{enum_name}")
@@ -642,18 +354,14 @@ def check_table(db, logger, table_name, struct_type = nil)
end
end
- if !struct_type
- return
- end
+ return if !struct_type
- struct_array = struct_type.to_type_tuple
+ struct_array = struct_type.type_array
column_array = get_column_array(db, table_name)
column_types = File.read("config/sql/#{table_name}.sql").match(/CREATE TABLE public\.#{table_name}\n\((?<types>[\d\D]*?)\);/)
- .try &.["types"].split(",").map { |line| line.strip }
+ .try &.["types"].split(",").map { |line| line.strip }.reject &.starts_with?("CONSTRAINT")
- if !column_types
- return
- end
+ return if !column_types
struct_array.each_with_index do |name, i|
if name != column_array[i]?
@@ -704,6 +412,15 @@ def check_table(db, logger, table_name, struct_type = nil)
end
end
end
+
+ return if column_array.size <= struct_array.size
+
+ column_array.each do |column|
+ if !struct_array.includes? column
+ logger.puts("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE")
+ db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE")
+ end
+ end
end
class PG::ResultSet
@@ -732,9 +449,7 @@ def cache_annotation(db, id, annotations)
body = XML.parse(annotations)
nodeset = body.xpath_nodes(%q(/document/annotations/annotation))
- if nodeset == 0
- return
- end
+ return if nodeset == 0
has_legacy_annotations = false
nodeset.each do |node|
@@ -744,13 +459,10 @@ def cache_annotation(db, id, annotations)
end
end
- if has_legacy_annotations
- # TODO: Update on conflict?
- db.exec("INSERT INTO annotations VALUES ($1, $2) ON CONFLICT DO NOTHING", id, annotations)
- end
+ db.exec("INSERT INTO annotations VALUES ($1, $2) ON CONFLICT DO NOTHING", id, annotations) if has_legacy_annotations
end
-def create_notification_stream(env, config, kemal_config, decrypt_function, topics, connection_channel)
+def create_notification_stream(env, topics, connection_channel)
connection = Channel(PQ::Notification).new(8)
connection_channel.send({true, connection})
@@ -765,12 +477,12 @@ def create_notification_stream(env, config, kemal_config, decrypt_function, topi
loop do
time_span = [0, 0, 0, 0]
time_span[rand(4)] = rand(30) + 5
- published = Time.utc - Time::Span.new(time_span[0], time_span[1], time_span[2], time_span[3])
+ published = Time.utc - Time::Span.new(days: time_span[0], hours: time_span[1], minutes: time_span[2], seconds: time_span[3])
video_id = TEST_IDS[rand(TEST_IDS.size)]
video = get_video(video_id, PG_DB)
video.published = published
- response = JSON.parse(video.to_json(locale, config, kemal_config, decrypt_function))
+ response = JSON.parse(video.to_json(locale))
if fields_text = env.params.query["fields"]?
begin
@@ -804,7 +516,7 @@ def create_notification_stream(env, config, kemal_config, decrypt_function, topi
when .match(/UC[A-Za-z0-9_-]{22}/)
PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid = $1 AND published > $2 ORDER BY published DESC LIMIT 15",
topic, Time.unix(since.not_nil!), as: ChannelVideo).each do |video|
- response = JSON.parse(video.to_json(locale, config, Kemal.config))
+ response = JSON.parse(video.to_json(locale))
if fields_text = env.params.query["fields"]?
begin
@@ -846,7 +558,7 @@ def create_notification_stream(env, config, kemal_config, decrypt_function, topi
video = get_video(video_id, PG_DB)
video.published = Time.unix(published)
- response = JSON.parse(video.to_json(locale, config, Kemal.config, decrypt_function))
+ response = JSON.parse(video.to_json(locale))
if fields_text = env.params.query["fields"]?
begin
@@ -884,26 +596,46 @@ def create_notification_stream(env, config, kemal_config, decrypt_function, topi
end
end
-def extract_initial_data(body)
- initial_data = body.match(/window\["ytInitialData"\] = (?<info>.*?);\n/).try &.["info"] || "{}"
+def extract_initial_data(body) : Hash(String, JSON::Any)
+ initial_data = body.match(/(window\["ytInitialData"\]|var\s+ytInitialData)\s*=\s*(?<info>.*?);+\s*\n/).try &.["info"] || "{}"
if initial_data.starts_with?("JSON.parse(\"")
- return JSON.parse(JSON.parse(%({"initial_data":"#{initial_data[12..-3]}"}))["initial_data"].as_s)
+ return JSON.parse(JSON.parse(%({"initial_data":"#{initial_data[12..-3]}"}))["initial_data"].as_s).as_h
else
- return JSON.parse(initial_data)
+ return JSON.parse(initial_data).as_h
end
end
def proxy_file(response, env)
if response.headers.includes_word?("Content-Encoding", "gzip")
- Gzip::Writer.open(env.response) do |deflate|
- response.pipe(deflate)
+ Compress::Gzip::Writer.open(env.response) do |deflate|
+ IO.copy response.body_io, deflate
end
elsif response.headers.includes_word?("Content-Encoding", "deflate")
- Flate::Writer.open(env.response) do |deflate|
- response.pipe(deflate)
+ Compress::Deflate::Writer.open(env.response) do |deflate|
+ IO.copy response.body_io, deflate
end
else
- response.pipe(env.response)
+ IO.copy response.body_io, env.response
+ end
+end
+
+# See https://github.com/kemalcr/kemal/pull/576
+class HTTP::Server::Response::Output
+ def close
+ return if closed?
+
+ unless response.wrote_headers?
+ response.content_length = @out_count
+ end
+
+ ensure_headers_written
+
+ super
+
+ if @chunked
+ @io << "0\r\n\r\n"
+ @io.flush
+ end
end
end
diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr
index 4c9bb2d6..0faa2e32 100644
--- a/src/invidious/helpers/i18n.cr
+++ b/src/invidious/helpers/i18n.cr
@@ -24,6 +24,8 @@ def translate(locale : Hash(String, JSON::Any) | Nil, translation : String, text
if !locale[translation].as_s.empty?
translation = locale[translation].as_s
end
+ else
+ raise "Invalid translation #{translation}"
end
end
diff --git a/src/invidious/helpers/jobs.cr b/src/invidious/helpers/jobs.cr
deleted file mode 100644
index f368d6df..00000000
--- a/src/invidious/helpers/jobs.cr
+++ /dev/null
@@ -1,370 +0,0 @@
-def refresh_channels(db, logger, config)
- max_channel = Channel(Int32).new
-
- spawn do
- max_threads = max_channel.receive
- active_threads = 0
- active_channel = Channel(Bool).new
-
- loop do
- db.query("SELECT id FROM channels ORDER BY updated") do |rs|
- rs.each do
- id = rs.read(String)
-
- if active_threads >= max_threads
- if active_channel.receive
- active_threads -= 1
- end
- end
-
- active_threads += 1
- spawn do
- begin
- channel = fetch_channel(id, db, config.full_refresh)
-
- db.exec("UPDATE channels SET updated = $1, author = $2, deleted = false WHERE id = $3", Time.utc, channel.author, id)
- rescue ex
- if ex.message == "Deleted or invalid channel"
- db.exec("UPDATE channels SET updated = $1, deleted = true WHERE id = $2", Time.utc, id)
- end
- logger.puts("#{id} : #{ex.message}")
- end
-
- active_channel.send(true)
- end
- end
- end
-
- sleep 1.minute
- Fiber.yield
- end
- end
-
- max_channel.send(config.channel_threads)
-end
-
-def refresh_feeds(db, logger, config)
- max_channel = Channel(Int32).new
- spawn do
- max_threads = max_channel.receive
- active_threads = 0
- active_channel = Channel(Bool).new
-
- loop do
- db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs|
- rs.each do
- email = rs.read(String)
- view_name = "subscriptions_#{sha256(email)}"
-
- if active_threads >= max_threads
- if active_channel.receive
- active_threads -= 1
- end
- end
-
- active_threads += 1
- spawn do
- begin
- # Drop outdated views
- column_array = get_column_array(db, view_name)
- ChannelVideo.to_type_tuple.each_with_index do |name, i|
- if name != column_array[i]?
- logger.puts("DROP MATERIALIZED VIEW #{view_name}")
- db.exec("DROP MATERIALIZED VIEW #{view_name}")
- raise "view does not exist"
- end
- end
-
- if !db.query_one("SELECT pg_get_viewdef('#{view_name}')", as: String).includes? "WHERE ((cv.ucid = ANY (u.subscriptions))"
- logger.puts("Materialized view #{view_name} is out-of-date, recreating...")
- db.exec("DROP MATERIALIZED VIEW #{view_name}")
- end
-
- db.exec("REFRESH MATERIALIZED VIEW #{view_name}")
- db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
- rescue ex
- # Rename old views
- begin
- legacy_view_name = "subscriptions_#{sha256(email)[0..7]}"
-
- db.exec("SELECT * FROM #{legacy_view_name} LIMIT 0")
- logger.puts("RENAME MATERIALIZED VIEW #{legacy_view_name}")
- db.exec("ALTER MATERIALIZED VIEW #{legacy_view_name} RENAME TO #{view_name}")
- rescue ex
- begin
- # While iterating through, we may have an email stored from a deleted account
- if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool)
- logger.puts("CREATE #{view_name}")
- db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(email)}")
- db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
- end
- rescue ex
- logger.puts("REFRESH #{email} : #{ex.message}")
- end
- end
- end
-
- active_channel.send(true)
- end
- end
- end
-
- sleep 5.seconds
- Fiber.yield
- end
- end
-
- max_channel.send(config.feed_threads)
-end
-
-def subscribe_to_feeds(db, logger, key, config)
- if config.use_pubsub_feeds
- case config.use_pubsub_feeds
- when Bool
- max_threads = config.use_pubsub_feeds.as(Bool).to_unsafe
- when Int32
- max_threads = config.use_pubsub_feeds.as(Int32)
- end
- max_channel = Channel(Int32).new
-
- spawn do
- max_threads = max_channel.receive
- active_threads = 0
- active_channel = Channel(Bool).new
-
- loop do
- db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > interval '4 days' OR subscribed IS NULL") do |rs|
- rs.each do
- ucid = rs.read(String)
-
- if active_threads >= max_threads.as(Int32)
- if active_channel.receive
- active_threads -= 1
- end
- end
-
- active_threads += 1
-
- spawn do
- begin
- response = subscribe_pubsub(ucid, key, config)
-
- if response.status_code >= 400
- logger.puts("#{ucid} : #{response.body}")
- end
- rescue ex
- logger.puts("#{ucid} : #{ex.message}")
- end
-
- active_channel.send(true)
- end
- end
- end
-
- sleep 1.minute
- Fiber.yield
- end
- end
-
- max_channel.send(max_threads.as(Int32))
- end
-end
-
-def pull_top_videos(config, db)
- loop do
- begin
- top = rank_videos(db, 40)
- rescue ex
- sleep 1.minute
- Fiber.yield
-
- next
- end
-
- if top.size == 0
- sleep 1.minute
- Fiber.yield
-
- next
- end
-
- videos = [] of Video
-
- top.each do |id|
- begin
- videos << get_video(id, db)
- rescue ex
- next
- end
- end
-
- yield videos
-
- sleep 1.minute
- Fiber.yield
- end
-end
-
-def pull_popular_videos(db)
- loop do
- videos = db.query_all("SELECT DISTINCT ON (ucid) * FROM channel_videos WHERE ucid IN \
- (SELECT channel FROM (SELECT UNNEST(subscriptions) AS channel FROM users) AS d \
- GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40) \
- ORDER BY ucid, published DESC", as: ChannelVideo).sort_by { |video| video.published }.reverse
-
- yield videos
-
- sleep 1.minute
- Fiber.yield
- end
-end
-
-def update_decrypt_function
- loop do
- begin
- decrypt_function = fetch_decrypt_function
- yield decrypt_function
- rescue ex
- next
- ensure
- sleep 1.minute
- Fiber.yield
- end
- end
-end
-
-def bypass_captcha(captcha_key, logger)
- loop do
- begin
- response = YT_POOL.client &.get("/watch?v=CvFH_6DNRCY&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999")
- if response.body.includes?("To continue with your YouTube experience, please fill out the form below.")
- html = XML.parse_html(response.body)
- form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil!
- site_key = form.xpath_node(%(.//div[@class="g-recaptcha"])).try &.["data-sitekey"]
-
- inputs = {} of String => String
- form.xpath_nodes(%(.//input[@name])).map do |node|
- inputs[node["name"]] = node["value"]
- end
-
- headers = response.cookies.add_request_headers(HTTP::Headers.new)
-
- response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/createTask", body: {
- "clientKey" => CONFIG.captcha_key,
- "task" => {
- "type" => "NoCaptchaTaskProxyless",
- # "type" => "NoCaptchaTask",
- "websiteURL" => "https://www.youtube.com/watch?v=CvFH_6DNRCY&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999",
- "websiteKey" => site_key,
- # "proxyType" => "http",
- # "proxyAddress" => CONFIG.proxy_address,
- # "proxyPort" => CONFIG.proxy_port,
- # "proxyLogin" => CONFIG.proxy_user,
- # "proxyPassword" => CONFIG.proxy_pass,
- # "userAgent" => "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36",
- },
- }.to_json).body)
-
- if response["error"]?
- raise response["error"].as_s
- end
-
- task_id = response["taskId"].as_i
-
- loop do
- sleep 10.seconds
-
- response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/getTaskResult", body: {
- "clientKey" => CONFIG.captcha_key,
- "taskId" => task_id,
- }.to_json).body)
-
- if response["status"]?.try &.== "ready"
- break
- elsif response["errorId"]?.try &.as_i != 0
- raise response["errorDescription"].as_s
- end
- end
-
- inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
- response = YT_POOL.client &.post("/das_captcha", headers, form: inputs)
-
- yield response.cookies.select { |cookie| cookie.name != "PREF" }
- elsif response.headers["Location"]?.try &.includes?("/sorry/index")
- location = response.headers["Location"].try { |u| URI.parse(u) }
- client = QUIC::Client.new(location.host.not_nil!)
- response = client.get(location.full_path)
-
- html = XML.parse_html(response.body)
- form = html.xpath_node(%(//form[@action="index"])).not_nil!
- site_key = form.xpath_node(%(.//div[@class="g-recaptcha"])).try &.["data-sitekey"]
-
- inputs = {} of String => String
- form.xpath_nodes(%(.//input[@name])).map do |node|
- inputs[node["name"]] = node["value"]
- end
-
- response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/createTask", body: {
- "clientKey" => CONFIG.captcha_key,
- "task" => {
- "type" => "NoCaptchaTaskProxyless",
- "websiteURL" => location.to_s,
- "websiteKey" => site_key,
- },
- }.to_json).body)
-
- if response["error"]?
- raise response["error"].as_s
- end
-
- task_id = response["taskId"].as_i
-
- loop do
- sleep 10.seconds
-
- response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/getTaskResult", body: {
- "clientKey" => CONFIG.captcha_key,
- "taskId" => task_id,
- }.to_json).body)
-
- if response["status"]?.try &.== "ready"
- break
- elsif response["errorId"]?.try &.as_i != 0
- raise response["errorDescription"].as_s
- end
- end
-
- inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
- client.close
- client = QUIC::Client.new("www.google.com")
- response = client.post(location.full_path, form: inputs)
- headers = HTTP::Headers{
- "Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0],
- }
- cookies = HTTP::Cookies.from_headers(headers)
-
- yield cookies
- end
- rescue ex
- logger.puts("Exception: #{ex.message}")
- ensure
- sleep 1.minute
- Fiber.yield
- end
- end
-end
-
-def find_working_proxies(regions)
- loop do
- regions.each do |region|
- proxies = get_proxies(region).first(20)
- proxies = proxies.map { |proxy| {ip: proxy[:ip], port: proxy[:port]} }
- # proxies = filter_proxies(proxies)
-
- yield region, proxies
- end
-
- sleep 1.minute
- Fiber.yield
- end
-end
diff --git a/src/invidious/helpers/macros.cr b/src/invidious/helpers/macros.cr
index ddfb9f8e..8b74bc86 100644
--- a/src/invidious/helpers/macros.cr
+++ b/src/invidious/helpers/macros.cr
@@ -1,43 +1,51 @@
-macro db_mapping(mapping)
- def initialize({{*mapping.keys.map { |id| "@#{id}".id }}})
+module DB::Serializable
+ macro included
+ {% verbatim do %}
+ macro finished
+ def self.type_array
+ \{{ @type.instance_vars
+ .reject { |var| var.annotation(::DB::Field) && var.annotation(::DB::Field)[:ignore] }
+ .map { |name| name.stringify }
+ }}
+ end
+
+ def initialize(tuple)
+ \{% for var in @type.instance_vars %}
+ \{% ann = var.annotation(::DB::Field) %}
+ \{% if ann && ann[:ignore] %}
+ \{% else %}
+ @\{{var.name}} = tuple[:\{{var.name.id}}]
+ \{% end %}
+ \{% end %}
+ end
+
+ def to_a
+ \{{ @type.instance_vars
+ .reject { |var| var.annotation(::DB::Field) && var.annotation(::DB::Field)[:ignore] }
+ .map { |name| name }
+ }}
+ end
+ end
+ {% end %}
end
-
- def to_a
- return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ]
- end
-
- def self.to_type_tuple
- return { {{*mapping.keys.map { |id| "#{id}" }}} }
- end
-
- DB.mapping( {{mapping}} )
-end
-
-macro json_mapping(mapping)
- def initialize({{*mapping.keys.map { |id| "@#{id}".id }}})
- end
-
- def to_a
- return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ]
- end
-
- patched_json_mapping( {{mapping}} )
- YAML.mapping( {{mapping}} )
end
-macro yaml_mapping(mapping)
- def initialize({{*mapping.keys.map { |id| "@#{id}".id }}})
- end
-
- def to_a
- return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ]
- end
-
- def to_tuple
- return { {{*mapping.keys.map { |id| "@#{id}".id }}} }
+module JSON::Serializable
+ macro included
+ {% verbatim do %}
+ macro finished
+ def initialize(tuple)
+ \{% for var in @type.instance_vars %}
+ \{% ann = var.annotation(::JSON::Field) %}
+ \{% if ann && ann[:ignore] %}
+ \{% else %}
+ @\{{var.name}} = tuple[:\{{var.name.id}}]
+ \{% end %}
+ \{% end %}
+ end
+ end
+ {% end %}
end
-
- YAML.mapping({{mapping}})
end
macro templated(filename, template = "template")
diff --git a/src/invidious/helpers/patch_mapping.cr b/src/invidious/helpers/patch_mapping.cr
deleted file mode 100644
index 19bd8ca1..00000000
--- a/src/invidious/helpers/patch_mapping.cr
+++ /dev/null
@@ -1,166 +0,0 @@
-# Overloads https://github.com/crystal-lang/crystal/blob/0.28.0/src/json/from_json.cr#L24
-def Object.from_json(string_or_io, default) : self
- parser = JSON::PullParser.new(string_or_io)
- new parser, default
-end
-
-# Adds configurable 'default'
-macro patched_json_mapping(_properties_, strict = false)
- {% for key, value in _properties_ %}
- {% _properties_[key] = {type: value} unless value.is_a?(HashLiteral) || value.is_a?(NamedTupleLiteral) %}
- {% end %}
-
- {% for key, value in _properties_ %}
- {% _properties_[key][:key_id] = key.id.gsub(/\?$/, "") %}
- {% end %}
-
- {% for key, value in _properties_ %}
- @{{value[:key_id]}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }}
-
- {% if value[:setter] == nil ? true : value[:setter] %}
- def {{value[:key_id]}}=(_{{value[:key_id]}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }})
- @{{value[:key_id]}} = _{{value[:key_id]}}
- end
- {% end %}
-
- {% if value[:getter] == nil ? true : value[:getter] %}
- def {{key.id}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }}
- @{{value[:key_id]}}
- end
- {% end %}
-
- {% if value[:presence] %}
- @{{value[:key_id]}}_present : Bool = false
-
- def {{value[:key_id]}}_present?
- @{{value[:key_id]}}_present
- end
- {% end %}
- {% end %}
-
- def initialize(%pull : ::JSON::PullParser, default = nil)
- {% for key, value in _properties_ %}
- %var{key.id} = nil
- %found{key.id} = false
- {% end %}
-
- %location = %pull.location
- begin
- %pull.read_begin_object
- rescue exc : ::JSON::ParseException
- raise ::JSON::MappingError.new(exc.message, self.class.to_s, nil, *%location, exc)
- end
- until %pull.kind.end_object?
- %key_location = %pull.location
- key = %pull.read_object_key
- case key
- {% for key, value in _properties_ %}
- when {{value[:key] || value[:key_id].stringify}}
- %found{key.id} = true
- begin
- %var{key.id} =
- {% if value[:nilable] || value[:default] != nil %} %pull.read_null_or { {% end %}
-
- {% if value[:root] %}
- %pull.on_key!({{value[:root]}}) do
- {% end %}
-
- {% if value[:converter] %}
- {{value[:converter]}}.from_json(%pull)
- {% elsif value[:type].is_a?(Path) || value[:type].is_a?(Generic) %}
- {{value[:type]}}.new(%pull)
- {% else %}
- ::Union({{value[:type]}}).new(%pull)
- {% end %}
-
- {% if value[:root] %}
- end
- {% end %}
-
- {% if value[:nilable] || value[:default] != nil %} } {% end %}
- rescue exc : ::JSON::ParseException
- raise ::JSON::MappingError.new(exc.message, self.class.to_s, {{value[:key] || value[:key_id].stringify}}, *%key_location, exc)
- end
- {% end %}
- else
- {% if strict %}
- raise ::JSON::MappingError.new("Unknown JSON attribute: #{key}", self.class.to_s, nil, *%key_location, nil)
- {% else %}
- %pull.skip
- {% end %}
- end
- end
- %pull.read_next
-
- {% for key, value in _properties_ %}
- {% unless value[:nilable] || value[:default] != nil %}
- if %var{key.id}.nil? && !%found{key.id} && !::Union({{value[:type]}}).nilable?
- raise ::JSON::MappingError.new("Missing JSON attribute: {{(value[:key] || value[:key_id]).id}}", self.class.to_s, nil, *%location, nil)
- end
- {% end %}
-
- {% if value[:nilable] %}
- {% if value[:default] != nil %}
- @{{value[:key_id]}} = %found{key.id} ? %var{key.id} : (default.responds_to?(:{{value[:key_id]}}) ? default.{{value[:key_id]}} : {{value[:default]}})
- {% else %}
- @{{value[:key_id]}} = %var{key.id}
- {% end %}
- {% elsif value[:default] != nil %}
- @{{value[:key_id]}} = %var{key.id}.nil? ? (default.responds_to?(:{{value[:key_id]}}) ? default.{{value[:key_id]}} : {{value[:default]}}) : %var{key.id}
- {% else %}
- @{{value[:key_id]}} = (%var{key.id}).as({{value[:type]}})
- {% end %}
-
- {% if value[:presence] %}
- @{{value[:key_id]}}_present = %found{key.id}
- {% end %}
- {% end %}
- end
-
- def to_json(json : ::JSON::Builder)
- json.object do
- {% for key, value in _properties_ %}
- _{{value[:key_id]}} = @{{value[:key_id]}}
-
- {% unless value[:emit_null] %}
- unless _{{value[:key_id]}}.nil?
- {% end %}
-
- json.field({{value[:key] || value[:key_id].stringify}}) do
- {% if value[:root] %}
- {% if value[:emit_null] %}
- if _{{value[:key_id]}}.nil?
- nil.to_json(json)
- else
- {% end %}
-
- json.object do
- json.field({{value[:root]}}) do
- {% end %}
-
- {% if value[:converter] %}
- if _{{value[:key_id]}}
- {{ value[:converter] }}.to_json(_{{value[:key_id]}}, json)
- else
- nil.to_json(json)
- end
- {% else %}
- _{{value[:key_id]}}.to_json(json)
- {% end %}
-
- {% if value[:root] %}
- {% if value[:emit_null] %}
- end
- {% end %}
- end
- end
- {% end %}
- end
-
- {% unless value[:emit_null] %}
- end
- {% end %}
- {% end %}
- end
- end
-end
diff --git a/src/invidious/helpers/signatures.cr b/src/invidious/helpers/signatures.cr
index 1d238576..f811500f 100644
--- a/src/invidious/helpers/signatures.cr
+++ b/src/invidious/helpers/signatures.cr
@@ -1,69 +1,53 @@
+alias SigProc = Proc(Array(String), Int32, Array(String))
+
def fetch_decrypt_function(id = "CvFH_6DNRCY")
- document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1").body
- url = document.match(/src="(?<url>\/yts\/jsbin\/player_ias-.{9}\/en_US\/base.js)"/).not_nil!["url"]
+ document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en").body
+ url = document.match(/src="(?<url>\/s\/player\/[^\/]+\/player_ias[^\/]+\/en_US\/base.js)"/).not_nil!["url"]
player = YT_POOL.client &.get(url).body
- function_name = player.match(/^(?<name>[^=]+)=function\(a\){a=a\.split\(""\)/m).not_nil!["name"]
- function_body = player.match(/^#{Regex.escape(function_name)}=function\(a\){(?<body>[^}]+)}/m).not_nil!["body"]
+ function_name = player.match(/^(?<name>[^=]+)=function\(\w\){\w=\w\.split\(""\);[^\. ]+\.[^( ]+/m).not_nil!["name"]
+ function_body = player.match(/^#{Regex.escape(function_name)}=function\(\w\){(?<body>[^}]+)}/m).not_nil!["body"]
function_body = function_body.split(";")[1..-2]
var_name = function_body[0][0, 2]
var_body = player.delete("\n").match(/var #{Regex.escape(var_name)}={(?<body>(.*?))};/).not_nil!["body"]
- operations = {} of String => String
+ operations = {} of String => SigProc
var_body.split("},").each do |operation|
op_name = operation.match(/^[^:]+/).not_nil![0]
op_body = operation.match(/\{[^}]+/).not_nil![0]
case op_body
when "{a.reverse()"
- operations[op_name] = "a"
+ operations[op_name] = ->(a : Array(String), b : Int32) { a.reverse }
when "{a.splice(0,b)"
- operations[op_name] = "b"
+ operations[op_name] = ->(a : Array(String), b : Int32) { a.delete_at(0..(b - 1)); a }
else
- operations[op_name] = "c"
+ operations[op_name] = ->(a : Array(String), b : Int32) { c = a[0]; a[0] = a[b % a.size]; a[b % a.size] = c; a }
end
end
- decrypt_function = [] of {name: String, value: Int32}
+ decrypt_function = [] of {SigProc, Int32}
function_body.each do |function|
function = function.lchop(var_name).delete("[].")
op_name = function.match(/[^\(]+/).not_nil![0]
- value = function.match(/\(a,(?<value>[\d]+)\)/).not_nil!["value"].to_i
+ value = function.match(/\(\w,(?<value>[\d]+)\)/).not_nil!["value"].to_i
- decrypt_function << {name: operations[op_name], value: value}
+ decrypt_function << {operations[op_name], value}
end
return decrypt_function
end
-def decrypt_signature(fmt, code)
- if !fmt["s"]?
- return ""
- end
-
- a = fmt["s"]
- a = a.split("")
+def decrypt_signature(fmt : Hash(String, JSON::Any))
+ return "" if !fmt["s"]? || !fmt["sp"]?
- code.each do |item|
- case item[:name]
- when "a"
- a.reverse!
- when "b"
- a.delete_at(0..(item[:value] - 1))
- when "c"
- a = splice(a, item[:value])
- end
+ sp = fmt["sp"].as_s
+ sig = fmt["s"].as_s.split("")
+ DECRYPT_FUNCTION.each do |proc, value|
+ sig = proc.call(sig, value)
end
- signature = a.join("")
- return "&#{fmt["sp"]?}=#{signature}"
-end
-
-def splice(a, b)
- c = a[0]
- a[0] = a[b % a.size]
- a[b % a.size] = c
- return a
+ return "&#{sp}=#{sig.join("")}"
end
diff --git a/src/invidious/helpers/static_file_handler.cr b/src/invidious/helpers/static_file_handler.cr
index 20d92b9c..be9d36ab 100644
--- a/src/invidious/helpers/static_file_handler.cr
+++ b/src/invidious/helpers/static_file_handler.cr
@@ -81,12 +81,12 @@ def send_file(env : HTTP::Server::Context, file_path : String, data : Slice(UInt
condition = config.is_a?(Hash) && config["gzip"]? == true && filesize > minsize && Kemal::Utils.zip_types(file_path)
if condition && request_headers.includes_word?("Accept-Encoding", "gzip")
env.response.headers["Content-Encoding"] = "gzip"
- Gzip::Writer.open(env.response) do |deflate|
+ Compress::Gzip::Writer.open(env.response) do |deflate|
IO.copy(file, deflate)
end
elsif condition && request_headers.includes_word?("Accept-Encoding", "deflate")
env.response.headers["Content-Encoding"] = "deflate"
- Flate::Writer.open(env.response) do |deflate|
+ Compress::Deflate::Writer.open(env.response) do |deflate|
IO.copy(file, deflate)
end
else
diff --git a/src/invidious/helpers/tokens.cr b/src/invidious/helpers/tokens.cr
index 30f7d4f4..39aae367 100644
--- a/src/invidious/helpers/tokens.cr
+++ b/src/invidious/helpers/tokens.cr
@@ -1,3 +1,5 @@
+require "crypto/subtle"
+
def generate_token(email, scopes, expire, key, db)
session = "v1:#{Base64.urlsafe_encode(Random::Secure.random_bytes(32))}"
PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", session, email, Time.utc)
@@ -41,15 +43,10 @@ def sign_token(key, hash)
string_to_sign = [] of String
hash.each do |key, value|
- if key == "signature"
- next
- end
+ next if key == "signature"
- if value.is_a?(JSON::Any)
- case value
- when .as_a?
- value = value.as_a.map { |item| item.as_s }
- end
+ if value.is_a?(JSON::Any) && value.as_a?
+ value = value.as_a.map { |i| i.as_s }
end
case value
@@ -76,32 +73,31 @@ def validate_request(token, session, request, key, db, locale = nil)
raise translate(locale, "Hidden field \"token\" is a required field")
end
- if token["signature"] != sign_token(key, token)
- raise translate(locale, "Invalid signature")
+ expire = token["expire"]?.try &.as_i
+ if expire.try &.< Time.utc.to_unix
+ raise translate(locale, "Token is expired, please try again")
end
if token["session"] != session
raise translate(locale, "Erroneous token")
end
- if token["nonce"]? && (nonce = db.query_one?("SELECT * FROM nonces WHERE nonce = $1", token["nonce"], as: {String, Time}))
- if nonce[1] > Time.utc
- db.exec("UPDATE nonces SET expire = $1 WHERE nonce = $2", Time.utc(1990, 1, 1), nonce[0])
- else
- raise translate(locale, "Erroneous token")
- end
- end
-
scopes = token["scopes"].as_a.map { |v| v.as_s }
scope = "#{request.method}:#{request.path.lchop("/api/v1/auth/").lstrip("/")}"
-
if !scopes_include_scope(scopes, scope)
raise translate(locale, "Invalid scope")
end
- expire = token["expire"]?.try &.as_i
- if expire.try &.< Time.utc.to_unix
- raise translate(locale, "Token is expired, please try again")
+ if !Crypto::Subtle.constant_time_compare(token["signature"].to_s, sign_token(key, token))
+ raise translate(locale, "Invalid signature")
+ end
+
+ if token["nonce"]? && (nonce = db.query_one?("SELECT * FROM nonces WHERE nonce = $1", token["nonce"], as: {String, Time}))
+ if nonce[1] > Time.utc
+ db.exec("UPDATE nonces SET expire = $1 WHERE nonce = $2", Time.utc(1990, 1, 1), nonce[0])
+ else
+ raise translate(locale, "Erroneous token")
+ end
end
return {scopes, expire, token["signature"].as_s}
diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr
index 6fcfa8d2..a51f15ce 100644
--- a/src/invidious/helpers/utils.cr
+++ b/src/invidious/helpers/utils.cr
@@ -2,13 +2,16 @@ require "lsquic"
require "pool/connection"
def add_yt_headers(request)
- request.headers["x-youtube-client-name"] ||= "1"
- request.headers["x-youtube-client-version"] ||= "1.20180719"
request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36"
request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
request.headers["accept-language"] ||= "en-us,en;q=0.5"
- request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
+ return if request.resource.starts_with? "/sorry/index"
+ request.headers["x-youtube-client-name"] ||= "1"
+ request.headers["x-youtube-client-version"] ||= "2.20200609"
+ if !CONFIG.cookies.empty?
+ request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
+ end
end
struct QUICPool
@@ -77,7 +80,8 @@ def elapsed_text(elapsed)
end
def make_client(url : URI, region = nil)
- client = HTTPClient.new(url)
+ # TODO: Migrate any applicable endpoints to QUIC
+ client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure)
client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC
client.read_timeout = 10.seconds
client.connect_timeout = 10.seconds
@@ -99,7 +103,7 @@ end
def decode_length_seconds(string)
length_seconds = string.gsub(/[^0-9:]/, "").split(":").map &.to_i
length_seconds = [0] * (3 - length_seconds.size) + length_seconds
- length_seconds = Time::Span.new(length_seconds[0], length_seconds[1], length_seconds[2])
+ length_seconds = Time::Span.new hours: length_seconds[0], minutes: length_seconds[1], seconds: length_seconds[2]
length_seconds = length_seconds.total_seconds.to_i
return length_seconds
@@ -161,6 +165,7 @@ def decode_date(string : String)
return Time.utc
when "yesterday"
return Time.utc - 1.day
+ else nil # Continue
end
# String matches format "20 hours ago", "4 months ago"...
@@ -315,7 +320,7 @@ def get_referer(env, fallback = "/", unroll = true)
end
referer = referer.full_path
- referer = "/" + referer.lstrip("\/\\")
+ referer = "/" + referer.gsub(/[^\/?@&%=\-_.0-9a-zA-Z]/, "").lstrip("/\\")
if referer == env.request.path
referer = fallback
@@ -324,47 +329,10 @@ def get_referer(env, fallback = "/", unroll = true)
return referer
end
-struct VarInt
- def self.from_io(io : IO, format = IO::ByteFormat::NetworkEndian) : Int32
- result = 0_u32
- num_read = 0
-
- loop do
- byte = io.read_byte
- raise "Invalid VarInt" if !byte
- value = byte & 0x7f
-
- result |= value.to_u32 << (7 * num_read)
- num_read += 1
-
- break if byte & 0x80 == 0
- raise "Invalid VarInt" if num_read > 5
- end
-
- result.to_i32
- end
-
- def self.to_io(io : IO, value : Int32)
- io.write_byte 0x00 if value == 0x00
- value = value.to_u32
-
- while value != 0
- byte = (value & 0x7f).to_u8
- value >>= 7
-
- if value != 0
- byte |= 0x80
- end
-
- io.write_byte byte
- end
- end
-end
-
def sha256(text)
digest = OpenSSL::Digest.new("SHA256")
digest << text
- return digest.hexdigest
+ return digest.final.hexstring
end
def subscribe_pubsub(topic, key, config)
@@ -383,10 +351,8 @@ def subscribe_pubsub(topic, key, config)
nonce = Random::Secure.hex(4)
signature = "#{time}:#{nonce}"
- host_url = make_host_url(config, Kemal.config)
-
body = {
- "hub.callback" => "#{host_url}/feed/webhook/v1:#{time}:#{nonce}:#{OpenSSL::HMAC.hexdigest(:sha1, key, signature)}",
+ "hub.callback" => "#{HOST_URL}/feed/webhook/v1:#{time}:#{nonce}:#{OpenSSL::HMAC.hexdigest(:sha1, key, signature)}",
"hub.topic" => "https://www.youtube.com/xml/feeds/videos.xml?#{topic}",
"hub.verify" => "async",
"hub.mode" => "subscribe",
diff --git a/src/invidious/jobs.cr b/src/invidious/jobs.cr
new file mode 100644
index 00000000..ec0cad64
--- /dev/null
+++ b/src/invidious/jobs.cr
@@ -0,0 +1,13 @@
+module Invidious::Jobs
+ JOBS = [] of BaseJob
+
+ def self.register(job : BaseJob)
+ JOBS << job
+ end
+
+ def self.start_all
+ JOBS.each do |job|
+ spawn { job.begin }
+ end
+ end
+end
diff --git a/src/invidious/jobs/base_job.cr b/src/invidious/jobs/base_job.cr
new file mode 100644
index 00000000..47e75864
--- /dev/null
+++ b/src/invidious/jobs/base_job.cr
@@ -0,0 +1,3 @@
+abstract class Invidious::Jobs::BaseJob
+ abstract def begin
+end
diff --git a/src/invidious/jobs/bypass_captcha_job.cr b/src/invidious/jobs/bypass_captcha_job.cr
new file mode 100644
index 00000000..8b69e01a
--- /dev/null
+++ b/src/invidious/jobs/bypass_captcha_job.cr
@@ -0,0 +1,131 @@
+class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob
+ private getter logger : Invidious::LogHandler
+ private getter config : Config
+
+ def initialize(@logger, @config)
+ end
+
+ def begin
+ loop do
+ begin
+ {"/watch?v=jNQXAC9IVRw&gl=US&hl=en&has_verified=1&bpctr=9999999999", produce_channel_videos_url(ucid: "UC4QobU6STFB0P71PMvOGN5A")}.each do |path|
+ response = YT_POOL.client &.get(path)
+ if response.body.includes?("To continue with your YouTube experience, please fill out the form below.")
+ html = XML.parse_html(response.body)
+ form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil!
+ site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"]
+ s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"]
+
+ inputs = {} of String => String
+ form.xpath_nodes(%(.//input[@name])).map do |node|
+ inputs[node["name"]] = node["value"]
+ end
+
+ headers = response.cookies.add_request_headers(HTTP::Headers.new)
+
+ response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/createTask", body: {
+ "clientKey" => config.captcha_key,
+ "task" => {
+ "type" => "NoCaptchaTaskProxyless",
+ "websiteURL" => "https://www.youtube.com#{path}",
+ "websiteKey" => site_key,
+ "recaptchaDataSValue" => s_value,
+ },
+ }.to_json).body)
+
+ raise response["error"].as_s if response["error"]?
+ task_id = response["taskId"].as_i
+
+ loop do
+ sleep 10.seconds
+
+ response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/getTaskResult", body: {
+ "clientKey" => config.captcha_key,
+ "taskId" => task_id,
+ }.to_json).body)
+
+ if response["status"]?.try &.== "ready"
+ break
+ elsif response["errorId"]?.try &.as_i != 0
+ raise response["errorDescription"].as_s
+ end
+ end
+
+ inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
+ headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
+ response = YT_POOL.client &.post("/das_captcha", headers, form: inputs)
+
+ response.cookies
+ .select { |cookie| cookie.name != "PREF" }
+ .each { |cookie| config.cookies << cookie }
+
+ # Persist cookies between runs
+ File.write("config/config.yml", config.to_yaml)
+ elsif response.headers["Location"]?.try &.includes?("/sorry/index")
+ location = response.headers["Location"].try { |u| URI.parse(u) }
+ headers = HTTP::Headers{":authority" => location.host.not_nil!}
+ response = YT_POOL.client &.get(location.full_path, headers)
+
+ html = XML.parse_html(response.body)
+ form = html.xpath_node(%(//form[@action="index"])).not_nil!
+ site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"]
+ s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"]
+
+ inputs = {} of String => String
+ form.xpath_nodes(%(.//input[@name])).map do |node|
+ inputs[node["name"]] = node["value"]
+ end
+
+ captcha_client = HTTPClient.new(URI.parse("https://api.anti-captcha.com"))
+ captcha_client.family = config.force_resolve || Socket::Family::INET
+ response = JSON.parse(captcha_client.post("/createTask", body: {
+ "clientKey" => config.captcha_key,
+ "task" => {
+ "type" => "NoCaptchaTaskProxyless",
+ "websiteURL" => location.to_s,
+ "websiteKey" => site_key,
+ "recaptchaDataSValue" => s_value,
+ },
+ }.to_json).body)
+
+ raise response["error"].as_s if response["error"]?
+ task_id = response["taskId"].as_i
+
+ loop do
+ sleep 10.seconds
+
+ response = JSON.parse(captcha_client.post("/getTaskResult", body: {
+ "clientKey" => config.captcha_key,
+ "taskId" => task_id,
+ }.to_json).body)
+
+ if response["status"]?.try &.== "ready"
+ break
+ elsif response["errorId"]?.try &.as_i != 0
+ raise response["errorDescription"].as_s
+ end
+ end
+
+ inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
+ headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
+ response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs)
+ headers = HTTP::Headers{
+ "Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0],
+ }
+ cookies = HTTP::Cookies.from_headers(headers)
+
+ cookies.each { |cookie| config.cookies << cookie }
+
+ # Persist cookies between runs
+ File.write("config/config.yml", config.to_yaml)
+ end
+ end
+ rescue ex
+ logger.puts("Exception: #{ex.message}")
+ ensure
+ sleep 1.minute
+ Fiber.yield
+ end
+ end
+ end
+end
diff --git a/src/invidious/jobs/notification_job.cr b/src/invidious/jobs/notification_job.cr
new file mode 100644
index 00000000..2f525e08
--- /dev/null
+++ b/src/invidious/jobs/notification_job.cr
@@ -0,0 +1,24 @@
+class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob
+ private getter connection_channel : Channel({Bool, Channel(PQ::Notification)})
+ private getter pg_url : URI
+
+ def initialize(@connection_channel, @pg_url)
+ end
+
+ def begin
+ connections = [] of Channel(PQ::Notification)
+
+ PG.connect_listen(pg_url, "notifications") { |event| connections.each(&.send(event)) }
+
+ loop do
+ action, connection = connection_channel.receive
+
+ case action
+ when true
+ connections << connection
+ when false
+ connections.delete(connection)
+ end
+ end
+ end
+end
diff --git a/src/invidious/jobs/pull_popular_videos_job.cr b/src/invidious/jobs/pull_popular_videos_job.cr
new file mode 100644
index 00000000..7a8ab84e
--- /dev/null
+++ b/src/invidious/jobs/pull_popular_videos_job.cr
@@ -0,0 +1,27 @@
+class Invidious::Jobs::PullPopularVideosJob < Invidious::Jobs::BaseJob
+ QUERY = <<-SQL
+ SELECT DISTINCT ON (ucid) *
+ FROM channel_videos
+ WHERE ucid IN (SELECT channel FROM (SELECT UNNEST(subscriptions) AS channel FROM users) AS d
+ GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40)
+ ORDER BY ucid, published DESC
+ SQL
+ POPULAR_VIDEOS = Atomic.new([] of ChannelVideo)
+ private getter db : DB::Database
+
+ def initialize(@db)
+ end
+
+ def begin
+ loop do
+ videos = db.query_all(QUERY, as: ChannelVideo)
+ .sort_by(&.published)
+ .reverse
+
+ POPULAR_VIDEOS.set(videos)
+
+ sleep 1.minute
+ Fiber.yield
+ end
+ end
+end
diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr
new file mode 100644
index 00000000..75fc474d
--- /dev/null
+++ b/src/invidious/jobs/refresh_channels_job.cr
@@ -0,0 +1,59 @@
+class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
+ private getter db : DB::Database
+ private getter logger : Invidious::LogHandler
+ private getter config : Config
+
+ def initialize(@db, @logger, @config)
+ end
+
+ def begin
+ max_threads = config.channel_threads
+ lim_threads = max_threads
+ active_threads = 0
+ active_channel = Channel(Bool).new
+ backoff = 1.seconds
+
+ loop do
+ db.query("SELECT id FROM channels ORDER BY updated") do |rs|
+ rs.each do
+ id = rs.read(String)
+
+ if active_threads >= lim_threads
+ if active_channel.receive
+ active_threads -= 1
+ end
+ end
+
+ active_threads += 1
+ spawn do
+ begin
+ channel = fetch_channel(id, db, config.full_refresh)
+
+ lim_threads = max_threads
+ db.exec("UPDATE channels SET updated = $1, author = $2, deleted = false WHERE id = $3", Time.utc, channel.author, id)
+ rescue ex
+ logger.puts("#{id} : #{ex.message}")
+ if ex.message == "Deleted or invalid channel"
+ db.exec("UPDATE channels SET updated = $1, deleted = true WHERE id = $2", Time.utc, id)
+ else
+ lim_threads = 1
+ logger.puts("#{id} : backing off for #{backoff}s")
+ sleep backoff
+ if backoff < 1.days
+ backoff += backoff
+ else
+ backoff = 1.days
+ end
+ end
+ end
+
+ active_channel.send(true)
+ end
+ end
+ end
+
+ sleep 1.minute
+ Fiber.yield
+ end
+ end
+end
diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr
new file mode 100644
index 00000000..eebdf0f3
--- /dev/null
+++ b/src/invidious/jobs/refresh_feeds_job.cr
@@ -0,0 +1,77 @@
+class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
+ private getter db : DB::Database
+ private getter logger : Invidious::LogHandler
+ private getter config : Config
+
+ def initialize(@db, @logger, @config)
+ end
+
+ def begin
+ max_threads = config.feed_threads
+ active_threads = 0
+ active_channel = Channel(Bool).new
+
+ loop do
+ db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs|
+ rs.each do
+ email = rs.read(String)
+ view_name = "subscriptions_#{sha256(email)}"
+
+ if active_threads >= max_threads
+ if active_channel.receive
+ active_threads -= 1
+ end
+ end
+
+ active_threads += 1
+ spawn do
+ begin
+ # Drop outdated views
+ column_array = get_column_array(db, view_name)
+ ChannelVideo.type_array.each_with_index do |name, i|
+ if name != column_array[i]?
+ logger.puts("DROP MATERIALIZED VIEW #{view_name}")
+ db.exec("DROP MATERIALIZED VIEW #{view_name}")
+ raise "view does not exist"
+ end
+ end
+
+ if !db.query_one("SELECT pg_get_viewdef('#{view_name}')", as: String).includes? "WHERE ((cv.ucid = ANY (u.subscriptions))"
+ logger.puts("Materialized view #{view_name} is out-of-date, recreating...")
+ db.exec("DROP MATERIALIZED VIEW #{view_name}")
+ end
+
+ db.exec("REFRESH MATERIALIZED VIEW #{view_name}")
+ db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
+ rescue ex
+ # Rename old views
+ begin
+ legacy_view_name = "subscriptions_#{sha256(email)[0..7]}"
+
+ db.exec("SELECT * FROM #{legacy_view_name} LIMIT 0")
+ logger.puts("RENAME MATERIALIZED VIEW #{legacy_view_name}")
+ db.exec("ALTER MATERIALIZED VIEW #{legacy_view_name} RENAME TO #{view_name}")
+ rescue ex
+ begin
+ # While iterating through, we may have an email stored from a deleted account
+ if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool)
+ logger.puts("CREATE #{view_name}")
+ db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(email)}")
+ db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
+ end
+ rescue ex
+ logger.puts("REFRESH #{email} : #{ex.message}")
+ end
+ end
+ end
+
+ active_channel.send(true)
+ end
+ end
+ end
+
+ sleep 5.seconds
+ Fiber.yield
+ end
+ end
+end
diff --git a/src/invidious/jobs/statistics_refresh_job.cr b/src/invidious/jobs/statistics_refresh_job.cr
new file mode 100644
index 00000000..021671be
--- /dev/null
+++ b/src/invidious/jobs/statistics_refresh_job.cr
@@ -0,0 +1,59 @@
+class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
+ STATISTICS = {
+ "version" => "2.0",
+ "software" => {
+ "name" => "invidious",
+ "version" => "",
+ "branch" => "",
+ },
+ "openRegistrations" => true,
+ "usage" => {
+ "users" => {
+ "total" => 0_i64,
+ "activeHalfyear" => 0_i64,
+ "activeMonth" => 0_i64,
+ },
+ },
+ "metadata" => {
+ "updatedAt" => Time.utc.to_unix,
+ "lastChannelRefreshedAt" => 0_i64,
+ },
+ }
+
+ private getter db : DB::Database
+ private getter config : Config
+
+ def initialize(@db, @config, @software_config : Hash(String, String))
+ end
+
+ def begin
+ load_initial_stats
+
+ loop do
+ refresh_stats
+ sleep 1.minute
+ Fiber.yield
+ end
+ end
+
+ # should only be called once at the very beginning
+ private def load_initial_stats
+ STATISTICS["software"] = {
+ "name" => @software_config["name"],
+ "version" => @software_config["version"],
+ "branch" => @software_config["branch"],
+ }
+ STATISTICS["openRegistration"] = config.registration_enabled
+ end
+
+ private def refresh_stats
+ users = STATISTICS.dig("usage", "users").as(Hash(String, Int64))
+ users["total"] = db.query_one("SELECT count(*) FROM users", as: Int64)
+ users["activeHalfyear"] = db.query_one("SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '6 months'", as: Int64)
+ users["activeMonth"] = db.query_one("SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '1 month'", as: Int64)
+ STATISTICS["metadata"] = {
+ "updatedAt" => Time.utc.to_unix,
+ "lastChannelRefreshedAt" => db.query_one?("SELECT updated FROM channels ORDER BY updated DESC LIMIT 1", as: Time).try &.to_unix || 0_i64,
+ }
+ end
+end
diff --git a/src/invidious/jobs/subscribe_to_feeds_job.cr b/src/invidious/jobs/subscribe_to_feeds_job.cr
new file mode 100644
index 00000000..3d3b2218
--- /dev/null
+++ b/src/invidious/jobs/subscribe_to_feeds_job.cr
@@ -0,0 +1,52 @@
+class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob
+ private getter db : DB::Database
+ private getter logger : Invidious::LogHandler
+ private getter hmac_key : String
+ private getter config : Config
+
+ def initialize(@db, @logger, @config, @hmac_key)
+ end
+
+ def begin
+ max_threads = 1
+ if config.use_pubsub_feeds.is_a?(Int32)
+ max_threads = config.use_pubsub_feeds.as(Int32)
+ end
+
+ active_threads = 0
+ active_channel = Channel(Bool).new
+
+ loop do
+ db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > interval '4 days' OR subscribed IS NULL") do |rs|
+ rs.each do
+ ucid = rs.read(String)
+
+ if active_threads >= max_threads.as(Int32)
+ if active_channel.receive
+ active_threads -= 1
+ end
+ end
+
+ active_threads += 1
+
+ spawn do
+ begin
+ response = subscribe_pubsub(ucid, hmac_key, config)
+
+ if response.status_code >= 400
+ logger.puts("#{ucid} : #{response.body}")
+ end
+ rescue ex
+ logger.puts("#{ucid} : #{ex.message}")
+ end
+
+ active_channel.send(true)
+ end
+ end
+ end
+
+ sleep 1.minute
+ Fiber.yield
+ end
+ end
+end
diff --git a/src/invidious/jobs/update_decrypt_function_job.cr b/src/invidious/jobs/update_decrypt_function_job.cr
new file mode 100644
index 00000000..5332c672
--- /dev/null
+++ b/src/invidious/jobs/update_decrypt_function_job.cr
@@ -0,0 +1,19 @@
+class Invidious::Jobs::UpdateDecryptFunctionJob < Invidious::Jobs::BaseJob
+ DECRYPT_FUNCTION = [] of {SigProc, Int32}
+
+ def begin
+ loop do
+ begin
+ decrypt_function = fetch_decrypt_function
+ DECRYPT_FUNCTION.clear
+ decrypt_function.each { |df| DECRYPT_FUNCTION << df }
+ rescue ex
+ # TODO: Log error
+ next
+ ensure
+ sleep 1.minute
+ Fiber.yield
+ end
+ end
+ end
+end
diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr
index 04a37b87..c69eb0c4 100644
--- a/src/invidious/mixes.cr
+++ b/src/invidious/mixes.cr
@@ -1,32 +1,32 @@
struct MixVideo
- db_mapping({
- title: String,
- id: String,
- author: String,
- ucid: String,
- length_seconds: Int32,
- index: Int32,
- rdid: String,
- })
+ include DB::Serializable
+
+ property title : String
+ property id : String
+ property author : String
+ property ucid : String
+ property length_seconds : Int32
+ property index : Int32
+ property rdid : String
end
struct Mix
- db_mapping({
- title: String,
- id: String,
- videos: Array(MixVideo),
- })
+ include DB::Serializable
+
+ property title : String
+ property id : String
+ property videos : Array(MixVideo)
end
def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
headers = HTTP::Headers.new
- headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"
if cookies
headers = cookies.add_request_headers(headers)
end
- response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en&has_verified=1&bpctr=9999999999", headers)
+ video_id = "CvFH_6DNRCY" if rdid.starts_with? "OLAK5uy_"
+ response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en", headers)
initial_data = extract_initial_data(response.body)
if !initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]?
@@ -49,23 +49,22 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
id = item["videoId"].as_s
title = item["title"]?.try &.["simpleText"].as_s
- if !title
- next
- end
+ next if !title
+
author = item["longBylineText"]["runs"][0]["text"].as_s
ucid = item["longBylineText"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"].as_s
length_seconds = decode_length_seconds(item["lengthText"]["simpleText"].as_s)
index = item["navigationEndpoint"]["watchEndpoint"]["index"].as_i
- videos << MixVideo.new(
- title,
- id,
- author,
- ucid,
- length_seconds,
- index,
- rdid
- )
+ videos << MixVideo.new({
+ title: title,
+ id: id,
+ author: author,
+ ucid: ucid,
+ length_seconds: length_seconds,
+ index: index,
+ rdid: rdid,
+ })
end
if !cookies
@@ -75,7 +74,11 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
videos.uniq! { |video| video.id }
videos = videos.first(50)
- return Mix.new(mix_title, rdid, videos)
+ return Mix.new({
+ title: mix_title,
+ id: rdid,
+ videos: videos,
+ })
end
def template_mix(mix)
diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr
index 9c8afd3c..c984a12a 100644
--- a/src/invidious/playlists.cr
+++ b/src/invidious/playlists.cr
@@ -1,26 +1,38 @@
struct PlaylistVideo
- def to_xml(host_url, auto_generated, xml : XML::Builder)
+ include DB::Serializable
+
+ property title : String
+ property id : String
+ property author : String
+ property ucid : String
+ property length_seconds : Int32
+ property published : Time
+ property plid : String
+ property index : Int64
+ property live_now : Bool
+
+ def to_xml(auto_generated, xml : XML::Builder)
xml.element("entry") do
xml.element("id") { xml.text "yt:video:#{self.id}" }
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
- xml.element("link", rel: "alternate", href: "#{host_url}/watch?v=#{self.id}")
+ xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?v=#{self.id}")
xml.element("author") do
if auto_generated
xml.element("name") { xml.text self.author }
- xml.element("uri") { xml.text "#{host_url}/channel/#{self.ucid}" }
+ xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
else
xml.element("name") { xml.text author }
- xml.element("uri") { xml.text "#{host_url}/channel/#{ucid}" }
+ xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
end
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
- xml.element("a", href: "#{host_url}/watch?v=#{self.id}") do
- xml.element("img", src: "#{host_url}/vi/#{self.id}/mqdefault.jpg")
+ xml.element("a", href: "#{HOST_URL}/watch?v=#{self.id}") do
+ xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
end
end
@@ -29,23 +41,23 @@ struct PlaylistVideo
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
- xml.element("media:thumbnail", url: "#{host_url}/vi/#{self.id}/mqdefault.jpg",
+ xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
end
end
end
- def to_xml(host_url, auto_generated, xml : XML::Builder? = nil)
+ def to_xml(auto_generated, xml : XML::Builder? = nil)
if xml
- to_xml(host_url, auto_generated, xml)
+ to_xml(auto_generated, xml)
else
XML.build do |json|
- to_xml(host_url, auto_generated, xml)
+ to_xml(auto_generated, xml)
end
end
end
- def to_json(locale, config, kemal_config, json : JSON::Builder, index : Int32?)
+ def to_json(locale, json : JSON::Builder, index : Int32?)
json.object do
json.field "title", self.title
json.field "videoId", self.id
@@ -55,7 +67,7 @@ struct PlaylistVideo
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
- generate_thumbnails(json, self.id, config, kemal_config)
+ generate_thumbnails(json, self.id)
end
if index
@@ -69,31 +81,32 @@ struct PlaylistVideo
end
end
- def to_json(locale, config, kemal_config, json : JSON::Builder? = nil, index : Int32? = nil)
+ def to_json(locale, json : JSON::Builder? = nil, index : Int32? = nil)
if json
- to_json(locale, config, kemal_config, json, index: index)
+ to_json(locale, json, index: index)
else
JSON.build do |json|
- to_json(locale, config, kemal_config, json, index: index)
+ to_json(locale, json, index: index)
end
end
end
-
- db_mapping({
- title: String,
- id: String,
- author: String,
- ucid: String,
- length_seconds: Int32,
- published: Time,
- plid: String,
- index: Int64,
- live_now: Bool,
- })
end
struct Playlist
- def to_json(offset, locale, config, kemal_config, json : JSON::Builder, continuation : String? = nil)
+ include DB::Serializable
+
+ property title : String
+ property id : String
+ property author : String
+ property author_thumbnail : String
+ property ucid : String
+ property description : String
+ property video_count : Int32
+ property views : Int64
+ property updated : Time
+ property thumbnail : String?
+
+ def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
json.object do
json.field "type", "playlist"
json.field "title", self.title
@@ -118,7 +131,7 @@ struct Playlist
end
end
- json.field "description", html_to_content(self.description_html)
+ json.field "description", self.description
json.field "descriptionHtml", self.description_html
json.field "videoCount", self.video_count
@@ -130,39 +143,30 @@ struct Playlist
json.array do
videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation)
videos.each_with_index do |video, index|
- video.to_json(locale, config, Kemal.config, json)
+ video.to_json(locale, json)
end
end
end
end
end
- def to_json(offset, locale, config, kemal_config, json : JSON::Builder? = nil, continuation : String? = nil)
+ def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil)
if json
- to_json(offset, locale, config, kemal_config, json, continuation: continuation)
+ to_json(offset, locale, json, continuation: continuation)
else
JSON.build do |json|
- to_json(offset, locale, config, kemal_config, json, continuation: continuation)
+ to_json(offset, locale, json, continuation: continuation)
end
end
end
- db_mapping({
- title: String,
- id: String,
- author: String,
- author_thumbnail: String,
- ucid: String,
- description_html: String,
- video_count: Int32,
- views: Int64,
- updated: Time,
- thumbnail: String?,
- })
-
def privacy
PlaylistPrivacy::Public
end
+
+ def description_html
+ HTML.escape(self.description).gsub("\n", "<br>")
+ end
end
enum PlaylistPrivacy
@@ -172,7 +176,30 @@ enum PlaylistPrivacy
end
struct InvidiousPlaylist
- def to_json(offset, locale, config, kemal_config, json : JSON::Builder, continuation : String? = nil)
+ include DB::Serializable
+
+ property title : String
+ property id : String
+ property author : String
+ property description : String = ""
+ property video_count : Int32
+ property created : Time
+ property updated : Time
+
+ @[DB::Field(converter: InvidiousPlaylist::PlaylistPrivacyConverter)]
+ property privacy : PlaylistPrivacy = PlaylistPrivacy::Private
+ property index : Array(Int64)
+
+ @[DB::Field(ignore: true)]
+ property thumbnail_id : String?
+
+ module PlaylistPrivacyConverter
+ def self.from_rs(rs)
+ return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8))))
+ end
+ end
+
+ def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
json.object do
json.field "type", "invidiousPlaylist"
json.field "title", self.title
@@ -195,43 +222,23 @@ struct InvidiousPlaylist
json.array do
videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation)
videos.each_with_index do |video, index|
- video.to_json(locale, config, Kemal.config, json, offset + index)
+ video.to_json(locale, json, offset + index)
end
end
end
end
end
- def to_json(offset, locale, config, kemal_config, json : JSON::Builder? = nil, continuation : String? = nil)
+ def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil)
if json
- to_json(offset, locale, config, kemal_config, json, continuation: continuation)
+ to_json(offset, locale, json, continuation: continuation)
else
JSON.build do |json|
- to_json(offset, locale, config, kemal_config, json, continuation: continuation)
+ to_json(offset, locale, json, continuation: continuation)
end
end
end
- property thumbnail_id
-
- module PlaylistPrivacyConverter
- def self.from_rs(rs)
- return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8))))
- end
- end
-
- db_mapping({
- title: String,
- id: String,
- author: String,
- description: {type: String, default: ""},
- video_count: Int32,
- created: Time,
- updated: Time,
- privacy: {type: PlaylistPrivacy, default: PlaylistPrivacy::Private, converter: PlaylistPrivacyConverter},
- index: Array(Int64),
- })
-
def thumbnail
@thumbnail_id ||= PG_DB.query_one?("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 1", self.id, self.index, as: String) || "-----------"
"/vi/#{@thumbnail_id}/mqdefault.jpg"
@@ -257,17 +264,17 @@ end
def create_playlist(db, title, privacy, user)
plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}"
- playlist = InvidiousPlaylist.new(
- title: title.byte_slice(0, 150),
- id: plid,
- author: user.email,
+ playlist = InvidiousPlaylist.new({
+ title: title.byte_slice(0, 150),
+ id: plid,
+ author: user.email,
description: "", # Max 5000 characters
video_count: 0,
- created: Time.utc,
- updated: Time.utc,
- privacy: privacy,
- index: [] of Int64,
- )
+ created: Time.utc,
+ updated: Time.utc,
+ privacy: privacy,
+ index: [] of Int64,
+ })
playlist_array = playlist.to_a
args = arg_array(playlist_array)
@@ -277,50 +284,25 @@ def create_playlist(db, title, privacy, user)
return playlist
end
-def extract_playlist(plid, nodeset, index)
- videos = [] of PlaylistVideo
-
- nodeset.each_with_index do |video, offset|
- anchor = video.xpath_node(%q(.//td[@class="pl-video-title"]))
- if !anchor
- next
- end
-
- title = anchor.xpath_node(%q(.//a)).not_nil!.content.strip(" \n")
- id = anchor.xpath_node(%q(.//a)).not_nil!["href"].lchop("/watch?v=")[0, 11]
-
- anchor = anchor.xpath_node(%q(.//div[@class="pl-video-owner"]/a))
- if anchor
- author = anchor.content
- ucid = anchor["href"].split("/")[2]
- else
- author = ""
- ucid = ""
- end
+def subscribe_playlist(db, user, playlist)
+ playlist = InvidiousPlaylist.new({
+ title: playlist.title.byte_slice(0, 150),
+ id: playlist.id,
+ author: user.email,
+ description: "", # Max 5000 characters
+ video_count: playlist.video_count,
+ created: Time.utc,
+ updated: playlist.updated,
+ privacy: PlaylistPrivacy::Private,
+ index: [] of Int64,
+ })
- anchor = video.xpath_node(%q(.//td[@class="pl-video-time"]/div/div[1]))
- if anchor && !anchor.content.empty?
- length_seconds = decode_length_seconds(anchor.content)
- live_now = false
- else
- length_seconds = 0
- live_now = true
- end
+ playlist_array = playlist.to_a
+ args = arg_array(playlist_array)
- videos << PlaylistVideo.new(
- title: title,
- id: id,
- author: author,
- ucid: ucid,
- length_seconds: length_seconds,
- published: Time.utc,
- plid: plid,
- index: (index + offset).to_i64,
- live_now: live_now
- )
- end
+ db.exec("INSERT INTO playlists VALUES (#{args})", args: playlist_array)
- return videos
+ return playlist
end
def produce_playlist_url(id, index)
@@ -368,58 +350,64 @@ def fetch_playlist(plid, locale)
plid = "UU#{plid.lchop("UC")}"
end
- response = YT_POOL.client &.get("/playlist?list=#{plid}&hl=en&disable_polymer=1")
+ response = YT_POOL.client &.get("/playlist?list=#{plid}&hl=en")
if response.status_code != 200
- raise translate(locale, "Not a playlist.")
+ if response.headers["location"]?.try &.includes? "/sorry/index"
+ raise "Could not extract playlist info. Instance is likely blocked."
+ else
+ raise translate(locale, "Not a playlist.")
+ end
end
- body = response.body.gsub(/<button[^>]+><span[^>]+>\s*less\s*<img[^>]+>\n<\/span><\/button>/, "")
- document = XML.parse_html(body)
+ initial_data = extract_initial_data(response.body)
+ playlist_info = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]?.try &.[0]["playlistSidebarPrimaryInfoRenderer"]?
- title = document.xpath_node(%q(//h1[@class="pl-header-title"]))
- if !title
- raise translate(locale, "Playlist does not exist.")
- end
- title = title.content.strip(" \n")
+ raise "Could not extract playlist info" if !playlist_info
+ title = playlist_info["title"]?.try &.["runs"][0]?.try &.["text"]?.try &.as_s || ""
- description_html = document.xpath_node(%q(//span[@class="pl-header-description-text"]/div/div[1])).try &.to_s ||
- document.xpath_node(%q(//span[@class="pl-header-description-text"])).try &.to_s || ""
+ desc_item = playlist_info["description"]?
+ description = desc_item.try &.["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || desc_item.try &.["simpleText"]?.try &.as_s || ""
- playlist_thumbnail = document.xpath_node(%q(//div[@class="pl-header-thumb"]/img)).try &.["data-thumb"]? ||
- document.xpath_node(%q(//div[@class="pl-header-thumb"]/img)).try &.["src"]
+ thumbnail = playlist_info["thumbnailRenderer"]?.try &.["playlistVideoThumbnailRenderer"]?
+ .try &.["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s
- # YouTube allows anonymous playlists, so most of this can be empty or optional
- anchor = document.xpath_node(%q(//ul[@class="pl-header-details"]))
- author = anchor.try &.xpath_node(%q(.//li[1]/a)).try &.content
- author ||= ""
- author_thumbnail = document.xpath_node(%q(//img[@class="channel-header-profile-image"])).try &.["src"]
- author_thumbnail ||= ""
- ucid = anchor.try &.xpath_node(%q(.//li[1]/a)).try &.["href"].split("/")[-1]
- ucid ||= ""
+ views = 0_i64
+ updated = Time.utc
+ video_count = 0
+ playlist_info["stats"]?.try &.as_a.each do |stat|
+ text = stat["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || stat["simpleText"]?.try &.as_s
+ next if !text
- video_count = anchor.try &.xpath_node(%q(.//li[2])).try &.content.gsub(/\D/, "").to_i?
- video_count ||= 0
+ if text.includes? "video"
+ video_count = text.gsub(/\D/, "").to_i? || 0
+ elsif text.includes? "view"
+ views = text.gsub(/\D/, "").to_i64? || 0_i64
+ else
+ updated = decode_date(text.lchop("Last updated on ").lchop("Updated "))
+ end
+ end
- views = anchor.try &.xpath_node(%q(.//li[3])).try &.content.gsub(/\D/, "").to_i64?
- views ||= 0_i64
+ author_info = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]?.try &.[1]["playlistSidebarSecondaryInfoRenderer"]?
+ .try &.["videoOwner"]["videoOwnerRenderer"]?
- updated = anchor.try &.xpath_node(%q(.//li[4])).try &.content.lchop("Last updated on ").lchop("Updated ").try { |date| decode_date(date) }
- updated ||= Time.utc
+ raise "Could not extract author info" if !author_info
- playlist = Playlist.new(
- title: title,
- id: plid,
- author: author,
- author_thumbnail: author_thumbnail,
- ucid: ucid,
- description_html: description_html,
- video_count: video_count,
- views: views,
- updated: updated,
- thumbnail: playlist_thumbnail,
- )
+ author_thumbnail = author_info["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s || ""
+ author = author_info["title"]["runs"][0]["text"]?.try &.as_s || ""
+ ucid = author_info["title"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"]?.try &.as_s || ""
- return playlist
+ return Playlist.new({
+ title: title,
+ id: plid,
+ author: author,
+ author_thumbnail: author_thumbnail,
+ ucid: ucid,
+ description: description,
+ video_count: video_count,
+ views: views,
+ updated: updated,
+ thumbnail: thumbnail,
+ })
end
def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil)
@@ -437,35 +425,26 @@ end
def fetch_playlist_videos(plid, video_count, offset = 0, locale = nil, continuation = nil)
if continuation
- html = YT_POOL.client &.get("/watch?v=#{continuation}&list=#{plid}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999")
- html = XML.parse_html(html.body)
-
- index = html.xpath_node(%q(//span[@id="playlist-current-index"])).try &.content.to_i?.try &.- 1
- offset = index || offset
+ response = YT_POOL.client &.get("/watch?v=#{continuation}&list=#{plid}&gl=US&hl=en")
+ initial_data = extract_initial_data(response.body)
+ offset = initial_data["currentVideoEndpoint"]?.try &.["watchEndpoint"]?.try &.["index"]?.try &.as_i64 || offset
end
if video_count > 100
url = produce_playlist_url(plid, offset)
response = YT_POOL.client &.get(url)
- response = JSON.parse(response.body)
- if !response["content_html"]? || response["content_html"].as_s.empty?
- raise translate(locale, "Empty playlist")
- end
-
- document = XML.parse_html(response["content_html"].as_s)
- nodeset = document.xpath_nodes(%q(.//tr[contains(@class, "pl-video")]))
- videos = extract_playlist(plid, nodeset, offset)
+ initial_data = JSON.parse(response.body).as_a.find(&.as_h.["response"]?).try &.as_h
elsif offset > 100
return [] of PlaylistVideo
else # Extract first page of videos
- response = YT_POOL.client &.get("/playlist?list=#{plid}&gl=US&hl=en&disable_polymer=1")
- document = XML.parse_html(response.body)
- nodeset = document.xpath_nodes(%q(.//tr[contains(@class, "pl-video")]))
-
- videos = extract_playlist(plid, nodeset, 0)
+ response = YT_POOL.client &.get("/playlist?list=#{plid}&gl=US&hl=en")
+ initial_data = extract_initial_data(response.body)
end
+ return [] of PlaylistVideo if !initial_data
+ videos = extract_playlist_videos(initial_data)
+
until videos.empty? || videos[0].index == offset
videos.shift
end
@@ -473,6 +452,45 @@ def fetch_playlist_videos(plid, video_count, offset = 0, locale = nil, continuat
return videos
end
+def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
+ videos = [] of PlaylistVideo
+
+ (initial_data["contents"]?.try &.["twoColumnBrowseResultsRenderer"]["tabs"].as_a.select(&.["tabRenderer"]["selected"]?.try &.as_bool)[0]["tabRenderer"]["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["playlistVideoListRenderer"]["contents"].as_a ||
+ initial_data["response"]?.try &.["continuationContents"]["playlistVideoListContinuation"]["contents"].as_a).try &.each do |item|
+ if i = item["playlistVideoRenderer"]?
+ video_id = i["navigationEndpoint"]["watchEndpoint"]["videoId"].as_s
+ plid = i["navigationEndpoint"]["watchEndpoint"]["playlistId"].as_s
+ index = i["navigationEndpoint"]["watchEndpoint"]["index"].as_i64
+
+ thumbnail = i["thumbnail"]["thumbnails"][0]["url"].as_s
+ title = i["title"].try { |t| t["simpleText"]? || t["runs"]?.try &.[0]["text"]? }.try &.as_s || ""
+ author = i["shortBylineText"]?.try &.["runs"][0]["text"].as_s || ""
+ ucid = i["shortBylineText"]?.try &.["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"].as_s || ""
+ length_seconds = i["lengthSeconds"]?.try &.as_s.to_i
+ live = false
+
+ if !length_seconds
+ live = true
+ length_seconds = 0
+ end
+
+ videos << PlaylistVideo.new({
+ title: title,
+ id: video_id,
+ author: author,
+ ucid: ucid,
+ length_seconds: length_seconds,
+ published: Time.utc,
+ plid: plid,
+ live_now: live,
+ index: index,
+ })
+ end
+ end
+
+ return videos
+end
+
def template_playlist(playlist)
html = <<-END_HTML
<h3>
diff --git a/src/invidious/routes/base_route.cr b/src/invidious/routes/base_route.cr
new file mode 100644
index 00000000..c6e6667e
--- /dev/null
+++ b/src/invidious/routes/base_route.cr
@@ -0,0 +1,9 @@
+abstract class Invidious::Routes::BaseRoute
+ private getter config : Config
+ private getter logger : Invidious::LogHandler
+
+ def initialize(@config, @logger)
+ end
+
+ abstract def handle(env)
+end
diff --git a/src/invidious/routes/embed/index.cr b/src/invidious/routes/embed/index.cr
new file mode 100644
index 00000000..79c91d86
--- /dev/null
+++ b/src/invidious/routes/embed/index.cr
@@ -0,0 +1,27 @@
+class Invidious::Routes::Embed::Index < Invidious::Routes::BaseRoute
+ def handle(env)
+ locale = LOCALES[env.get("preferences").as(Preferences).locale]?
+
+ if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
+ begin
+ playlist = get_playlist(PG_DB, plid, locale: locale)
+ offset = env.params.query["index"]?.try &.to_i? || 0
+ videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale)
+ rescue ex
+ error_message = ex.message
+ env.response.status_code = 500
+ return templated "error"
+ end
+
+ url = "/embed/#{videos[0].id}?#{env.params.query}"
+
+ if env.params.query.size > 0
+ url += "?#{env.params.query}"
+ end
+ else
+ url = "/"
+ end
+
+ env.redirect url
+ end
+end
diff --git a/src/invidious/routes/embed/show.cr b/src/invidious/routes/embed/show.cr
new file mode 100644
index 00000000..23c2b86f
--- /dev/null
+++ b/src/invidious/routes/embed/show.cr
@@ -0,0 +1,174 @@
+class Invidious::Routes::Embed::Show < Invidious::Routes::BaseRoute
+ def handle(env)
+ locale = LOCALES[env.get("preferences").as(Preferences).locale]?
+ id = env.params.url["id"]
+
+ plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
+ continuation = process_continuation(PG_DB, env.params.query, plid, id)
+
+ if md = env.params.query["playlist"]?
+ .try &.match(/[a-zA-Z0-9_-]{11}(,[a-zA-Z0-9_-]{11})*/)
+ video_series = md[0].split(",")
+ env.params.query.delete("playlist")
+ end
+
+ preferences = env.get("preferences").as(Preferences)
+
+ if id.includes?("%20") || id.includes?("+") || env.params.query.to_s.includes?("%20") || env.params.query.to_s.includes?("+")
+ id = env.params.url["id"].gsub("%20", "").delete("+")
+
+ url = "/embed/#{id}"
+
+ if env.params.query.size > 0
+ url += "?#{env.params.query.to_s.gsub("%20", "").delete("+")}"
+ end
+
+ return env.redirect url
+ end
+
+ # YouTube embed supports `videoseries` with either `list=PLID`
+ # or `playlist=VIDEO_ID,VIDEO_ID`
+ case id
+ when "videoseries"
+ url = ""
+
+ if plid
+ begin
+ playlist = get_playlist(PG_DB, plid, locale: locale)
+ offset = env.params.query["index"]?.try &.to_i? || 0
+ videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale)
+ rescue ex
+ error_message = ex.message
+ env.response.status_code = 500
+ return templated "error"
+ end
+
+ url = "/embed/#{videos[0].id}"
+ elsif video_series
+ url = "/embed/#{video_series.shift}"
+ env.params.query["playlist"] = video_series.join(",")
+ else
+ return env.redirect "/"
+ end
+
+ if env.params.query.size > 0
+ url += "?#{env.params.query}"
+ end
+
+ return env.redirect url
+ when "live_stream"
+ response = YT_POOL.client &.get("/embed/live_stream?channel=#{env.params.query["channel"]? || ""}")
+ video_id = response.body.match(/"video_id":"(?<video_id>[a-zA-Z0-9_-]{11})"/).try &.["video_id"]
+
+ env.params.query.delete_all("channel")
+
+ if !video_id || video_id == "live_stream"
+ error_message = "Video is unavailable."
+ return templated "error"
+ end
+
+ url = "/embed/#{video_id}"
+
+ if env.params.query.size > 0
+ url += "?#{env.params.query}"
+ end
+
+ return env.redirect url
+ when id.size > 11
+ url = "/embed/#{id[0, 11]}"
+
+ if env.params.query.size > 0
+ url += "?#{env.params.query}"
+ end
+
+ return env.redirect url
+ else nil # Continue
+ end
+
+ params = process_video_params(env.params.query, preferences)
+
+ user = env.get?("user").try &.as(User)
+ if user
+ subscriptions = user.subscriptions
+ watched = user.watched
+ notifications = user.notifications
+ end
+ subscriptions ||= [] of String
+
+ begin
+ video = get_video(id, PG_DB, region: params.region)
+ rescue ex : VideoRedirect
+ return env.redirect env.request.resource.gsub(id, ex.video_id)
+ rescue ex
+ error_message = ex.message
+ env.response.status_code = 500
+ return templated "error"
+ end
+
+ if preferences.annotations_subscribed &&
+ subscriptions.includes?(video.ucid) &&
+ (env.params.query["iv_load_policy"]? || "1") == "1"
+ params.annotations = true
+ end
+
+ # if watched && !watched.includes? id
+ # PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.as(User).email)
+ # end
+
+ if notifications && notifications.includes? id
+ PG_DB.exec("UPDATE users SET notifications = array_remove(notifications, $1) WHERE email = $2", id, user.as(User).email)
+ env.get("user").as(User).notifications.delete(id)
+ notifications.delete(id)
+ end
+
+ fmt_stream = video.fmt_stream
+ adaptive_fmts = video.adaptive_fmts
+
+ if params.local
+ fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) }
+ adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) }
+ end
+
+ video_streams = video.video_streams
+ audio_streams = video.audio_streams
+
+ if audio_streams.empty? && !video.live_now
+ if params.quality == "dash"
+ env.params.query.delete_all("quality")
+ return env.redirect "/embed/#{id}?#{env.params.query}"
+ elsif params.listen
+ env.params.query.delete_all("listen")
+ env.params.query["listen"] = "0"
+ return env.redirect "/embed/#{id}?#{env.params.query}"
+ end
+ end
+
+ captions = video.captions
+
+ preferred_captions = captions.select { |caption|
+ params.preferred_captions.includes?(caption.name.simpleText) ||
+ params.preferred_captions.includes?(caption.languageCode.split("-")[0])
+ }
+ preferred_captions.sort_by! { |caption|
+ (params.preferred_captions.index(caption.name.simpleText) ||
+ params.preferred_captions.index(caption.languageCode.split("-")[0])).not_nil!
+ }
+ captions = captions - preferred_captions
+
+ aspect_ratio = nil
+
+ thumbnail = "/vi/#{video.id}/maxres.jpg"
+
+ if params.raw
+ url = fmt_stream[0]["url"].as_s
+
+ fmt_stream.each do |fmt|
+ url = fmt["url"].as_s if fmt["quality"].as_s == params.quality
+ end
+
+ return env.redirect url
+ end
+
+ rendered "embed"
+ end
+end
diff --git a/src/invidious/routes/home.cr b/src/invidious/routes/home.cr
new file mode 100644
index 00000000..9b1bf61b
--- /dev/null
+++ b/src/invidious/routes/home.cr
@@ -0,0 +1,34 @@
+class Invidious::Routes::Home < Invidious::Routes::BaseRoute
+ def handle(env)
+ preferences = env.get("preferences").as(Preferences)
+ locale = LOCALES[preferences.locale]?
+ user = env.get? "user"
+
+ case preferences.default_home
+ when ""
+ templated "empty"
+ when "Popular"
+ templated "popular"
+ when "Trending"
+ env.redirect "/feed/trending"
+ when "Subscriptions"
+ if user
+ env.redirect "/feed/subscriptions"
+ else
+ templated "popular"
+ end
+ when "Playlists"
+ if user
+ env.redirect "/view_all_playlists"
+ else
+ templated "popular"
+ end
+ else
+ templated "empty"
+ end
+ end
+
+ private def popular_videos
+ Jobs::PullPopularVideosJob::POPULAR_VIDEOS.get
+ end
+end
diff --git a/src/invidious/routes/licenses.cr b/src/invidious/routes/licenses.cr
new file mode 100644
index 00000000..38fde7bb
--- /dev/null
+++ b/src/invidious/routes/licenses.cr
@@ -0,0 +1,6 @@
+class Invidious::Routes::Licenses < Invidious::Routes::BaseRoute
+ def handle(env)
+ locale = LOCALES[env.get("preferences").as(Preferences).locale]?
+ rendered "licenses"
+ end
+end
diff --git a/src/invidious/routes/privacy.cr b/src/invidious/routes/privacy.cr
new file mode 100644
index 00000000..4565c94c
--- /dev/null
+++ b/src/invidious/routes/privacy.cr
@@ -0,0 +1,6 @@
+class Invidious::Routes::Privacy < Invidious::Routes::BaseRoute
+ def handle(env)
+ locale = LOCALES[env.get("preferences").as(Preferences).locale]?
+ templated "privacy"
+ end
+end
diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr
new file mode 100644
index 00000000..4eee7793
--- /dev/null
+++ b/src/invidious/routes/watch.cr
@@ -0,0 +1,186 @@
+class Invidious::Routes::Watch < Invidious::Routes::BaseRoute
+ def handle(env)
+ locale = LOCALES[env.get("preferences").as(Preferences).locale]?
+ region = env.params.query["region"]?
+
+ if env.params.query.to_s.includes?("%20") || env.params.query.to_s.includes?("+")
+ url = "/watch?" + env.params.query.to_s.gsub("%20", "").delete("+")
+ return env.redirect url
+ end
+
+ if env.params.query["v"]?
+ id = env.params.query["v"]
+
+ if env.params.query["v"].empty?
+ error_message = "Invalid parameters."
+ env.response.status_code = 400
+ return templated "error"
+ end
+
+ if id.size > 11
+ url = "/watch?v=#{id[0, 11]}"
+ env.params.query.delete_all("v")
+ if env.params.query.size > 0
+ url += "&#{env.params.query}"
+ end
+
+ return env.redirect url
+ end
+ else
+ return env.redirect "/"
+ end
+
+ plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
+ continuation = process_continuation(PG_DB, env.params.query, plid, id)
+
+ nojs = env.params.query["nojs"]?
+
+ nojs ||= "0"
+ nojs = nojs == "1"
+
+ preferences = env.get("preferences").as(Preferences)
+
+ user = env.get?("user").try &.as(User)
+ if user
+ subscriptions = user.subscriptions
+ watched = user.watched
+ notifications = user.notifications
+ end
+ subscriptions ||= [] of String
+
+ params = process_video_params(env.params.query, preferences)
+ env.params.query.delete_all("listen")
+
+ begin
+ video = get_video(id, PG_DB, region: params.region)
+ rescue ex : VideoRedirect
+ return env.redirect env.request.resource.gsub(id, ex.video_id)
+ rescue ex
+ error_message = ex.message
+ env.response.status_code = 500
+ logger.puts("#{id} : #{ex.message}")
+ return templated "error"
+ end
+
+ if preferences.annotations_subscribed &&
+ subscriptions.includes?(video.ucid) &&
+ (env.params.query["iv_load_policy"]? || "1") == "1"
+ params.annotations = true
+ end
+ env.params.query.delete_all("iv_load_policy")
+
+ if watched && !watched.includes? id
+ PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.as(User).email)
+ end
+
+ if notifications && notifications.includes? id
+ PG_DB.exec("UPDATE users SET notifications = array_remove(notifications, $1) WHERE email = $2", id, user.as(User).email)
+ env.get("user").as(User).notifications.delete(id)
+ notifications.delete(id)
+ end
+
+ if nojs
+ if preferences
+ source = preferences.comments[0]
+ if source.empty?
+ source = preferences.comments[1]
+ end
+
+ if source == "youtube"
+ begin
+ comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+ rescue ex
+ if preferences.comments[1] == "reddit"
+ comments, reddit_thread = fetch_reddit_comments(id)
+ comment_html = template_reddit_comments(comments, locale)
+
+ comment_html = fill_links(comment_html, "https", "www.reddit.com")
+ comment_html = replace_links(comment_html)
+ end
+ end
+ elsif source == "reddit"
+ begin
+ comments, reddit_thread = fetch_reddit_comments(id)
+ comment_html = template_reddit_comments(comments, locale)
+
+ comment_html = fill_links(comment_html, "https", "www.reddit.com")
+ comment_html = replace_links(comment_html)
+ rescue ex
+ if preferences.comments[1] == "youtube"
+ comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+ end
+ end
+ end
+ else
+ comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+ end
+
+ comment_html ||= ""
+ end
+
+ fmt_stream = video.fmt_stream
+ adaptive_fmts = video.adaptive_fmts
+
+ if params.local
+ fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) }
+ adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) }
+ end
+
+ video_streams = video.video_streams
+ audio_streams = video.audio_streams
+
+ # Older videos may not have audio sources available.
+ # We redirect here so they're not unplayable
+ if audio_streams.empty? && !video.live_now
+ if params.quality == "dash"
+ env.params.query.delete_all("quality")
+ env.params.query["quality"] = "medium"
+ return env.redirect "/watch?#{env.params.query}"
+ elsif params.listen
+ env.params.query.delete_all("listen")
+ env.params.query["listen"] = "0"
+ return env.redirect "/watch?#{env.params.query}"
+ end
+ end
+
+ captions = video.captions
+
+ preferred_captions = captions.select { |caption|
+ params.preferred_captions.includes?(caption.name.simpleText) ||
+ params.preferred_captions.includes?(caption.languageCode.split("-")[0])
+ }
+ preferred_captions.sort_by! { |caption|
+ (params.preferred_captions.index(caption.name.simpleText) ||
+ params.preferred_captions.index(caption.languageCode.split("-")[0])).not_nil!
+ }
+ captions = captions - preferred_captions
+
+ aspect_ratio = "16:9"
+
+ thumbnail = "/vi/#{video.id}/maxres.jpg"
+
+ if params.raw
+ if params.listen
+ url = audio_streams[0]["url"].as_s
+
+ audio_streams.each do |fmt|
+ if fmt["bitrate"].as_i == params.quality.rchop("k").to_i
+ url = fmt["url"].as_s
+ end
+ end
+ else
+ url = fmt_stream[0]["url"].as_s
+
+ fmt_stream.each do |fmt|
+ if fmt["quality"].as_s == params.quality
+ url = fmt["url"].as_s
+ end
+ end
+ end
+
+ return env.redirect url
+ end
+
+ templated "watch"
+ end
+end
diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr
new file mode 100644
index 00000000..c09dda38
--- /dev/null
+++ b/src/invidious/routing.cr
@@ -0,0 +1,8 @@
+module Invidious::Routing
+ macro get(path, controller)
+ get {{ path }} do |env|
+ controller_instance = {{ controller }}.new(config, logger)
+ controller_instance.handle(env)
+ end
+ end
+end
diff --git a/src/invidious/search.cr b/src/invidious/search.cr
index 92996f75..85fd024a 100644
--- a/src/invidious/search.cr
+++ b/src/invidious/search.cr
@@ -1,5 +1,20 @@
struct SearchVideo
- def to_xml(host_url, auto_generated, query_params, xml : XML::Builder)
+ include DB::Serializable
+
+ property title : String
+ property id : String
+ property author : String
+ property ucid : String
+ property published : Time
+ property views : Int64
+ property description_html : String
+ property length_seconds : Int32
+ property live_now : Bool
+ property paid : Bool
+ property premium : Bool
+ property premiere_timestamp : Time?
+
+ def to_xml(auto_generated, query_params, xml : XML::Builder)
query_params["v"] = self.id
xml.element("entry") do
@@ -7,22 +22,22 @@ struct SearchVideo
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
- xml.element("link", rel: "alternate", href: "#{host_url}/watch?#{query_params}")
+ xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}")
xml.element("author") do
if auto_generated
xml.element("name") { xml.text self.author }
- xml.element("uri") { xml.text "#{host_url}/channel/#{self.ucid}" }
+ xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
else
xml.element("name") { xml.text author }
- xml.element("uri") { xml.text "#{host_url}/channel/#{ucid}" }
+ xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
end
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
- xml.element("a", href: "#{host_url}/watch?#{query_params}") do
- xml.element("img", src: "#{host_url}/vi/#{self.id}/mqdefault.jpg")
+ xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do
+ xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text html_to_content(self.description_html) }
@@ -33,7 +48,7 @@ struct SearchVideo
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
- xml.element("media:thumbnail", url: "#{host_url}/vi/#{self.id}/mqdefault.jpg",
+ xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
xml.element("media:description") { xml.text html_to_content(self.description_html) }
end
@@ -44,17 +59,17 @@ struct SearchVideo
end
end
- def to_xml(host_url, auto_generated, query_params, xml : XML::Builder | Nil = nil)
+ def to_xml(auto_generated, query_params, xml : XML::Builder | Nil = nil)
if xml
- to_xml(host_url, auto_generated, query_params, xml)
+ to_xml(HOST_URL, auto_generated, query_params, xml)
else
XML.build do |json|
- to_xml(host_url, auto_generated, query_params, xml)
+ to_xml(HOST_URL, auto_generated, query_params, xml)
end
end
end
- def to_json(locale, config, kemal_config, json : JSON::Builder)
+ def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "video"
json.field "title", self.title
@@ -65,7 +80,7 @@ struct SearchVideo
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
- generate_thumbnails(json, self.id, config, kemal_config)
+ generate_thumbnails(json, self.id)
end
json.field "description", html_to_content(self.description_html)
@@ -78,45 +93,49 @@ struct SearchVideo
json.field "liveNow", self.live_now
json.field "paid", self.paid
json.field "premium", self.premium
+ json.field "isUpcoming", self.is_upcoming
+
+ if self.premiere_timestamp
+ json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
+ end
end
end
- def to_json(locale, config, kemal_config, json : JSON::Builder | Nil = nil)
+ def to_json(locale, json : JSON::Builder | Nil = nil)
if json
- to_json(locale, config, kemal_config, json)
+ to_json(locale, json)
else
JSON.build do |json|
- to_json(locale, config, kemal_config, json)
+ to_json(locale, json)
end
end
end
- db_mapping({
- title: String,
- id: String,
- author: String,
- ucid: String,
- published: Time,
- views: Int64,
- description_html: String,
- length_seconds: Int32,
- live_now: Bool,
- paid: Bool,
- premium: Bool,
- premiere_timestamp: Time?,
- })
+ def is_upcoming
+ premiere_timestamp ? true : false
+ end
end
struct SearchPlaylistVideo
- db_mapping({
- title: String,
- id: String,
- length_seconds: Int32,
- })
+ include DB::Serializable
+
+ property title : String
+ property id : String
+ property length_seconds : Int32
end
struct SearchPlaylist
- def to_json(locale, config, kemal_config, json : JSON::Builder)
+ include DB::Serializable
+
+ property title : String
+ property id : String
+ property author : String
+ property ucid : String
+ property video_count : Int32
+ property videos : Array(SearchPlaylistVideo)
+ property thumbnail : String?
+
+ def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "playlist"
json.field "title", self.title
@@ -137,7 +156,7 @@ struct SearchPlaylist
json.field "lengthSeconds", video.length_seconds
json.field "videoThumbnails" do
- generate_thumbnails(json, video.id, config, Kemal.config)
+ generate_thumbnails(json, video.id)
end
end
end
@@ -146,29 +165,29 @@ struct SearchPlaylist
end
end
- def to_json(locale, config, kemal_config, json : JSON::Builder | Nil = nil)
+ def to_json(locale, json : JSON::Builder | Nil = nil)
if json
- to_json(locale, config, kemal_config, json)
+ to_json(locale, json)
else
JSON.build do |json|
- to_json(locale, config, kemal_config, json)
+ to_json(locale, json)
end
end
end
-
- db_mapping({
- title: String,
- id: String,
- author: String,
- ucid: String,
- video_count: Int32,
- videos: Array(SearchPlaylistVideo),
- thumbnail: String?,
- })
end
struct SearchChannel
- def to_json(locale, config, kemal_config, json : JSON::Builder)
+ include DB::Serializable
+
+ property author : String
+ property ucid : String
+ property author_thumbnail : String
+ property subscriber_count : Int32
+ property video_count : Int32
+ property description_html : String
+ property auto_generated : Bool
+
+ def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "channel"
json.field "author", self.author
@@ -198,85 +217,50 @@ struct SearchChannel
end
end
- def to_json(locale, config, kemal_config, json : JSON::Builder | Nil = nil)
+ def to_json(locale, json : JSON::Builder | Nil = nil)
if json
- to_json(locale, config, kemal_config, json)
+ to_json(locale, json)
else
JSON.build do |json|
- to_json(locale, config, kemal_config, json)
+ to_json(locale, json)
end
end
end
-
- db_mapping({
- author: String,
- ucid: String,
- author_thumbnail: String,
- subscriber_count: Int32,
- video_count: Int32,
- description_html: String,
- auto_generated: Bool,
- })
end
alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist
def channel_search(query, page, channel)
- response = YT_POOL.client &.get("/channel/#{channel}?disable_polymer=1&hl=en&gl=US")
- document = XML.parse_html(response.body)
- canonical = document.xpath_node(%q(//link[@rel="canonical"]))
-
- if !canonical
- response = YT_POOL.client &.get("/c/#{channel}?disable_polymer=1&hl=en&gl=US")
- document = XML.parse_html(response.body)
- canonical = document.xpath_node(%q(//link[@rel="canonical"]))
- end
+ response = YT_POOL.client &.get("/channel/#{channel}?hl=en&gl=US")
+ response = YT_POOL.client &.get("/user/#{channel}?hl=en&gl=US") if response.headers["location"]?
+ response = YT_POOL.client &.get("/c/#{channel}?hl=en&gl=US") if response.headers["location"]?
- if !canonical
- response = YT_POOL.client &.get("/user/#{channel}?disable_polymer=1&hl=en&gl=US")
- document = XML.parse_html(response.body)
- canonical = document.xpath_node(%q(//link[@rel="canonical"]))
- end
+ ucid = response.body.match(/\\"channelId\\":\\"(?<ucid>[^\\]+)\\"/).try &.["ucid"]?
- if !canonical
- return 0, [] of SearchItem
- end
-
- ucid = canonical["href"].split("/")[-1]
+ return 0, [] of SearchItem if !ucid
url = produce_channel_search_url(ucid, query, page)
response = YT_POOL.client &.get(url)
- json = JSON.parse(response.body)
-
- if json["content_html"]? && !json["content_html"].as_s.empty?
- document = XML.parse_html(json["content_html"].as_s)
- nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
-
- count = nodeset.size
- items = extract_items(nodeset)
- else
- count = 0
- items = [] of SearchItem
- end
+ initial_data = JSON.parse(response.body).as_a.find &.["response"]?
+ return 0, [] of SearchItem if !initial_data
+ author = initial_data["response"]?.try &.["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s
+ items = extract_items(initial_data.as_h, author, ucid)
- return count, items
+ return items.size, items
end
def search(query, page = 1, search_params = produce_search_params(content_type: "all"), region = nil)
- if query.empty?
- return {0, [] of SearchItem}
- end
+ return 0, [] of SearchItem if query.empty?
- html = YT_POOL.client(region, &.get("/results?q=#{URI.encode_www_form(query)}&page=#{page}&sp=#{search_params}&hl=en&disable_polymer=1").body)
- if html.empty?
- return {0, [] of SearchItem}
- end
+ body = YT_POOL.client(region, &.get("/results?q=#{URI.encode_www_form(query)}&page=#{page}&sp=#{search_params}&hl=en").body)
+ return 0, [] of SearchItem if body.empty?
+
+ initial_data = extract_initial_data(body)
+ items = extract_items(initial_data)
- html = XML.parse_html(html)
- nodeset = html.xpath_nodes(%q(//ol[@class="item-section"]/li))
- items = extract_items(nodeset)
+ # initial_data["estimatedResults"]?.try &.as_s.to_i64
- return {nodeset.size, items}
+ return items.size, items
end
def produce_search_params(sort : String = "relevance", date : String = "", content_type : String = "",
@@ -310,6 +294,7 @@ def produce_search_params(sort : String = "relevance", date : String = "", conte
object["2:embedded"].as(Hash)["1:varint"] = 4_i64
when "year"
object["2:embedded"].as(Hash)["1:varint"] = 5_i64
+ else nil # Ignore
end
case content_type
@@ -334,6 +319,7 @@ def produce_search_params(sort : String = "relevance", date : String = "", conte
object["2:embedded"].as(Hash)["3:varint"] = 1_i64
when "long"
object["2:embedded"].as(Hash)["3:varint"] = 2_i64
+ else nil # Ignore
end
features.each do |feature|
@@ -358,6 +344,7 @@ def produce_search_params(sort : String = "relevance", date : String = "", conte
object["2:embedded"].as(Hash)["23:varint"] = 1_i64
when "hdr"
object["2:embedded"].as(Hash)["25:varint"] = 1_i64
+ else nil # Ignore
end
end
@@ -379,12 +366,9 @@ def produce_channel_search_url(ucid, query, page)
"2:string" => ucid,
"3:base64" => {
"2:string" => "search",
- "6:varint" => 2_i64,
"7:varint" => 1_i64,
- "12:varint" => 1_i64,
- "13:string" => "",
- "23:varint" => 0_i64,
"15:string" => "#{page}",
+ "23:varint" => 0_i64,
},
"11:string" => query,
},
diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr
index 3a9c6935..8d078387 100644
--- a/src/invidious/trending.cr
+++ b/src/invidious/trending.cr
@@ -1,7 +1,4 @@
def fetch_trending(trending_type, region, locale)
- headers = HTTP::Headers.new
- headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
-
region ||= "US"
region = region.upcase
@@ -11,7 +8,7 @@ def fetch_trending(trending_type, region, locale)
if trending_type && trending_type != "Default"
trending_type = trending_type.downcase.capitalize
- response = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en", headers).body
+ response = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en").body
initial_data = extract_initial_data(response)
@@ -21,51 +18,28 @@ def fetch_trending(trending_type, region, locale)
if url
url["channelListSubMenuAvatarRenderer"]["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"]
url = url["channelListSubMenuAvatarRenderer"]["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s
- url += "&disable_polymer=1&gl=#{region}&hl=en"
+ url = "#{url}&gl=#{region}&hl=en"
trending = YT_POOL.client &.get(url).body
plid = extract_plid(url)
else
- trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en&disable_polymer=1").body
+ trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en").body
end
else
- trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en&disable_polymer=1").body
+ trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en").body
end
- trending = XML.parse_html(trending)
- nodeset = trending.xpath_nodes(%q(//ul/li[@class="expanded-shelf-content-item-wrapper"]))
- trending = extract_videos(nodeset)
+ initial_data = extract_initial_data(trending)
+ trending = extract_videos(initial_data)
return {trending, plid}
end
def extract_plid(url)
- wrapper = HTTP::Params.parse(URI.parse(url).query.not_nil!)["bp"]
-
- wrapper = URI.decode_www_form(wrapper)
- wrapper = Base64.decode(wrapper)
-
- # 0xe2 0x02 0x2e
- wrapper += 3
-
- # 0x0a
- wrapper += 1
-
- # Looks like "/m/[a-z0-9]{5}", not sure what it does here
-
- item_size = wrapper[0]
- wrapper += 1
- item = wrapper[0, item_size]
- wrapper += item.size
-
- # 0x12
- wrapper += 1
-
- plid_size = wrapper[0]
- wrapper += 1
- plid = wrapper[0, plid_size]
- wrapper += plid.size
-
- plid = String.new(plid)
-
- return plid
+ return url.try { |i| URI.parse(i).query }
+ .try { |i| HTTP::Params.parse(i)["bp"] }
+ .try { |i| URI.decode_www_form(i) }
+ .try { |i| Base64.decode(i) }
+ .try { |i| IO::Memory.new(i) }
+ .try { |i| Protodec::Any.parse(i) }
+ .try &.["44:0:embedded"]?.try &.["2:1:string"]?.try &.as_s
end
diff --git a/src/invidious/users.cr b/src/invidious/users.cr
index afb100f2..46bf8865 100644
--- a/src/invidious/users.cr
+++ b/src/invidious/users.cr
@@ -4,6 +4,20 @@ require "crypto/bcrypt/password"
MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" }
struct User
+ include DB::Serializable
+
+ property updated : Time
+ property notifications : Array(String)
+ property subscriptions : Array(String)
+ property email : String
+
+ @[DB::Field(converter: User::PreferencesConverter)]
+ property preferences : Preferences
+ property password : String?
+ property token : String
+ property watched : Array(String)
+ property feed_needs_update : Bool?
+
module PreferencesConverter
def self.from_rs(rs)
begin
@@ -13,31 +27,78 @@ struct User
end
end
end
-
- db_mapping({
- updated: Time,
- notifications: Array(String),
- subscriptions: Array(String),
- email: String,
- preferences: {
- type: Preferences,
- converter: PreferencesConverter,
- },
- password: String?,
- token: String,
- watched: Array(String),
- feed_needs_update: Bool?,
- })
end
struct Preferences
- module ProcessString
+ include JSON::Serializable
+ include YAML::Serializable
+
+ property annotations : Bool = CONFIG.default_user_preferences.annotations
+ property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed
+ property autoplay : Bool = CONFIG.default_user_preferences.autoplay
+
+ @[JSON::Field(converter: Preferences::StringToArray)]
+ @[YAML::Field(converter: Preferences::StringToArray)]
+ property captions : Array(String) = CONFIG.default_user_preferences.captions
+
+ @[JSON::Field(converter: Preferences::StringToArray)]
+ @[YAML::Field(converter: Preferences::StringToArray)]
+ property comments : Array(String) = CONFIG.default_user_preferences.comments
+ property continue : Bool = CONFIG.default_user_preferences.continue
+ property continue_autoplay : Bool = CONFIG.default_user_preferences.continue_autoplay
+
+ @[JSON::Field(converter: Preferences::BoolToString)]
+ @[YAML::Field(converter: Preferences::BoolToString)]
+ property dark_mode : String = CONFIG.default_user_preferences.dark_mode
+ property latest_only : Bool = CONFIG.default_user_preferences.latest_only
+ property listen : Bool = CONFIG.default_user_preferences.listen
+ property local : Bool = CONFIG.default_user_preferences.local
+
+ @[JSON::Field(converter: Preferences::ProcessString)]
+ property locale : String = CONFIG.default_user_preferences.locale
+
+ @[JSON::Field(converter: Preferences::ClampInt)]
+ property max_results : Int32 = CONFIG.default_user_preferences.max_results
+ property notifications_only : Bool = CONFIG.default_user_preferences.notifications_only
+
+ @[JSON::Field(converter: Preferences::ProcessString)]
+ property player_style : String = CONFIG.default_user_preferences.player_style
+
+ @[JSON::Field(converter: Preferences::ProcessString)]
+ property quality : String = CONFIG.default_user_preferences.quality
+ property default_home : String = CONFIG.default_user_preferences.default_home
+ property feed_menu : Array(String) = CONFIG.default_user_preferences.feed_menu
+ property related_videos : Bool = CONFIG.default_user_preferences.related_videos
+
+ @[JSON::Field(converter: Preferences::ProcessString)]
+ property sort : String = CONFIG.default_user_preferences.sort
+ property speed : Float32 = CONFIG.default_user_preferences.speed
+ property thin_mode : Bool = CONFIG.default_user_preferences.thin_mode
+ property unseen_only : Bool = CONFIG.default_user_preferences.unseen_only
+ property video_loop : Bool = CONFIG.default_user_preferences.video_loop
+ property volume : Int32 = CONFIG.default_user_preferences.volume
+
+ module BoolToString
def self.to_json(value : String, json : JSON::Builder)
json.string value
end
def self.from_json(value : JSON::PullParser) : String
- HTML.escape(value.read_string[0, 100])
+ begin
+ result = value.read_string
+
+ if result.empty?
+ CONFIG.default_user_preferences.dark_mode
+ else
+ result
+ end
+ rescue ex
+ if value.read_bool
+ "dark"
+ else
+ "light"
+ end
+ end
end
def self.to_yaml(value : String, yaml : YAML::Nodes::Builder)
@@ -45,7 +106,20 @@ struct Preferences
end
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String
- HTML.escape(node.value[0, 100])
+ unless node.is_a?(YAML::Nodes::Scalar)
+ node.raise "Expected scalar, not #{node.class}"
+ end
+
+ case node.value
+ when "true"
+ "dark"
+ when "false"
+ "light"
+ when ""
+ CONFIG.default_user_preferences.dark_mode
+ else
+ node.value
+ end
end
end
@@ -67,33 +141,130 @@ struct Preferences
end
end
- json_mapping({
- annotations: {type: Bool, default: CONFIG.default_user_preferences.annotations},
- annotations_subscribed: {type: Bool, default: CONFIG.default_user_preferences.annotations_subscribed},
- autoplay: {type: Bool, default: CONFIG.default_user_preferences.autoplay},
- captions: {type: Array(String), default: CONFIG.default_user_preferences.captions, converter: ConfigPreferences::StringToArray},
- comments: {type: Array(String), default: CONFIG.default_user_preferences.comments, converter: ConfigPreferences::StringToArray},
- continue: {type: Bool, default: CONFIG.default_user_preferences.continue},
- continue_autoplay: {type: Bool, default: CONFIG.default_user_preferences.continue_autoplay},
- dark_mode: {type: String, default: CONFIG.default_user_preferences.dark_mode, converter: ConfigPreferences::BoolToString},
- latest_only: {type: Bool, default: CONFIG.default_user_preferences.latest_only},
- listen: {type: Bool, default: CONFIG.default_user_preferences.listen},
- local: {type: Bool, default: CONFIG.default_user_preferences.local},
- locale: {type: String, default: CONFIG.default_user_preferences.locale, converter: ProcessString},
- max_results: {type: Int32, default: CONFIG.default_user_preferences.max_results, converter: ClampInt},
- notifications_only: {type: Bool, default: CONFIG.default_user_preferences.notifications_only},
- player_style: {type: String, default: CONFIG.default_user_preferences.player_style, converter: ProcessString},
- quality: {type: String, default: CONFIG.default_user_preferences.quality, converter: ProcessString},
- default_home: {type: String, default: CONFIG.default_user_preferences.default_home},
- feed_menu: {type: Array(String), default: CONFIG.default_user_preferences.feed_menu},
- related_videos: {type: Bool, default: CONFIG.default_user_preferences.related_videos},
- sort: {type: String, default: CONFIG.default_user_preferences.sort, converter: ProcessString},
- speed: {type: Float32, default: CONFIG.default_user_preferences.speed},
- thin_mode: {type: Bool, default: CONFIG.default_user_preferences.thin_mode},
- unseen_only: {type: Bool, default: CONFIG.default_user_preferences.unseen_only},
- video_loop: {type: Bool, default: CONFIG.default_user_preferences.video_loop},
- volume: {type: Int32, default: CONFIG.default_user_preferences.volume},
- })
+ module FamilyConverter
+ def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder)
+ case value
+ when Socket::Family::UNSPEC
+ yaml.scalar nil
+ when Socket::Family::INET
+ yaml.scalar "ipv4"
+ when Socket::Family::INET6
+ yaml.scalar "ipv6"
+ when Socket::Family::UNIX
+ raise "Invalid socket family #{value}"
+ end
+ end
+
+ def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family
+ if node.is_a?(YAML::Nodes::Scalar)
+ case node.value.downcase
+ when "ipv4"
+ Socket::Family::INET
+ when "ipv6"
+ Socket::Family::INET6
+ else
+ Socket::Family::UNSPEC
+ end
+ else
+ node.raise "Expected scalar, not #{node.class}"
+ end
+ end
+ end
+
+ module ProcessString
+ def self.to_json(value : String, json : JSON::Builder)
+ json.string value
+ end
+
+ def self.from_json(value : JSON::PullParser) : String
+ HTML.escape(value.read_string[0, 100])
+ end
+
+ def self.to_yaml(value : String, yaml : YAML::Nodes::Builder)
+ yaml.scalar value
+ end
+
+ def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String
+ HTML.escape(node.value[0, 100])
+ end
+ end
+
+ module StringToArray
+ def self.to_json(value : Array(String), json : JSON::Builder)
+ json.array do
+ value.each do |element|
+ json.string element
+ end
+ end
+ end
+
+ def self.from_json(value : JSON::PullParser) : Array(String)
+ begin
+ result = [] of String
+ value.read_array do
+ result << HTML.escape(value.read_string[0, 100])
+ end
+ rescue ex
+ result = [HTML.escape(value.read_string[0, 100]), ""]
+ end
+
+ result
+ end
+
+ def self.to_yaml(value : Array(String), yaml : YAML::Nodes::Builder)
+ yaml.sequence do
+ value.each do |element|
+ yaml.scalar element
+ end
+ end
+ end
+
+ def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(String)
+ begin
+ unless node.is_a?(YAML::Nodes::Sequence)
+ node.raise "Expected sequence, not #{node.class}"
+ end
+
+ result = [] of String
+ node.nodes.each do |item|
+ unless item.is_a?(YAML::Nodes::Scalar)
+ node.raise "Expected scalar, not #{item.class}"
+ end
+
+ result << HTML.escape(item.value[0, 100])
+ end
+ rescue ex
+ if node.is_a?(YAML::Nodes::Scalar)
+ result = [HTML.escape(node.value[0, 100]), ""]
+ else
+ result = ["", ""]
+ end
+ end
+
+ result
+ end
+ end
+
+ module StringToCookies
+ def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder)
+ (value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml)
+ end
+
+ def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies
+ unless node.is_a?(YAML::Nodes::Scalar)
+ node.raise "Expected scalar, not #{node.class}"
+ end
+
+ cookies = HTTP::Cookies.new
+ node.value.split(";").each do |cookie|
+ next if cookie.strip.empty?
+ name, value = cookie.split("=", 2)
+ cookies << HTTP::Cookie.new(name.strip, value.strip)
+ end
+
+ cookies
+ end
+ end
end
def get_user(sid, headers, db, refresh = true)
@@ -103,8 +274,7 @@ def get_user(sid, headers, db, refresh = true)
if refresh && Time.utc - user.updated > 1.minute
user, sid = fetch_user(sid, headers, db)
user_array = user.to_a
-
- user_array[4] = user_array[4].to_json
+ user_array[4] = user_array[4].to_json # User preferences
args = arg_array(user_array)
db.exec("INSERT INTO users VALUES (#{args}) \
@@ -122,8 +292,7 @@ def get_user(sid, headers, db, refresh = true)
else
user, sid = fetch_user(sid, headers, db)
user_array = user.to_a
-
- user_array[4] = user_array[4].to_json
+ user_array[4] = user_array[4].to_json # User preferences
args = arg_array(user.to_a)
db.exec("INSERT INTO users VALUES (#{args}) \
@@ -166,7 +335,17 @@ def fetch_user(sid, headers, db)
token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
- user = User.new(Time.utc, [] of String, channels, email, CONFIG.default_user_preferences, nil, token, [] of String, true)
+ user = User.new({
+ updated: Time.utc,
+ notifications: [] of String,
+ subscriptions: channels,
+ email: email,
+ preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple),
+ password: nil,
+ token: token,
+ watched: [] of String,
+ feed_needs_update: true,
+ })
return user, sid
end
@@ -174,7 +353,17 @@ def create_user(sid, email, password)
password = Crypto::Bcrypt::Password.create(password, cost: 10)
token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
- user = User.new(Time.utc, [] of String, [] of String, email, CONFIG.default_user_preferences, password.to_s, token, [] of String, true)
+ user = User.new({
+ updated: Time.utc,
+ notifications: [] of String,
+ subscriptions: [] of String,
+ email: email,
+ preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple),
+ password: password.to_s,
+ token: token,
+ watched: [] of String,
+ feed_needs_update: true,
+ })
return user, sid
end
@@ -267,7 +456,7 @@ def subscribe_ajax(channel_id, action, env_headers)
end
headers = cookies.add_request_headers(headers)
- if match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[A-Za-z0-9\_\-\=]+)"/)
+ if match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[^"]+)"/)
session_token = match["session_token"]
headers["content-type"] = "application/x-www-form-urlencoded"
@@ -281,48 +470,6 @@ def subscribe_ajax(channel_id, action, env_headers)
end
end
-# TODO: Playlist stub, sync with YouTube for Google accounts
-# def playlist_ajax(video_ids, source_playlist_id, name, privacy, action, env_headers)
-# headers = HTTP::Headers.new
-# headers["Cookie"] = env_headers["Cookie"]
-#
-# html = YT_POOL.client &.get("/view_all_playlists?disable_polymer=1", headers)
-#
-# cookies = HTTP::Cookies.from_headers(headers)
-# html.cookies.each do |cookie|
-# if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name
-# if cookies[cookie.name]?
-# cookies[cookie.name] = cookie
-# else
-# cookies << cookie
-# end
-# end
-# end
-# headers = cookies.add_request_headers(headers)
-#
-# if match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[A-Za-z0-9\_\-\=]+)"/)
-# session_token = match["session_token"]
-#
-# headers["content-type"] = "application/x-www-form-urlencoded"
-#
-# post_req = {
-# video_ids: [] of String,
-# source_playlist_id: "",
-# n: name,
-# p: privacy,
-# session_token: session_token,
-# }
-# post_url = "/playlist_ajax?#{action}=1"
-#
-# response = client.post(post_url, headers, form: post_req)
-# if response.status_code == 200
-# return JSON.parse(response.body)["result"]["playlistId"].as_s
-# else
-# return nil
-# end
-# end
-# end
-
def get_subscription_feed(db, user, max_results = 40, page = 1)
limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE)
offset = (page - 1) * limit
@@ -350,6 +497,7 @@ def get_subscription_feed(db, user, max_results = 40, page = 1)
notifications.sort_by! { |video| video.author }
when "channel name - reverse"
notifications.sort_by! { |video| video.author }.reverse!
+ else nil # Ignore
end
else
if user.preferences.latest_only
@@ -398,6 +546,7 @@ def get_subscription_feed(db, user, max_results = 40, page = 1)
videos.sort_by! { |video| video.author }
when "channel name - reverse"
videos.sort_by! { |video| video.author }.reverse!
+ else nil # Ignore
end
notifications = PG_DB.query_one("SELECT notifications FROM users WHERE email = $1", user.email, as: Array(String))
diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr
index 1c7599f8..8e314fe0 100644
--- a/src/invidious/videos.cr
+++ b/src/invidious/videos.cr
@@ -222,53 +222,73 @@ VIDEO_FORMATS = {
}
struct VideoPreferences
- json_mapping({
- annotations: Bool,
- autoplay: Bool,
- comments: Array(String),
- continue: Bool,
- continue_autoplay: Bool,
- controls: Bool,
- listen: Bool,
- local: Bool,
- preferred_captions: Array(String),
- player_style: String,
- quality: String,
- raw: Bool,
- region: String?,
- related_videos: Bool,
- speed: (Float32 | Float64),
- video_end: (Float64 | Int32),
- video_loop: Bool,
- video_start: (Float64 | Int32),
- volume: Int32,
- })
+ include JSON::Serializable
+
+ property annotations : Bool
+ property autoplay : Bool
+ property comments : Array(String)
+ property continue : Bool
+ property continue_autoplay : Bool
+ property controls : Bool
+ property listen : Bool
+ property local : Bool
+ property preferred_captions : Array(String)
+ property player_style : String
+ property quality : String
+ property raw : Bool
+ property region : String?
+ property related_videos : Bool
+ property speed : Float32 | Float64
+ property video_end : Float64 | Int32
+ property video_loop : Bool
+ property video_start : Float64 | Int32
+ property volume : Int32
end
struct Video
- property player_json : JSON::Any?
- property recommended_json : JSON::Any?
+ include DB::Serializable
+
+ property id : String
+
+ @[DB::Field(converter: Video::JSONConverter)]
+ property info : Hash(String, JSON::Any)
+ property updated : Time
+
+ @[DB::Field(ignore: true)]
+ property captions : Array(Caption)?
+
+ @[DB::Field(ignore: true)]
+ property adaptive_fmts : Array(Hash(String, JSON::Any))?
+
+ @[DB::Field(ignore: true)]
+ property fmt_stream : Array(Hash(String, JSON::Any))?
+
+ @[DB::Field(ignore: true)]
+ property description : String?
- module HTTPParamConverter
+ module JSONConverter
def self.from_rs(rs)
- HTTP::Params.parse(rs.read(String))
+ JSON.parse(rs.read(String)).as_h
end
end
- def to_json(locale, config, kemal_config, decrypt_function, json : JSON::Builder)
+ def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "video"
json.field "title", self.title
json.field "videoId", self.id
+
+ json.field "error", info["reason"] if info["reason"]?
+
json.field "videoThumbnails" do
- generate_thumbnails(json, self.id, config, kemal_config)
+ generate_thumbnails(json, self.id)
end
json.field "storyboards" do
- generate_storyboards(json, self.id, self.storyboards, config, kemal_config)
+ generate_storyboards(json, self.id, self.storyboards)
end
- json.field "description", html_to_content(self.description_html)
+ json.field "description", self.description
json.field "descriptionHtml", self.description_html
json.field "published", self.published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
@@ -307,43 +327,39 @@ struct Video
json.field "lengthSeconds", self.length_seconds
json.field "allowRatings", self.allow_ratings
- json.field "rating", self.info["avg_rating"].to_f32
+ json.field "rating", self.average_rating
json.field "isListed", self.is_listed
json.field "liveNow", self.live_now
json.field "isUpcoming", self.is_upcoming
if self.premiere_timestamp
- json.field "premiereTimestamp", self.premiere_timestamp.not_nil!.to_unix
+ json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
end
- if player_response["streamingData"]?.try &.["hlsManifestUrl"]?
- host_url = make_host_url(config, kemal_config)
-
- hlsvp = player_response["streamingData"]["hlsManifestUrl"].as_s
- hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", host_url)
-
+ if hlsvp = self.hls_manifest_url
+ hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL)
json.field "hlsUrl", hlsvp
end
- json.field "dashUrl", "#{make_host_url(config, kemal_config)}/api/manifest/dash/id/#{id}"
+ json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{id}"
json.field "adaptiveFormats" do
json.array do
- self.adaptive_fmts(decrypt_function).each do |fmt|
+ self.adaptive_fmts.each do |fmt|
json.object do
- json.field "index", fmt["index"]
- json.field "bitrate", fmt["bitrate"]
- json.field "init", fmt["init"]
+ json.field "index", "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}"
+ json.field "bitrate", fmt["bitrate"].as_i.to_s
+ json.field "init", "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}"
json.field "url", fmt["url"]
- json.field "itag", fmt["itag"]
- json.field "type", fmt["type"]
- json.field "clen", fmt["clen"]
- json.field "lmt", fmt["lmt"]
- json.field "projectionType", fmt["projection_type"]
+ json.field "itag", fmt["itag"].as_i.to_s
+ json.field "type", fmt["mimeType"]
+ json.field "clen", fmt["contentLength"]
+ json.field "lmt", fmt["lastModified"]
+ json.field "projectionType", fmt["projectionType"]
fmt_info = itag_to_metadata?(fmt["itag"])
if fmt_info
- fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.to_i || 30
+ fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
json.field "fps", fps
json.field "container", fmt_info["ext"]
json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
@@ -369,16 +385,16 @@ struct Video
json.field "formatStreams" do
json.array do
- self.fmt_stream(decrypt_function).each do |fmt|
+ self.fmt_stream.each do |fmt|
json.object do
json.field "url", fmt["url"]
- json.field "itag", fmt["itag"]
- json.field "type", fmt["type"]
+ json.field "itag", fmt["itag"].as_i.to_s
+ json.field "type", fmt["mimeType"]
json.field "quality", fmt["quality"]
fmt_info = itag_to_metadata?(fmt["itag"])
if fmt_info
- fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.to_i || 30
+ fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
json.field "fps", fps
json.field "container", fmt_info["ext"]
json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
@@ -416,15 +432,13 @@ struct Video
json.field "recommendedVideos" do
json.array do
- self.info["rvs"]?.try &.split(",").each do |rv|
- rv = HTTP::Params.parse(rv)
-
+ self.related_videos.each do |rv|
if rv["id"]?
json.object do
json.field "videoId", rv["id"]
json.field "title", rv["title"]
json.field "videoThumbnails" do
- generate_thumbnails(json, rv["id"], config, kemal_config)
+ generate_thumbnails(json, rv["id"])
end
json.field "author", rv["author"]
@@ -437,7 +451,7 @@ struct Video
qualities.each do |quality|
json.object do
- json.field "url", rv["author_thumbnail"].gsub(/s\d+-/, "s#{quality}-")
+ json.field "url", rv["author_thumbnail"]?.try &.gsub(/s\d+-/, "s#{quality}-")
json.field "width", quality
json.field "height", quality
end
@@ -446,9 +460,9 @@ struct Video
end
end
- json.field "lengthSeconds", rv["length_seconds"].to_i
- json.field "viewCountText", rv["short_view_count_text"]
- json.field "viewCount", rv["view_count"]?.try &.to_i64
+ json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i
+ json.field "viewCountText", rv["short_view_count_text"]?
+ json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64
end
end
end
@@ -457,266 +471,164 @@ struct Video
end
end
- def to_json(locale, config, kemal_config, decrypt_function, json : JSON::Builder | Nil = nil)
+ def to_json(locale, json : JSON::Builder | Nil = nil)
if json
- to_json(locale, config, kemal_config, decrypt_function, json)
+ to_json(locale, json)
else
JSON.build do |json|
- to_json(locale, config, kemal_config, decrypt_function, json)
+ to_json(locale, json)
end
end
end
- # `description_html` is stored in DB as `description`, which can be
- # quite confusing. Since it currently isn't very practical to rename
- # it, we instead define a getter and setter here.
- def description_html
- self.description
+ def title
+ info["videoDetails"]["title"]?.try &.as_s || ""
end
- def description_html=(other : String)
- self.description = other
+ def ucid
+ info["videoDetails"]["channelId"]?.try &.as_s || ""
end
- def allow_ratings
- allow_ratings = player_response["videoDetails"]?.try &.["allowRatings"]?.try &.as_bool
-
- if allow_ratings.nil?
- return true
- end
+ def author
+ info["videoDetails"]["author"]?.try &.as_s || ""
+ end
- return allow_ratings
+ def length_seconds : Int32
+ info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["lengthSeconds"]?.try &.as_s.to_i ||
+ info["videoDetails"]["lengthSeconds"]?.try &.as_s.to_i || 0
end
- def live_now
- live_now = player_response["videoDetails"]?.try &.["isLive"]?.try &.as_bool
+ def views : Int64
+ info["videoDetails"]["viewCount"]?.try &.as_s.to_i64 || 0_i64
+ end
- if live_now.nil?
- return false
- end
+ def likes : Int64
+ info["likes"]?.try &.as_i64 || 0_i64
+ end
- return live_now
+ def dislikes : Int64
+ info["dislikes"]?.try &.as_i64 || 0_i64
end
- def is_listed
- is_listed = player_response["videoDetails"]?.try &.["isCrawlable"]?.try &.as_bool
+ def average_rating : Float64
+ # (likes / (likes + dislikes) * 4 + 1)
+ info["videoDetails"]["averageRating"]?.try { |t| t.as_f? || t.as_i64?.try &.to_f64 }.try &.round(4) || 0.0
+ end
- if is_listed.nil?
- return true
- end
+ def published : Time
+ info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["publishDate"]?.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location.local) } || Time.local
+ end
- return is_listed
+ def published=(other : Time)
+ info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d"))
end
- def is_upcoming
- is_upcoming = player_response["videoDetails"]?.try &.["isUpcoming"]?.try &.as_bool
+ def cookie
+ info["cookie"]?.try &.as_h.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
+ end
- if is_upcoming.nil?
- return false
- end
+ def allow_ratings
+ r = info["videoDetails"]["allowRatings"]?.try &.as_bool
+ r.nil? ? false : r
+ end
- return is_upcoming
+ def live_now
+ info["videoDetails"]["isLiveContent"]?.try &.as_bool || false
end
- def premiere_timestamp
- if self.is_upcoming
- premiere_timestamp = player_response["playabilityStatus"]?
- .try &.["liveStreamability"]?
- .try &.["liveStreamabilityRenderer"]?
- .try &.["offlineSlate"]?
- .try &.["liveStreamOfflineSlateRenderer"]?
- .try &.["scheduledStartTime"]?.try &.as_s.to_i64
- end
+ def is_listed
+ info["videoDetails"]["isCrawlable"]?.try &.as_bool || false
+ end
- if premiere_timestamp
- premiere_timestamp = Time.unix(premiere_timestamp)
- end
+ def is_upcoming
+ info["videoDetails"]["isUpcoming"]?.try &.as_bool || false
+ end
- return premiere_timestamp
+ def premiere_timestamp : Time?
+ info["microformat"]?.try &.["playerMicroformatRenderer"]?
+ .try &.["liveBroadcastDetails"]?.try &.["startTimestamp"]?.try { |t| Time.parse_rfc3339(t.as_s) }
end
def keywords
- keywords = player_response["videoDetails"]?.try &.["keywords"]?.try &.as_a
- keywords ||= [] of String
-
- return keywords
+ info["videoDetails"]["keywords"]?.try &.as_a.map &.as_s || [] of String
end
- def fmt_stream(decrypt_function)
- streams = [] of HTTP::Params
-
- if fmt_streams = player_response["streamingData"]?.try &.["formats"]?
- fmt_streams.as_a.each do |fmt_stream|
- if !fmt_stream.as_h?
- next
- end
-
- fmt = {} of String => String
+ def related_videos
+ info["relatedVideos"]?.try &.as_a.map { |h| h.as_h.transform_values &.as_s } || [] of Hash(String, String)
+ end
- fmt["lmt"] = fmt_stream["lastModified"]?.try &.as_s || "0"
- fmt["projection_type"] = "1"
- fmt["type"] = fmt_stream["mimeType"].as_s
- fmt["clen"] = fmt_stream["contentLength"]?.try &.as_s || "0"
- fmt["bitrate"] = fmt_stream["bitrate"]?.try &.as_i.to_s || "0"
- fmt["itag"] = fmt_stream["itag"].as_i.to_s
- if fmt_stream["url"]?
- fmt["url"] = fmt_stream["url"].as_s
- end
- if fmt_stream["cipher"]?
- HTTP::Params.parse(fmt_stream["cipher"].as_s).each do |key, value|
- fmt[key] = value
- end
- end
- fmt["quality"] = fmt_stream["quality"].as_s
+ def allowed_regions
+ info["microformat"]?.try &.["playerMicroformatRenderer"]?
+ .try &.["availableCountries"]?.try &.as_a.map &.as_s || [] of String
+ end
- if fmt_stream["width"]?
- fmt["size"] = "#{fmt_stream["width"]}x#{fmt_stream["height"]}"
- fmt["height"] = fmt_stream["height"].as_i.to_s
- end
+ def author_thumbnail : String
+ info["authorThumbnail"]?.try &.as_s || ""
+ end
- if fmt_stream["fps"]?
- fmt["fps"] = fmt_stream["fps"].as_i.to_s
- end
+ def sub_count_text : String
+ info["subCountText"]?.try &.as_s || "-"
+ end
- if fmt_stream["qualityLabel"]?
- fmt["quality_label"] = fmt_stream["qualityLabel"].as_s
- end
+ def fmt_stream
+ return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream
- params = HTTP::Params.new
- fmt.each do |key, value|
- params[key] = value
+ fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
+ fmt_stream.each do |fmt|
+ if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
+ s.each do |k, v|
+ fmt[k] = JSON::Any.new(v)
end
-
- streams << params
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}#{decrypt_signature(fmt)}")
end
- streams.sort_by! { |stream| stream["height"].to_i }.reverse!
- elsif fmt_stream = self.info["url_encoded_fmt_stream_map"]?
- fmt_stream.split(",").each do |string|
- if !string.empty?
- streams << HTTP::Params.parse(string)
- end
- end
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}&region=#{self.info["region"]}") if self.info["region"]?
end
-
- streams.each { |s| s.add("label", "#{s["quality"]} - #{s["type"].split(";")[0].split("/")[1]}") }
- streams = streams.uniq { |s| s["label"] }
-
- if self.info["region"]?
- streams.each do |fmt|
- fmt["url"] += "&region=" + self.info["region"]
- end
- end
-
- streams.each do |fmt|
- fmt["url"] += "&host=" + (URI.parse(fmt["url"]).host || "")
- fmt["url"] += decrypt_signature(fmt, decrypt_function)
- end
-
- return streams
+ fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
+ @fmt_stream = fmt_stream
+ return @fmt_stream.as(Array(Hash(String, JSON::Any)))
end
- def adaptive_fmts(decrypt_function)
- adaptive_fmts = [] of HTTP::Params
-
- if fmts = player_response["streamingData"]?.try &.["adaptiveFormats"]?
- fmts.as_a.each do |adaptive_fmt|
- next if !adaptive_fmt.as_h?
- fmt = {} of String => String
-
- if init = adaptive_fmt["initRange"]?
- fmt["init"] = "#{init["start"]}-#{init["end"]}"
- end
- fmt["init"] ||= "0-0"
-
- fmt["lmt"] = adaptive_fmt["lastModified"]?.try &.as_s || "0"
- fmt["projection_type"] = "1"
- fmt["type"] = adaptive_fmt["mimeType"].as_s
- fmt["clen"] = adaptive_fmt["contentLength"]?.try &.as_s || "0"
- fmt["bitrate"] = adaptive_fmt["bitrate"]?.try &.as_i.to_s || "0"
- fmt["itag"] = adaptive_fmt["itag"].as_i.to_s
- if adaptive_fmt["url"]?
- fmt["url"] = adaptive_fmt["url"].as_s
- end
- if adaptive_fmt["cipher"]?
- HTTP::Params.parse(adaptive_fmt["cipher"].as_s).each do |key, value|
- fmt[key] = value
- end
- end
- if index = adaptive_fmt["indexRange"]?
- fmt["index"] = "#{index["start"]}-#{index["end"]}"
- end
- fmt["index"] ||= "0-0"
-
- if adaptive_fmt["width"]?
- fmt["size"] = "#{adaptive_fmt["width"]}x#{adaptive_fmt["height"]}"
- end
-
- if adaptive_fmt["fps"]?
- fmt["fps"] = adaptive_fmt["fps"].as_i.to_s
- end
-
- if adaptive_fmt["qualityLabel"]?
- fmt["quality_label"] = adaptive_fmt["qualityLabel"].as_s
+ def adaptive_fmts
+ return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts
+ fmt_stream = info["streamingData"]?.try &.["adaptiveFormats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
+ fmt_stream.each do |fmt|
+ if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
+ s.each do |k, v|
+ fmt[k] = JSON::Any.new(v)
end
-
- params = HTTP::Params.new
- fmt.each do |key, value|
- params[key] = value
- end
-
- adaptive_fmts << params
- end
- elsif fmts = self.info["adaptive_fmts"]?
- fmts.split(",") do |string|
- adaptive_fmts << HTTP::Params.parse(string)
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}#{decrypt_signature(fmt)}")
end
- end
-
- if self.info["region"]?
- adaptive_fmts.each do |fmt|
- fmt["url"] += "&region=" + self.info["region"]
- end
- end
- adaptive_fmts.each do |fmt|
- fmt["url"] += "&host=" + (URI.parse(fmt["url"]).host || "")
- fmt["url"] += decrypt_signature(fmt, decrypt_function)
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}&region=#{self.info["region"]}") if self.info["region"]?
end
-
- return adaptive_fmts
+ # See https://github.com/TeamNewPipe/NewPipe/issues/2415
+ # Some streams are segmented by URL `sq/` rather than index, for now we just filter them out
+ fmt_stream.reject! { |f| !f["indexRange"]? }
+ fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
+ @adaptive_fmts = fmt_stream
+ return @adaptive_fmts.as(Array(Hash(String, JSON::Any)))
end
- def video_streams(adaptive_fmts)
- video_streams = adaptive_fmts.select { |s| s["type"].starts_with? "video" }
-
- return video_streams
+ def video_streams
+ adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("video")
end
- def audio_streams(adaptive_fmts)
- audio_streams = adaptive_fmts.select { |s| s["type"].starts_with? "audio" }
- audio_streams.sort_by! { |s| s["bitrate"].to_i }.reverse!
- audio_streams.each do |stream|
- stream["bitrate"] = (stream["bitrate"].to_f64/1000).to_i.to_s
- end
-
- return audio_streams
- end
-
- def player_response
- @player_json = JSON.parse(@info["player_response"]) if !@player_json
- @player_json.not_nil!
+ def audio_streams
+ adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("audio")
end
def storyboards
- storyboards = player_response["storyboards"]?
+ storyboards = info["storyboards"]?
.try &.as_h
.try &.["playerStoryboardSpecRenderer"]?
.try &.["spec"]?
.try &.as_s.split("|")
if !storyboards
- if storyboard = player_response["storyboards"]?
+ if storyboard = info["storyboards"]?
.try &.as_h
.try &.["playerLiveStoryboardSpecRenderer"]?
.try &.["spec"]?
@@ -744,9 +656,7 @@ struct Video
storyboard_height: Int32,
storyboard_count: Int32)
- if !storyboards
- return items
- end
+ return items if !storyboards
url = URI.parse(storyboards.shift)
params = HTTP::Params.parse(url.query || "")
@@ -780,220 +690,169 @@ struct Video
end
def paid
- reason = player_response["playabilityStatus"]?.try &.["reason"]?
+ reason = info["playabilityStatus"]?.try &.["reason"]?
paid = reason == "This video requires payment to watch." ? true : false
-
- return paid
+ paid
end
def premium
- if info["premium"]?
- self.info["premium"] == "true"
- else
- false
+ keywords.includes? "YouTube Red"
+ end
+
+ def captions : Array(Caption)
+ return @captions.as(Array(Caption)) if @captions
+ captions = info["captions"]?.try &.["playerCaptionsTracklistRenderer"]?.try &.["captionTracks"]?.try &.as_a.map do |caption|
+ caption = Caption.from_json(caption.to_json)
+ caption.name.simpleText = caption.name.simpleText.split(" - ")[0]
+ caption
end
+ captions ||= [] of Caption
+ @captions = captions
+ return @captions.as(Array(Caption))
end
- def captions
- captions = [] of Caption
- if player_response["captions"]?
- caption_list = player_response["captions"]["playerCaptionsTracklistRenderer"]["captionTracks"]?.try &.as_a
- caption_list ||= [] of JSON::Any
+ def description
+ description = info["microformat"]?.try &.["playerMicroformatRenderer"]?
+ .try &.["description"]?.try &.["simpleText"]?.try &.as_s || ""
+ end
- caption_list.each do |caption|
- caption = Caption.from_json(caption.to_json)
- caption.name.simpleText = caption.name.simpleText.split(" - ")[0]
- captions << caption
- end
- end
+ # TODO
+ def description=(value : String)
+ @description = value
+ end
- return captions
+ def description_html
+ info["descriptionHtml"]?.try &.as_s || "<p></p>"
end
- def short_description
- short_description = self.description_html.gsub(/(<br>)|(<br\/>|"|\n)/, {
- "<br>": " ",
- "<br/>": " ",
- "\"": "&quot;",
- "\n": " ",
- })
- short_description = XML.parse_html(short_description).content[0..200].strip(" ")
-
- if short_description.empty?
- short_description = " "
- end
+ def description_html=(value : String)
+ info["descriptionHtml"] = JSON::Any.new(value)
+ end
- return short_description
- end
-
- def length_seconds
- player_response["videoDetails"]["lengthSeconds"].as_s.to_i
- end
-
- db_mapping({
- id: String,
- info: {
- type: HTTP::Params,
- default: HTTP::Params.parse(""),
- converter: Video::HTTPParamConverter,
- },
- updated: Time,
- title: String,
- views: Int64,
- likes: Int32,
- dislikes: Int32,
- wilson_score: Float64,
- published: Time,
- description: String,
- language: String?,
- author: String,
- ucid: String,
- allowed_regions: Array(String),
- is_family_friendly: Bool,
- genre: String,
- genre_url: String,
- license: String,
- sub_count_text: String,
- author_thumbnail: String,
- })
-end
+ def short_description
+ info["shortDescription"]?.try &.as_s? || ""
+ end
-struct Caption
- json_mapping({
- name: CaptionName,
- baseUrl: String,
- languageCode: String,
- })
-end
+ def hls_manifest_url : String?
+ info["streamingData"]?.try &.["hlsManifestUrl"]?.try &.as_s
+ end
-struct CaptionName
- json_mapping({
- simpleText: String,
- })
-end
+ def dash_manifest_url
+ info["streamingData"]?.try &.["dashManifestUrl"]?.try &.as_s
+ end
-class VideoRedirect < Exception
- property video_id : String
+ def genre : String
+ info["genre"]?.try &.as_s || ""
+ end
- def initialize(@video_id)
+ def genre_url : String?
+ info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil
end
-end
-def get_video(id, db, refresh = true, region = nil, force_refresh = false)
- if (video = db.query_one?("SELECT * FROM videos WHERE id = $1", id, as: Video)) && !region
- # If record was last updated over 10 minutes ago, or video has since premiered,
- # refresh (expire param in response lasts for 6 hours)
- if (refresh &&
- (Time.utc - video.updated > 10.minutes) ||
- (video.premiere_timestamp && video.premiere_timestamp.as(Time) < Time.utc)) ||
- force_refresh
- begin
- video = fetch_video(id, region)
- video_array = video.to_a
+ def license : String?
+ info["license"]?.try &.as_s
+ end
- args = arg_array(video_array[1..-1], 2)
+ def is_family_friendly : Bool
+ info["microformat"]?.try &.["playerMicroformatRenderer"]["isFamilySafe"]?.try &.as_bool || false
+ end
- db.exec("UPDATE videos SET (info,updated,title,views,likes,dislikes,wilson_score,\
- published,description,language,author,ucid,allowed_regions,is_family_friendly,\
- genre,genre_url,license,sub_count_text,author_thumbnail)\
- = (#{args}) WHERE id = $1", args: video_array)
- rescue ex
- db.exec("DELETE FROM videos * WHERE id = $1", id)
- raise ex
- end
- end
- else
- video = fetch_video(id, region)
- video_array = video.to_a
+ def wilson_score : Float64
+ ci_lower_bound(likes, likes + dislikes).round(4)
+ end
- args = arg_array(video_array)
+ def engagement : Float64
+ ((likes + dislikes) / views).round(4)
+ end
- if !region
- db.exec("INSERT INTO videos VALUES (#{args}) ON CONFLICT (id) DO NOTHING", args: video_array)
- end
+ def reason : String?
+ info["reason"]?.try &.as_s
end
- return video
+ def session_token : String?
+ info["sessionToken"]?.try &.as_s?
+ end
end
-def extract_recommended(recommended_videos)
- rvs = [] of HTTP::Params
+struct CaptionName
+ include JSON::Serializable
- recommended_videos.try &.each do |compact_renderer|
- if compact_renderer["compactRadioRenderer"]? || compact_renderer["compactPlaylistRenderer"]?
- # TODO
- elsif video_renderer = compact_renderer["compactVideoRenderer"]?
- recommended_video = HTTP::Params.new
- recommended_video["id"] = video_renderer["videoId"].as_s
- recommended_video["title"] = video_renderer["title"]["simpleText"].as_s
+ property simpleText : String
+end
- next if !video_renderer["shortBylineText"]?
+struct Caption
+ include JSON::Serializable
- recommended_video["author"] = video_renderer["shortBylineText"]["runs"].as_a[0]["text"].as_s
- recommended_video["ucid"] = video_renderer["shortBylineText"]["runs"].as_a[0]["navigationEndpoint"]["browseEndpoint"]["browseId"].as_s
- recommended_video["author_thumbnail"] = video_renderer["channelThumbnail"]["thumbnails"][0]["url"].as_s
+ property name : CaptionName
+ property baseUrl : String
+ property languageCode : String
+end
- if view_count = video_renderer["viewCountText"]?.try { |field| field["simpleText"]?.try &.as_s || field["runs"][0]?.try &.["text"].as_s }.try &.delete(", views watching").to_i64?.try &.to_s
- recommended_video["view_count"] = view_count
- recommended_video["short_view_count_text"] = "#{number_to_short_text(view_count.to_i64)} views"
- end
- recommended_video["length_seconds"] = decode_length_seconds(video_renderer["lengthText"]?.try &.["simpleText"]?.try &.as_s || "0:00").to_s
+class VideoRedirect < Exception
+ property video_id : String
- rvs << recommended_video
- end
+ def initialize(@video_id)
end
-
- rvs
end
-def extract_polymer_config(body, html)
- params = HTTP::Params.new
-
- params["session_token"] = body.match(/"XSRF_TOKEN":"(?<session_token>[A-Za-z0-9\_\-\=]+)"/).try &.["session_token"] || ""
-
- html_info = JSON.parse(body.match(/ytplayer\.config = (?<info>.*?);ytplayer\.load/).try &.["info"] || "{}").try &.["args"]?.try &.as_h
+def parse_related(r : JSON::Any) : JSON::Any?
+ # TODO: r["endScreenPlaylistRenderer"], etc.
+ return if !r["endScreenVideoRenderer"]?
+ r = r["endScreenVideoRenderer"].as_h
+
+ return if !r["lengthInSeconds"]?
+
+ rv = {} of String => JSON::Any
+ rv["author"] = r["shortBylineText"]["runs"][0]?.try &.["text"] || JSON::Any.new("")
+ rv["ucid"] = r["shortBylineText"]["runs"][0]?.try &.["navigationEndpoint"]["browseEndpoint"]["browseId"] || JSON::Any.new("")
+ rv["author_url"] = JSON::Any.new("/channel/#{rv["ucid"]}")
+ rv["length_seconds"] = JSON::Any.new(r["lengthInSeconds"].as_i.to_s)
+ rv["title"] = r["title"]["simpleText"]
+ rv["short_view_count_text"] = JSON::Any.new(r["shortViewCountText"]?.try &.["simpleText"]?.try &.as_s || "")
+ rv["view_count"] = JSON::Any.new(r["title"]["accessibility"]?.try &.["accessibilityData"]["label"].as_s.match(/(?<views>[1-9](\d+,?)*) views/).try &.["views"].gsub(/\D/, "") || "")
+ rv["id"] = r["videoId"]
+ JSON::Any.new(rv)
+end
- if html_info
- html_info.each do |key, value|
- params[key] = value.to_s
- end
+def extract_polymer_config(body)
+ params = {} of String => JSON::Any
+ player_response = body.match(/window\["ytInitialPlayerResponse"\]\s*=\s*(?<info>.*?);\n/)
+ .try { |r| JSON.parse(r["info"]).as_h }
+
+ if body.includes?("To continue with your YouTube experience, please fill out the form below.") ||
+ body.includes?("https://www.google.com/sorry/index")
+ params["reason"] = JSON::Any.new("Could not extract video info. Instance is likely blocked.")
+ elsif !player_response
+ params["reason"] = JSON::Any.new("Video unavailable.")
+ elsif player_response["playabilityStatus"]?.try &.["status"]?.try &.as_s != "OK"
+ reason = player_response["playabilityStatus"]["errorScreen"]?.try &.["playerErrorMessageRenderer"]?.try &.["subreason"]?.try { |s| s["simpleText"]?.try &.as_s || s["runs"].as_a.map { |r| r["text"] }.join("") } ||
+ player_response["playabilityStatus"]["reason"].as_s
+ params["reason"] = JSON::Any.new(reason)
end
- initial_data = extract_initial_data(body)
-
- primary_results = initial_data["contents"]?
- .try &.["twoColumnWatchNextResults"]?
- .try &.["results"]?
- .try &.["results"]?
- .try &.["contents"]?
+ params["sessionToken"] = JSON::Any.new(body.match(/"XSRF_TOKEN":"(?<session_token>[^"]+)"/).try &.["session_token"]?)
+ params["shortDescription"] = JSON::Any.new(body.match(/"og:description" content="(?<description>[^"]+)"/).try &.["description"]?)
- comment_continuation = primary_results.try &.as_a.select { |object| object["itemSectionRenderer"]? }[0]?
- .try &.["itemSectionRenderer"]?
- .try &.["continuations"]?
- .try &.[0]?
- .try &.["nextContinuationData"]?
+ return params if !player_response
- params["ctoken"] = comment_continuation.try &.["continuation"]?.try &.as_s || ""
- params["itct"] = comment_continuation.try &.["clickTrackingParams"]?.try &.as_s || ""
+ {"captions", "microformat", "playabilityStatus", "storyboards", "videoDetails"}.each do |f|
+ params[f] = player_response[f] if player_response[f]?
+ end
- rvs = initial_data["contents"]?
- .try &.["twoColumnWatchNextResults"]?
- .try &.["secondaryResults"]?
- .try &.["secondaryResults"]?
- .try &.["results"]?
- .try &.as_a
+ yt_initial_data = body.match(/(window\["ytInitialData"\]|var\s+ytInitialData)\s*=\s*(?<info>.*?);\s*\n/)
+ .try { |r| JSON.parse(r["info"]).as_h }
- params["rvs"] = extract_recommended(rvs).join(",")
-
- # TODO: Watching now
- params["views"] = primary_results.try &.as_a.select { |object| object["videoPrimaryInfoRenderer"]? }[0]?
- .try &.["videoPrimaryInfoRenderer"]?
- .try &.["viewCount"]?
- .try &.["videoViewCountRenderer"]?
- .try &.["viewCount"]?
- .try &.["simpleText"]?
- .try &.as_s.gsub(/\D/, "").to_i64.to_s || "0"
+ params["relatedVideos"] = yt_initial_data.try &.["playerOverlays"]?.try &.["playerOverlayRenderer"]?
+ .try &.["endScreen"]?.try &.["watchNextEndScreenRenderer"]?.try &.["results"]?.try &.as_a.compact_map { |r|
+ parse_related r
+ }.try { |a| JSON::Any.new(a) } || yt_initial_data.try &.["webWatchNextResponseExtensionData"]?.try &.["relatedVideoArgs"]?
+ .try &.as_s.split(",").map { |r|
+ r = HTTP::Params.parse(r).to_h
+ JSON::Any.new(Hash.zip(r.keys, r.values.map { |v| JSON::Any.new(v) }))
+ }.try { |a| JSON::Any.new(a) } || JSON::Any.new([] of JSON::Any)
+ primary_results = yt_initial_data.try &.["contents"]?.try &.["twoColumnWatchNextResults"]?.try &.["results"]?
+ .try &.["results"]?.try &.["contents"]?
sentiment_bar = primary_results.try &.as_a.select { |object| object["videoPrimaryInfoRenderer"]? }[0]?
.try &.["videoPrimaryInfoRenderer"]?
.try &.["sentimentBar"]?
@@ -1001,34 +860,13 @@ def extract_polymer_config(body, html)
.try &.["tooltip"]?
.try &.as_s
- likes, dislikes = sentiment_bar.try &.split(" / ").map { |a| a.delete(", ").to_i32 }[0, 2] || {0, 0}
-
- params["likes"] = "#{likes}"
- params["dislikes"] = "#{dislikes}"
-
- published = primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]?
- .try &.["videoSecondaryInfoRenderer"]?
- .try &.["dateText"]?
- .try &.["simpleText"]?
- .try &.as_s.split(" ")[-3..-1].join(" ")
-
- if published
- params["published"] = Time.parse(published, "%b %-d, %Y", Time::Location.local).to_unix.to_s
- else
- params["published"] = Time.utc(1990, 1, 1).to_unix.to_s
- end
-
- params["description_html"] = "<p></p>"
+ likes, dislikes = sentiment_bar.try &.split(" / ", 2).map &.gsub(/\D/, "").to_i64 || {0_i64, 0_i64}
+ params["likes"] = JSON::Any.new(likes)
+ params["dislikes"] = JSON::Any.new(dislikes)
- description_html = primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]?
- .try &.["videoSecondaryInfoRenderer"]?
- .try &.["description"]?
- .try &.["runs"]?
- .try &.as_a
-
- if description_html
- params["description_html"] = content_to_comment_html(description_html)
- end
+ params["descriptionHtml"] = JSON::Any.new(primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]?
+ .try &.["videoSecondaryInfoRenderer"]?.try &.["description"]?.try &.["runs"]?
+ .try &.as_a.try { |t| content_to_comment_html(t).gsub("\n", "<br/>") } || "<p></p>")
metadata = primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]?
.try &.["videoSecondaryInfoRenderer"]?
@@ -1037,9 +875,8 @@ def extract_polymer_config(body, html)
.try &.["rows"]?
.try &.as_a
- params["genre"] = ""
- params["genre_ucid"] = ""
- params["license"] = ""
+ params["genre"] = params["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["category"]? || JSON::Any.new("")
+ params["genreUrl"] = JSON::Any.new(nil)
metadata.try &.each do |row|
title = row["metadataRowRenderer"]?.try &.["title"]?.try &.["simpleText"]?.try &.as_s
@@ -1051,219 +888,130 @@ def extract_polymer_config(body, html)
contents = contents.try &.["runs"]?
.try &.as_a[0]?
- params["genre"] = contents.try &.["text"]?
- .try &.as_s || ""
- params["genre_ucid"] = contents.try &.["navigationEndpoint"]?
- .try &.["browseEndpoint"]?
- .try &.["browseId"]?.try &.as_s || ""
+ params["genre"] = JSON::Any.new(contents.try &.["text"]?.try &.as_s || "")
+ params["genreUcid"] = JSON::Any.new(contents.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]?
+ .try &.["browseId"]?.try &.as_s || "")
elsif title.try &.== "License"
contents = contents.try &.["runs"]?
.try &.as_a[0]?
- params["license"] = contents.try &.["text"]?
- .try &.as_s || ""
+ params["license"] = JSON::Any.new(contents.try &.["text"]?.try &.as_s || "")
elsif title.try &.== "Licensed to YouTube by"
- params["license"] = contents.try &.["simpleText"]?
- .try &.as_s || ""
+ params["license"] = JSON::Any.new(contents.try &.["simpleText"]?.try &.as_s || "")
end
end
author_info = primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]?
- .try &.["videoSecondaryInfoRenderer"]?
- .try &.["owner"]?
- .try &.["videoOwnerRenderer"]?
+ .try &.["videoSecondaryInfoRenderer"]?.try &.["owner"]?.try &.["videoOwnerRenderer"]?
- params["author_thumbnail"] = author_info.try &.["thumbnail"]?
- .try &.["thumbnails"]?
- .try &.as_a[0]?
- .try &.["url"]?
- .try &.as_s || ""
+ params["authorThumbnail"] = JSON::Any.new(author_info.try &.["thumbnail"]?
+ .try &.["thumbnails"]?.try &.as_a[0]?.try &.["url"]?
+ .try &.as_s || "")
- params["sub_count_text"] = author_info.try &.["subscriberCountText"]?
- .try &.["simpleText"]?
- .try &.as_s.gsub(/\D/, "") || "0"
+ params["subCountText"] = JSON::Any.new(author_info.try &.["subscriberCountText"]?
+ .try { |t| t["simpleText"]? || t["runs"]?.try &.[0]?.try &.["text"]? }.try &.as_s.split(" ", 2)[0] || "-")
- return params
-end
+ initial_data = body.match(/ytplayer\.config\s*=\s*(?<info>.*?);ytplayer\.web_player_context_config/)
+ .try { |r| JSON.parse(r["info"]) }.try &.["args"]["player_response"]?
+ .try &.as_s?.try &.try { |r| JSON.parse(r).as_h }
-def extract_player_config(body, html)
- params = HTTP::Params.new
+ return params if !initial_data
- if md = body.match(/'XSRF_TOKEN': "(?<session_token>[A-Za-z0-9\_\-\=]+)"/)
- params["session_token"] = md["session_token"]
+ {"playabilityStatus", "streamingData"}.each do |f|
+ params[f] = initial_data[f] if initial_data[f]?
end
- if md = body.match(/'RELATED_PLAYER_ARGS': (?<json>.*?),\n/)
- recommended_json = JSON.parse(md["json"])
- rvs_params = recommended_json["rvs"].as_s.split(",").map { |params| HTTP::Params.parse(params) }
-
- if watch_next_response = recommended_json["watch_next_response"]?
- watch_next_json = JSON.parse(watch_next_response.as_s)
- rvs = watch_next_json["contents"]?
- .try &.["twoColumnWatchNextResults"]?
- .try &.["secondaryResults"]?
- .try &.["secondaryResults"]?
- .try &.["results"]?
- .try &.as_a
-
- rvs = extract_recommended(rvs).compact_map do |rv|
- if !rv["short_view_count_text"]?
- rv_params = rvs_params.select { |rv_params| rv_params["id"]? == (rv["id"]? || "") }[0]?
+ params
+end
- if rv_params.try &.["short_view_count_text"]?
- rv["short_view_count_text"] = rv_params.not_nil!["short_view_count_text"]
- rv
- else
- nil
- end
- else
- rv
- end
+def get_video(id, db, refresh = true, region = nil, force_refresh = false)
+ if (video = db.query_one?("SELECT * FROM videos WHERE id = $1", id, as: Video)) && !region
+ # If record was last updated over 10 minutes ago, or video has since premiered,
+ # refresh (expire param in response lasts for 6 hours)
+ if (refresh &&
+ (Time.utc - video.updated > 10.minutes) ||
+ (video.premiere_timestamp.try &.< Time.utc)) ||
+ force_refresh
+ begin
+ video = fetch_video(id, region)
+ db.exec("UPDATE videos SET (id, info, updated) = ($1, $2, $3) WHERE id = $1", video.id, video.info.to_json, video.updated)
+ rescue ex
+ db.exec("DELETE FROM videos * WHERE id = $1", id)
+ raise ex
end
- params["rvs"] = (rvs.map &.to_s).join(",")
- end
- end
-
- html_info = body.match(/ytplayer\.config = (?<info>.*?);ytplayer\.load/).try &.["info"]
-
- if html_info
- JSON.parse(html_info)["args"].as_h.each do |key, value|
- params[key] = value.to_s
end
else
- error_message = html.xpath_node(%q(//h1[@id="unavailable-message"]))
- if error_message
- params["reason"] = error_message.content.strip
- elsif body.includes?("To continue with your YouTube experience, please fill out the form below.") ||
- body.includes?("https://www.google.com/sorry/index")
- params["reason"] = "Could not extract video info. Instance is likely blocked."
- else
- params["reason"] = "Video unavailable."
+ video = fetch_video(id, region)
+ if !region
+ db.exec("INSERT INTO videos VALUES ($1, $2, $3) ON CONFLICT (id) DO NOTHING", video.id, video.info.to_json, video.updated)
end
end
- return params
+ return video
end
def fetch_video(id, region)
- response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999"))
+ response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&has_verified=1&bpctr=9999999999"))
if md = response.headers["location"]?.try &.match(/v=(?<id>[a-zA-Z0-9_-]{11})/)
raise VideoRedirect.new(video_id: md["id"])
end
- html = XML.parse_html(response.body)
- info = extract_player_config(response.body, html)
- info["cookie"] = response.cookies.to_h.map { |name, cookie| "#{name}=#{cookie.value}" }.join("; ")
-
- allowed_regions = html.xpath_node(%q(//meta[@itemprop="regionsAllowed"])).try &.["content"].split(",")
- if !allowed_regions || allowed_regions == [""]
- allowed_regions = [] of String
- end
+ info = extract_polymer_config(response.body)
+ info["cookie"] = JSON::Any.new(response.cookies.to_h.transform_values { |v| JSON::Any.new(v.value) })
+ allowed_regions = info["microformat"]?.try &.["playerMicroformatRenderer"]["availableCountries"]?.try &.as_a.map &.as_s || [] of String
# Check for region-blocks
- if info["reason"]? && info["reason"].includes?("your country")
+ if info["reason"]?.try &.as_s.includes?("your country")
bypass_regions = PROXY_LIST.keys & allowed_regions
if !bypass_regions.empty?
region = bypass_regions[rand(bypass_regions.size)]
- response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999"))
-
- html = XML.parse_html(response.body)
- info = extract_player_config(response.body, html)
+ response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&has_verified=1&bpctr=9999999999"))
- info["region"] = region if region
- info["cookie"] = response.cookies.to_h.map { |name, cookie| "#{name}=#{cookie.value}" }.join("; ")
+ region_info = extract_polymer_config(response.body)
+ region_info["region"] = JSON::Any.new(region) if region
+ region_info["cookie"] = JSON::Any.new(response.cookies.to_h.transform_values { |v| JSON::Any.new(v.value) })
+ info = region_info if !region_info["reason"]?
end
end
# Try to pull streams from embed URL
if info["reason"]?
embed_page = YT_POOL.client &.get("/embed/#{id}").body
- sts = embed_page.match(/"sts"\s*:\s*(?<sts>\d+)/).try &.["sts"]?
- sts ||= ""
- embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?video_id=#{id}&eurl=https://youtube.googleapis.com/v/#{id}&gl=US&hl=en&disable_polymer=1&sts=#{sts}").body)
+ sts = embed_page.match(/"sts"\s*:\s*(?<sts>\d+)/).try &.["sts"]? || ""
+ embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?html5=1&video_id=#{id}&eurl=https://youtube.googleapis.com/v/#{id}&gl=US&hl=en&sts=#{sts}").body)
- if !embed_info["reason"]?
- embed_info.each do |key, value|
- info[key] = value.to_s
+ if embed_info["player_response"]?
+ player_response = JSON.parse(embed_info["player_response"])
+ {"captions", "microformat", "playabilityStatus", "streamingData", "videoDetails", "storyboards"}.each do |f|
+ info[f] = player_response[f] if player_response[f]?
end
- else
- raise info["reason"]
end
- end
- if info["reason"]? && !info["player_response"]?
- raise info["reason"]
- end
+ initial_data = JSON.parse(embed_info["watch_next_response"]) if embed_info["watch_next_response"]?
- player_json = JSON.parse(info["player_response"])
- if reason = player_json["playabilityStatus"]?.try &.["reason"]?.try &.as_s
- raise reason
+ info["relatedVideos"] = initial_data.try &.["playerOverlays"]?.try &.["playerOverlayRenderer"]?
+ .try &.["endScreen"]?.try &.["watchNextEndScreenRenderer"]?.try &.["results"]?.try &.as_a.compact_map { |r|
+ parse_related r
+ }.try { |a| JSON::Any.new(a) } || embed_info["rvs"]?.try &.split(",").map { |r|
+ r = HTTP::Params.parse(r).to_h
+ JSON::Any.new(Hash.zip(r.keys, r.values.map { |v| JSON::Any.new(v) }))
+ }.try { |a| JSON::Any.new(a) } || JSON::Any.new([] of JSON::Any)
end
- title = player_json["videoDetails"]["title"].as_s
- author = player_json["videoDetails"]["author"]?.try &.as_s || ""
- ucid = player_json["videoDetails"]["channelId"]?.try &.as_s || ""
-
- info["premium"] = html.xpath_node(%q(.//span[text()="Premium"])) ? "true" : "false"
-
- views = html.xpath_node(%q(//meta[@itemprop="interactionCount"]))
- .try &.["content"].to_i64? || 0_i64
+ raise info["reason"]?.try &.as_s || "" if !info["videoDetails"]?
- likes = html.xpath_node(%q(//button[@title="I like this"]/span))
- .try &.content.delete(",").try &.to_i? || 0
-
- dislikes = html.xpath_node(%q(//button[@title="I dislike this"]/span))
- .try &.content.delete(",").try &.to_i? || 0
-
- avg_rating = (likes.to_f/(likes.to_f + dislikes.to_f) * 4 + 1)
- avg_rating = avg_rating.nan? ? 0.0 : avg_rating
- info["avg_rating"] = "#{avg_rating}"
-
- description_html = html.xpath_node(%q(//p[@id="eow-description"])).try &.to_xml(options: XML::SaveOptions::NO_DECL) || "<p></p>"
- wilson_score = ci_lower_bound(likes, likes + dislikes)
-
- published = html.xpath_node(%q(//meta[@itemprop="datePublished"])).try &.["content"]
- published ||= Time.utc.to_s("%Y-%m-%d")
- published = Time.parse(published, "%Y-%m-%d", Time::Location.local)
-
- is_family_friendly = html.xpath_node(%q(//meta[@itemprop="isFamilyFriendly"])).try &.["content"] == "True"
- is_family_friendly ||= true
-
- genre = html.xpath_node(%q(//meta[@itemprop="genre"])).try &.["content"]
- genre ||= ""
-
- genre_url = html.xpath_node(%(//ul[contains(@class, "watch-info-tag-list")]/li/a[text()="#{genre}"])).try &.["href"]?
- genre_url ||= ""
-
- # YouTube provides invalid URLs for some genres, so we fix that here
- case genre
- when "Comedy"
- genre_url = "/channel/UCQZ43c4dAA9eXCQuXWu9aTw"
- when "Education"
- genre_url = "/channel/UCdxpofrI-dO6oYfsqHDHphw"
- when "Gaming"
- genre_url = "/channel/UCOpNcN46UbXVtpKMrmU4Abg"
- when "Movies"
- genre_url = "/channel/UClgRkhTL3_hImCAmdLfDE4g"
- when "Nonprofits & Activism"
- genre_url = "/channel/UCfFyYRYslvuhwMDnx6KjUvw"
- when "Trailers"
- genre_url = "/channel/UClgRkhTL3_hImCAmdLfDE4g"
- end
-
- license = html.xpath_node(%q(//h4[contains(text(),"License")]/parent::*/ul/li)).try &.content || ""
- sub_count_text = html.xpath_node(%q(//span[contains(@class, "subscriber-count")])).try &.["title"]? || "0"
- author_thumbnail = html.xpath_node(%(//span[@class="yt-thumb-clip"]/img)).try &.["data-thumb"]?.try &.gsub(/^\/\//, "https://") || ""
-
- video = Video.new(id, info, Time.utc, title, views, likes, dislikes, wilson_score, published, description_html,
- nil, author, ucid, allowed_regions, is_family_friendly, genre, genre_url, license, sub_count_text, author_thumbnail)
+ video = Video.new({
+ id: id,
+ info: info,
+ updated: Time.utc,
+ })
return video
end
-def itag_to_metadata?(itag : String)
- return VIDEO_FORMATS[itag]?
+def itag_to_metadata?(itag : JSON::Any)
+ return VIDEO_FORMATS[itag.to_s]?
end
def process_continuation(db, query, plid, id)
@@ -1365,34 +1113,34 @@ def process_video_params(query, preferences)
controls ||= 1
controls = controls >= 1
- params = VideoPreferences.new(
- annotations: annotations,
- autoplay: autoplay,
- comments: comments,
- continue: continue,
- continue_autoplay: continue_autoplay,
- controls: controls,
- listen: listen,
- local: local,
- player_style: player_style,
+ params = VideoPreferences.new({
+ annotations: annotations,
+ autoplay: autoplay,
+ comments: comments,
+ continue: continue,
+ continue_autoplay: continue_autoplay,
+ controls: controls,
+ listen: listen,
+ local: local,
+ player_style: player_style,
preferred_captions: preferred_captions,
- quality: quality,
- raw: raw,
- region: region,
- related_videos: related_videos,
- speed: speed,
- video_end: video_end,
- video_loop: video_loop,
- video_start: video_start,
- volume: volume,
- )
+ quality: quality,
+ raw: raw,
+ region: region,
+ related_videos: related_videos,
+ speed: speed,
+ video_end: video_end,
+ video_loop: video_loop,
+ video_start: video_start,
+ volume: volume,
+ })
return params
end
-def build_thumbnails(id, config, kemal_config)
+def build_thumbnails(id)
return {
- {name: "maxres", host: "#{make_host_url(config, kemal_config)}", url: "maxres", height: 720, width: 1280},
+ {name: "maxres", host: "#{HOST_URL}", url: "maxres", height: 720, width: 1280},
{name: "maxresdefault", host: "https://i.ytimg.com", url: "maxresdefault", height: 720, width: 1280},
{name: "sddefault", host: "https://i.ytimg.com", url: "sddefault", height: 480, width: 640},
{name: "high", host: "https://i.ytimg.com", url: "hqdefault", height: 360, width: 480},
@@ -1404,9 +1152,9 @@ def build_thumbnails(id, config, kemal_config)
}
end
-def generate_thumbnails(json, id, config, kemal_config)
+def generate_thumbnails(json, id)
json.array do
- build_thumbnails(id, config, kemal_config).each do |thumbnail|
+ build_thumbnails(id).each do |thumbnail|
json.object do
json.field "quality", thumbnail[:name]
json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg"
@@ -1417,7 +1165,7 @@ def generate_thumbnails(json, id, config, kemal_config)
end
end
-def generate_storyboards(json, id, storyboards, config, kemal_config)
+def generate_storyboards(json, id, storyboards)
json.array do
storyboards.each do |storyboard|
json.object do
diff --git a/src/invidious/views/add_playlist_items.ecr b/src/invidious/views/add_playlist_items.ecr
index f1899faa..09eacbc8 100644
--- a/src/invidious/views/add_playlist_items.ecr
+++ b/src/invidious/views/add_playlist_items.ecr
@@ -20,12 +20,14 @@
<div class="pure-u-1 pure-u-lg-1-5"></div>
</div>
-<script>
-var playlist_data = {
- csrf_token: '<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>',
-}
+<script id="playlist_data" type="application/json">
+<%=
+{
+ "csrf_token" => URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "")
+}.to_pretty_json
+%>
</script>
-<script src="/js/playlist_widget.js"></script>
+<script src="/js/playlist_widget.js?v=<%= ASSET_COMMIT %>"></script>
<div class="pure-g">
<% videos.each_slice(4) do |slice| %>
diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr
index b5eb46ea..caa0ad0e 100644
--- a/src/invidious/views/channel.ecr
+++ b/src/invidious/views/channel.ecr
@@ -28,7 +28,7 @@
</div>
<div class="h-box">
- <p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %></span></p>
+ <p><span style="white-space:pre-wrap"><%= channel.description_html %></span></p>
</div>
<div class="h-box">
@@ -92,7 +92,7 @@
<div class="pure-g h-box">
<div class="pure-u-1 pure-u-lg-1-5">
<% if page > 1 %>
- <a href="/channel/<%= channel.ucid %>?page=<%= page - 1 %><% if sort_by != "newest" %>&sort_by=<%= sort_by %><% end %>">
+ <a href="/channel/<%= channel.ucid %>?page=<%= page - 1 %><% if sort_by != "newest" %>&sort_by=<%= HTML.escape(sort_by) %><% end %>">
<%= translate(locale, "Previous page") %>
</a>
<% end %>
@@ -100,7 +100,7 @@
<div class="pure-u-1 pure-u-lg-3-5"></div>
<div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
<% if count == 60 %>
- <a href="/channel/<%= channel.ucid %>?page=<%= page + 1 %><% if sort_by != "newest" %>&sort_by=<%= sort_by %><% end %>">
+ <a href="/channel/<%= channel.ucid %>?page=<%= page + 1 %><% if sort_by != "newest" %>&sort_by=<%= HTML.escape(sort_by) %><% end %>">
<%= translate(locale, "Next page") %>
</a>
<% end %>
diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr
index 218cc2d4..69724390 100644
--- a/src/invidious/views/community.ecr
+++ b/src/invidious/views/community.ecr
@@ -71,14 +71,16 @@
</div>
<% end %>
-<script>
-var community_data = {
- ucid: '<%= channel.ucid %>',
- youtube_comments_text: '<%= HTML.escape(translate(locale, "View YouTube comments")) %>',
- comments_text: '<%= HTML.escape(translate(locale, "View `x` comments", "{commentCount}")) %>',
- hide_replies_text: '<%= HTML.escape(translate(locale, "Hide replies")) %>',
- show_replies_text: '<%= HTML.escape(translate(locale, "Show replies")) %>',
- preferences: <%= env.get("preferences").as(Preferences).to_json %>,
-}
+<script id="community_data" type="application/json">
+<%=
+{
+ "ucid" => channel.ucid,
+ "youtube_comments_text" => HTML.escape(translate(locale, "View YouTube comments")),
+ "comments_text" => HTML.escape(translate(locale, "View `x` comments", "{commentCount}")),
+ "hide_replies_text" => HTML.escape(translate(locale, "Hide replies")),
+ "show_replies_text" => HTML.escape(translate(locale, "Show replies")),
+ "preferences" => env.get("preferences").as(Preferences)
+}.to_pretty_json
+%>
</script>
<script src="/js/community.js?v=<%= ASSET_COMMIT %>"></script>
diff --git a/src/invidious/views/components/feed_menu.ecr b/src/invidious/views/components/feed_menu.ecr
index f72db2da..3dbeaf37 100644
--- a/src/invidious/views/components/feed_menu.ecr
+++ b/src/invidious/views/components/feed_menu.ecr
@@ -1,19 +1,11 @@
-<div class="h-box pure-g">
- <div class="pure-u-1 pure-u-md-1-4"></div>
- <div class="pure-u-1 pure-u-md-1-2">
- <div class="pure-g">
- <% feed_menu = env.get("preferences").as(Preferences).feed_menu.dup %>
- <% if !env.get?("user") %>
- <% feed_menu.reject! {|item| {"Subscriptions", "Playlists"}.includes? item} %>
- <% end %>
- <% feed_menu.each do |feed| %>
- <div class="pure-u-1-2 pure-u-md-1-<%= feed_menu.size %>">
- <a href="/feed/<%= feed.downcase %>" class="pure-menu-heading" style="text-align:center">
- <%= translate(locale, feed) %>
- </a>
- </div>
- <% end %>
- </div>
- </div>
- <div class="pure-u-1 pure-u-md-1-4"></div>
+<div class="feed-menu">
+ <% feed_menu = env.get("preferences").as(Preferences).feed_menu.dup %>
+ <% if !env.get?("user") %>
+ <% feed_menu.reject! {|item| {"Subscriptions", "Playlists"}.includes? item} %>
+ <% end %>
+ <% feed_menu.each do |feed| %>
+ <a href="/feed/<%= feed.downcase %>" class="feed-menu-item pure-menu-heading">
+ <%= translate(locale, feed) %>
+ </a>
+ <% end %>
</div>
diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr
index 61ec3ce4..e4a60697 100644
--- a/src/invidious/views/components/item.ecr
+++ b/src/invidious/views/components/item.ecr
@@ -44,7 +44,7 @@
<% end %>
</div>
<% end %>
- <p><%= item.title %></p>
+ <p><%= HTML.escape(item.title) %></p>
</a>
<p>
<b>
@@ -57,10 +57,10 @@
<div class="thumbnail">
<img class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/>
<% if plid = env.get?("remove_playlist_items") %>
- <form onsubmit="return false" action="/playlist_ajax?action_remove_video=1&set_video_id=<%= item.index %>&playlist_id=<%= plid %>&referer=<%= env.get("current_page") %>" method="post">
+ <form data-onsubmit="return_false" action="/playlist_ajax?action_remove_video=1&set_video_id=<%= item.index %>&playlist_id=<%= plid %>&referer=<%= env.get("current_page") %>" method="post">
<input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>">
<p class="watched">
- <a onclick="remove_playlist_item(this)" data-index="<%= item.index %>" data-plid="<%= plid %>" href="javascript:void(0)">
+ <a data-onclick="remove_playlist_item" data-index="<%= item.index %>" data-plid="<%= plid %>" href="javascript:void(0)">
<button type="submit" style="all:unset">
<i class="icon ion-md-trash"></i>
</button>
@@ -76,7 +76,7 @@
<% end %>
</div>
<% end %>
- <p><a href="/watch?v=<%= item.id %>"><%= item.title %></a></p>
+ <p><a href="/watch?v=<%= item.id %>"><%= HTML.escape(item.title) %></a></p>
</a>
<p>
<b>
@@ -85,7 +85,7 @@
</p>
<h5 class="pure-g">
- <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp && item.premiere_timestamp.not_nil! > Time.utc %>
+ <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp.try &.> Time.utc %>
<div class="pure-u-2-3"><%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %></div>
<% elsif Time.utc - item.published > 1.minute %>
<div class="pure-u-2-3"><%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %></div>
@@ -103,13 +103,12 @@
<div class="thumbnail">
<img class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/>
<% if env.get? "show_watched" %>
- <form onsubmit="return false" action="/watch_ajax?action_mark_watched=1&id=<%= item.id %>&referer=<%= env.get("current_page") %>" method="post">
+ <form data-onsubmit="return_false" action="/watch_ajax?action_mark_watched=1&id=<%= item.id %>&referer=<%= env.get("current_page") %>" method="post">
<input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>">
<p class="watched">
- <a onclick="mark_watched(this)" data-id="<%= item.id %>" href="javascript:void(0)">
+ <a data-onclick="mark_watched" data-id="<%= item.id %>" href="javascript:void(0)">
<button type="submit" style="all:unset">
- <i onmouseenter='this.setAttribute("class", "icon ion-ios-eye-off")'
- onmouseleave='this.setAttribute("class", "icon ion-ios-eye")'
+ <i data-mouse="switch_classes" data-switch-classes="ion-ios-eye-off,ion-ios-eye"
class="icon ion-ios-eye">
</i>
</button>
@@ -117,10 +116,10 @@
</p>
</form>
<% elsif plid = env.get? "add_playlist_items" %>
- <form onsubmit="return false" action="/playlist_ajax?action_add_video=1&video_id=<%= item.id %>&playlist_id=<%= plid %>&referer=<%= env.get("current_page") %>" method="post">
+ <form data-onsubmit="return_false" action="/playlist_ajax?action_add_video=1&video_id=<%= item.id %>&playlist_id=<%= plid %>&referer=<%= env.get("current_page") %>" method="post">
<input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>">
<p class="watched">
- <a onclick="add_playlist_item(this)" data-id="<%= item.id %>" data-plid="<%= plid %>" href="javascript:void(0)">
+ <a data-onclick="add_playlist_item" data-id="<%= item.id %>" data-plid="<%= plid %>" href="javascript:void(0)">
<button type="submit" style="all:unset">
<i class="icon ion-md-add"></i>
</button>
@@ -137,7 +136,7 @@
</div>
</a>
<% end %>
- <p><a href="/watch?v=<%= item.id %>"><%= item.title %></a></p>
+ <p><a href="/watch?v=<%= item.id %>"><%= HTML.escape(item.title) %></a></p>
<p style="display: flex;">
<b style="flex: 1;">
<a style="width:100%" href="/channel/<%= item.ucid %>"><%= item.author %></a>
@@ -148,7 +147,7 @@
</p>
<h5 class="pure-g">
- <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp && item.premiere_timestamp.not_nil! > Time.utc %>
+ <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp.try &.> Time.utc %>
<div class="pure-u-2-3"><%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %></div>
<% elsif Time.utc - item.published > 1.minute %>
<div class="pure-u-2-3"><%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %></div>
diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr
index ba6311cb..0e6664fa 100644
--- a/src/invidious/views/components/player.ecr
+++ b/src/invidious/views/components/player.ecr
@@ -1,28 +1,25 @@
-<video style="outline:none;width:100%;background-color:#000" playsinline poster="<%= thumbnail %>" title="<%= HTML.escape(video.title) %>"
- id="player" class="video-js player-style-<%= params.player_style %>"
- onmouseenter='this["data-title"]=this["title"];this["title"]=""'
- onmouseleave='this["title"]=this["data-title"];this["data-title"]=""'
- oncontextmenu='this["title"]=this["data-title"]'
+<video style="outline:none;width:100%;background-color:#000" playsinline poster="<%= thumbnail %>"
+ id="player" class="on-video_player video-js player-style-<%= params.player_style %>"
<% if params.autoplay %>autoplay<% end %>
<% if params.video_loop %>loop<% end %>
<% if params.controls %>controls<% end %>>
- <% if hlsvp && !CONFIG.disabled?("livestreams") %>
- <source src="<%= hlsvp %>?local=true" type="application/x-mpegURL" label="livestream">
+ <% if (hlsvp = video.hls_manifest_url) && !CONFIG.disabled?("livestreams") %>
+ <source src="<%= URI.parse(hlsvp).full_path %>?local=true" type="application/x-mpegURL" label="livestream">
<% else %>
<% if params.listen %>
<% audio_streams.each_with_index do |fmt, i| %>
- <source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["type"] %>' label="<%= fmt["bitrate"] %>k" selected="<%= i == 0 ? true : false %>">
+ <source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["mimeType"] %>' label="<%= fmt["bitrate"] %>k" selected="<%= i == 0 ? true : false %>">
<% end %>
<% else %>
<% if params.quality == "dash" %>
- <source src="/api/manifest/dash/id/<%= video.id %>?local=true" type='application/dash+xml' label="dash">
+ <source src="/api/manifest/dash/id/<%= video.id %>?local=true&unique_res=1" type='application/dash+xml' label="dash">
<% end %>
<% fmt_stream.each_with_index do |fmt, i| %>
<% if params.quality %>
- <source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["type"] %>' label="<%= fmt["label"] %>" selected="<%= params.quality == fmt["label"].split(" - ")[0] %>">
+ <source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["mimeType"] %>' label="<%= fmt["quality"] %>" selected="<%= params.quality == fmt["quality"] %>">
<% else %>
- <source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["type"] %>' label="<%= fmt["label"] %>" selected="<%= i == 0 ? true : false %>">
+ <source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["mimeType"] %>' label="<%= fmt["quality"] %>" selected="<%= i == 0 ? true : false %>">
<% end %>
<% end %>
<% end %>
@@ -39,12 +36,14 @@
<% end %>
</video>
-<script>
-var player_data = {
- aspect_ratio: '<%= aspect_ratio %>',
- title: "<%= video.title.dump_unquoted %>",
- description: "<%= HTML.escape(video.short_description) %>",
- thumbnail: "<%= thumbnail %>"
-}
+<script id="player_data" type="application/json">
+<%=
+{
+ "aspect_ratio" => aspect_ratio,
+ "title" => video.title,
+ "description" => HTML.escape(video.short_description),
+ "thumbnail" => thumbnail
+}.to_pretty_json
+%>
</script>
<script src="/js/player.js?v=<%= ASSET_COMMIT %>"></script>
diff --git a/src/invidious/views/components/player_sources.ecr b/src/invidious/views/components/player_sources.ecr
index d950e0da..d02f82d2 100644
--- a/src/invidious/views/components/player_sources.ecr
+++ b/src/invidious/views/components/player_sources.ecr
@@ -3,6 +3,8 @@
<link rel="stylesheet" href="/css/videojs.markers.min.css?v=<%= ASSET_COMMIT %>">
<link rel="stylesheet" href="/css/videojs-share.css?v=<%= ASSET_COMMIT %>">
<link rel="stylesheet" href="/css/videojs-vtt-thumbnails.css?v=<%= ASSET_COMMIT %>">
+<link rel="stylesheet" href="/css/videojs-vtt-thumbnails-fix.css?v=<%= ASSET_COMMIT %>">
+<script src="/js/global.js?v=<%= ASSET_COMMIT %>"></script>
<script src="/js/video.min.js?v=<%= ASSET_COMMIT %>"></script>
<script src="/js/videojs-contrib-quality-levels.min.js?v=<%= ASSET_COMMIT %>"></script>
<script src="/js/videojs-http-source-selector.min.js?v=<%= ASSET_COMMIT %>"></script>
diff --git a/src/invidious/views/components/subscribe_widget.ecr b/src/invidious/views/components/subscribe_widget.ecr
index 471e6c1c..ac2fbf1d 100644
--- a/src/invidious/views/components/subscribe_widget.ecr
+++ b/src/invidious/views/components/subscribe_widget.ecr
@@ -19,15 +19,17 @@
</p>
<% end %>
- <script>
- var subscribe_data = {
- ucid: '<%= ucid %>',
- author: '<%= HTML.escape(author) %>',
- sub_count_text: '<%= HTML.escape(sub_count_text) %>',
- csrf_token: '<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>',
- subscribe_text: '<%= HTML.escape(translate(locale, "Subscribe")) %>',
- unsubscribe_text: '<%= HTML.escape(translate(locale, "Unsubscribe")) %>'
- }
+ <script id="subscribe_data" type="application/json">
+ <%=
+ {
+ "ucid" => ucid,
+ "author" => HTML.escape(author),
+ "sub_count_text" => HTML.escape(sub_count_text),
+ "csrf_token" => URI.encode_www_form(env.get?("csrf_token").try &.as(String) || ""),
+ "subscribe_text" => HTML.escape(translate(locale, "Subscribe")),
+ "unsubscribe_text" => HTML.escape(translate(locale, "Unsubscribe"))
+ }.to_pretty_json
+ %>
</script>
<script src="/js/subscribe_widget.js?v=<%= ASSET_COMMIT %>"></script>
<% else %>
diff --git a/src/invidious/views/embed.ecr b/src/invidious/views/embed.ecr
index 6c06bf2e..48dbc55f 100644
--- a/src/invidious/views/embed.ecr
+++ b/src/invidious/views/embed.ecr
@@ -10,33 +10,24 @@
<script src="/js/videojs-overlay.min.js?v=<%= ASSET_COMMIT %>"></script>
<link rel="stylesheet" href="/css/default.css?v=<%= ASSET_COMMIT %>">
<link rel="stylesheet" href="/css/darktheme.css?v=<%= ASSET_COMMIT %>">
+ <link rel="stylesheet" href="/css/embed.css?v=<%= ASSET_COMMIT %>">
<title><%= HTML.escape(video.title) %> - Invidious</title>
- <style>
- #player {
- position: fixed;
- right: 0;
- bottom: 0;
- min-width: 100%;
- min-height: 100%;
- width: auto;
- height: auto;
- z-index: -100;
- }
- </style>
</head>
<body>
-<script>
-var video_data = {
- id: '<%= video.id %>',
- index: '<%= continuation %>',
- plid: '<%= plid %>',
- length_seconds: '<%= video.length_seconds.to_f %>',
- video_series: <%= video_series.to_json %>,
- params: <%= params.to_json %>,
- preferences: <%= preferences.to_json %>,
- premiere_timestamp: <%= video.premiere_timestamp.try &.to_unix || "null" %>
-}
+<script id="video_data" type="application/json">
+<%=
+{
+ "id" => video.id,
+ "index" => continuation,
+ "plid" => plid,
+ "length_seconds" => video.length_seconds.to_f,
+ "video_series" => video_series,
+ "params" => params,
+ "preferences" => preferences,
+ "premiere_timestamp" => video.premiere_timestamp.try &.to_unix
+}.to_pretty_json
+%>
</script>
<%= rendered "components/player" %>
diff --git a/src/invidious/views/history.ecr b/src/invidious/views/history.ecr
index 7d7ded2c..fe8c70b9 100644
--- a/src/invidious/views/history.ecr
+++ b/src/invidious/views/history.ecr
@@ -18,10 +18,12 @@
</div>
</div>
-<script>
-var watched_data = {
- csrf_token: '<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>',
-}
+<script id="watched_data" type="application/json">
+<%=
+{
+ "csrf_token" => URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "")
+}.to_pretty_json
+%>
</script>
<script src="/js/watched_widget.js"></script>
@@ -34,10 +36,10 @@ var watched_data = {
<% if !env.get("preferences").as(Preferences).thin_mode %>
<div class="thumbnail">
<img class="thumbnail" src="/vi/<%= item %>/mqdefault.jpg"/>
- <form onsubmit="return false" action="/watch_ajax?action_mark_unwatched=1&id=<%= item %>&referer=<%= env.get("current_page") %>" method="post">
+ <form data-onsubmit="return_false" action="/watch_ajax?action_mark_unwatched=1&id=<%= item %>&referer=<%= env.get("current_page") %>" method="post">
<input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>">
<p class="watched">
- <a onclick="mark_unwatched(this)" data-id="<%= item %>" href="javascript:void(0)">
+ <a data-onclick="mark_unwatched" data-id="<%= item %>" href="javascript:void(0)">
<button type="submit" style="all:unset">
<i class="icon ion-md-trash"></i>
</button>
diff --git a/src/invidious/views/login.ecr b/src/invidious/views/login.ecr
index 59fa90e5..b6e8117b 100644
--- a/src/invidious/views/login.ecr
+++ b/src/invidious/views/login.ecr
@@ -22,7 +22,43 @@
<hr>
<% case account_type when %>
- <% when "invidious" %>
+ <% when "google" %>
+ <form class="pure-form pure-form-stacked" action="/login?referer=<%= URI.encode_www_form(referer) %>&type=google" method="post">
+ <fieldset>
+ <% if email %>
+ <input name="email" type="hidden" value="<%= email %>">
+ <% else %>
+ <label for="email"><%= translate(locale, "E-mail") %> :</label>
+ <input required class="pure-input-1" name="email" type="email" placeholder="<%= translate(locale, "E-mail") %>">
+ <% end %>
+
+ <% if password %>
+ <input name="password" type="hidden" value="<%= HTML.escape(password) %>">
+ <% else %>
+ <label for="password"><%= translate(locale, "Password") %> :</label>
+ <input required class="pure-input-1" name="password" type="password" placeholder="<%= translate(locale, "Password") %>">
+ <% end %>
+
+ <% if prompt %>
+ <label for="tfa"><%= translate(locale, prompt) %> :</label>
+ <input required class="pure-input-1" name="tfa" type="text" placeholder="<%= translate(locale, prompt) %>">
+ <% end %>
+
+ <% if tfa %>
+ <input type="hidden" name="tfa" value="<%= tfa %>">
+ <% end %>
+
+ <% if captcha %>
+ <img style="width:50%" src="/Captcha?v=2&ctoken=<%= captcha[:tokens][0] %>"/>
+ <input type="hidden" name="token" value="<%= captcha[:tokens][0] %>">
+ <label for="answer"><%= translate(locale, "Answer") %> :</label>
+ <input type="text" name="answer" type="text" placeholder="<%= translate(locale, "Answer") %>">
+ <% end %>
+
+ <button type="submit" class="pure-button pure-button-primary"><%= translate(locale, "Sign In") %></button>
+ </fieldset>
+ </form>
+ <% else # "invidious" %>
<form class="pure-form pure-form-stacked" action="/login?referer=<%= URI.encode_www_form(referer) %>&type=invidious" method="post">
<fieldset>
<% if email %>
@@ -50,7 +86,7 @@
<input type="hidden" name="captcha_type" value="image">
<label for="answer"><%= translate(locale, "Time (h:mm:ss):") %></label>
<input type="text" name="answer" type="text" placeholder="h:mm:ss">
- <% when "text" %>
+ <% else # "text" %>
<% captcha = captcha.not_nil! %>
<% captcha[:tokens].each_with_index do |token, i| %>
<input type="hidden" name="token[<%= i %>]" value="<%= URI.encode_www_form(token) %>">
@@ -71,7 +107,7 @@
<%= translate(locale, "Text CAPTCHA") %>
</button>
</label>
- <% when "text" %>
+ <% else # "text" %>
<label>
<button type="submit" name="change_type" class="pure-button pure-button-primary" value="image">
<%= translate(locale, "Image CAPTCHA") %>
@@ -85,42 +121,6 @@
<% end %>
</fieldset>
</form>
- <% when "google" %>
- <form class="pure-form pure-form-stacked" action="/login?referer=<%= URI.encode_www_form(referer) %>&type=google" method="post">
- <fieldset>
- <% if email %>
- <input name="email" type="hidden" value="<%= email %>">
- <% else %>
- <label for="email"><%= translate(locale, "E-mail") %> :</label>
- <input required class="pure-input-1" name="email" type="email" placeholder="<%= translate(locale, "E-mail") %>">
- <% end %>
-
- <% if password %>
- <input name="password" type="hidden" value="<%= HTML.escape(password) %>">
- <% else %>
- <label for="password"><%= translate(locale, "Password") %> :</label>
- <input required class="pure-input-1" name="password" type="password" placeholder="<%= translate(locale, "Password") %>">
- <% end %>
-
- <% if prompt %>
- <label for="tfa"><%= translate(locale, prompt) %> :</label>
- <input required class="pure-input-1" name="tfa" type="text" placeholder="<%= translate(locale, prompt) %>">
- <% end %>
-
- <% if tfa %>
- <input type="hidden" name="tfa" value="<%= tfa %>">
- <% end %>
-
- <% if captcha %>
- <img style="width:50%" src="/Captcha?v=2&ctoken=<%= captcha[:tokens][0] %>"/>
- <input type="hidden" name="token" value="<%= captcha[:tokens][0] %>">
- <label for="answer"><%= translate(locale, "Answer") %> :</label>
- <input type="text" name="answer" type="text" placeholder="<%= translate(locale, "Answer") %>">
- <% end %>
-
- <button type="submit" class="pure-button pure-button-primary"><%= translate(locale, "Sign In") %></button>
- </fieldset>
- </form>
<% end %>
</div>
</div>
diff --git a/src/invidious/views/playlist.ecr b/src/invidious/views/playlist.ecr
index cb643aaa..bb721c3a 100644
--- a/src/invidious/views/playlist.ecr
+++ b/src/invidious/views/playlist.ecr
@@ -45,6 +45,12 @@
<% if playlist.is_a?(InvidiousPlaylist) && playlist.author == user.try &.email %>
<div class="pure-u-1-3"><a href="/edit_playlist?list=<%= plid %>"><i class="icon ion-md-create"></i></a></div>
<div class="pure-u-1-3"><a href="/delete_playlist?list=<%= plid %>"><i class="icon ion-md-trash"></i></a></div>
+ <% else %>
+ <% if PG_DB.query_one?("SELECT id FROM playlists WHERE id = $1", playlist.id, as: String).nil? %>
+ <div class="pure-u-1-3"><a href="/subscribe_playlist?list=<%= plid %>"><i class="icon ion-md-add"></i></a></div>
+ <% else %>
+ <div class="pure-u-1-3"><a href="/delete_playlist?list=<%= plid %>"><i class="icon ion-md-trash"></i></a></div>
+ <% end %>
<% end %>
<div class="pure-u-1-3"><a href="/feed/playlist/<%= plid %>"><i class="icon ion-logo-rss"></i></a></div>
</div>
@@ -69,12 +75,14 @@
</div>
<% if playlist.is_a?(InvidiousPlaylist) && playlist.author == user.try &.email %>
-<script>
-var playlist_data = {
- csrf_token: '<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>',
-}
+<script id="playlist_data" type="application/json">
+<%=
+{
+ "csrf_token" => URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "")
+}.to_pretty_json
+%>
</script>
-<script src="/js/playlist_widget.js"></script>
+<script src="/js/playlist_widget.js?v=<%= ASSET_COMMIT %>"></script>
<% end %>
<div class="pure-g">
diff --git a/src/invidious/views/playlists.ecr b/src/invidious/views/playlists.ecr
index a32192b5..0c48be96 100644
--- a/src/invidious/views/playlists.ecr
+++ b/src/invidious/views/playlists.ecr
@@ -90,7 +90,7 @@
<div class="pure-u-1 pure-u-md-4-5"></div>
<div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
<% if continuation %>
- <a href="/channel/<%= channel.ucid %>/playlists?continuation=<%= continuation %><% if sort_by != "last" %>&sort_by=<%= sort_by %><% end %>">
+ <a href="/channel/<%= channel.ucid %>/playlists?continuation=<%= continuation %><% if sort_by != "last" %>&sort_by=<%= HTML.escape(sort_by) %><% end %>">
<%= translate(locale, "Next page") %>
</a>
<% end %>
diff --git a/src/invidious/views/preferences.ecr b/src/invidious/views/preferences.ecr
index 17e5804e..fb5bd44b 100644
--- a/src/invidious/views/preferences.ecr
+++ b/src/invidious/views/preferences.ecr
@@ -2,12 +2,6 @@
<title><%= translate(locale, "Preferences") %> - Invidious</title>
<% end %>
-<script>
-function update_value(element) {
- document.getElementById('volume-value').innerText = element.value;
-}
-</script>
-
<div class="h-box">
<form class="pure-form pure-form-aligned" action="/preferences?referer=<%= URI.encode_www_form(referer) %>" method="post">
<fieldset>
@@ -65,7 +59,7 @@ function update_value(element) {
<div class="pure-control-group">
<label for="volume"><%= translate(locale, "Player volume: ") %></label>
- <input name="volume" id="volume" oninput="update_value(this);" type="range" min="0" max="100" step="5" value="<%= preferences.volume %>">
+ <input name="volume" id="volume" data-onrange="update_volume_value" type="range" min="0" max="100" step="5" value="<%= preferences.volume %>">
<span class="pure-form-message-inline" id="volume-value"><%= preferences.volume %></span>
</div>
@@ -205,7 +199,7 @@ function update_value(element) {
<% # Web notifications are only supported over HTTPS %>
<% if Kemal.config.ssl || config.https_only %>
<div class="pure-control-group">
- <a href="#" onclick="Notification.requestPermission()"><%= translate(locale, "Enable web notifications") %></a>
+ <a href="#" data-onclick="notification_requestPermission"><%= translate(locale, "Enable web notifications") %></a>
</div>
<% end %>
<% end %>
@@ -234,11 +228,6 @@ function update_value(element) {
</div>
<div class="pure-control-group">
- <label for="top_enabled"><%= translate(locale, "Top enabled: ") %></label>
- <input name="top_enabled" id="top_enabled" type="checkbox" <% if config.top_enabled %>checked<% end %>>
- </div>
-
- <div class="pure-control-group">
<label for="captcha_enabled"><%= translate(locale, "CAPTCHA enabled: ") %></label>
<input name="captcha_enabled" id="captcha_enabled" type="checkbox" <% if config.captcha_enabled %>checked<% end %>>
</div>
diff --git a/src/invidious/views/search.ecr b/src/invidious/views/search.ecr
index d084bd31..bc13b7ea 100644
--- a/src/invidious/views/search.ecr
+++ b/src/invidious/views/search.ecr
@@ -2,6 +2,24 @@
<title><%= search_query.not_nil!.size > 30 ? HTML.escape(query.not_nil![0,30].rstrip(".") + "...") : HTML.escape(query.not_nil!) %> - Invidious</title>
<% end %>
+<div class="pure-g h-box v-box">
+ <div class="pure-u-1 pure-u-lg-1-5">
+ <% if page > 1 %>
+ <a href="/search?q=<%= HTML.escape(query.not_nil!) %>&page=<%= page - 1 %>">
+ <%= translate(locale, "Previous page") %>
+ </a>
+ <% end %>
+ </div>
+ <div class="pure-u-1 pure-u-lg-3-5"></div>
+ <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
+ <% if count >= 20 %>
+ <a href="/search?q=<%= HTML.escape(query.not_nil!) %>&page=<%= page + 1 %>">
+ <%= translate(locale, "Next page") %>
+ </a>
+ <% end %>
+ </div>
+</div>
+
<div class="pure-g">
<% videos.each_slice(4) do |slice| %>
<% slice.each do |item| %>
diff --git a/src/invidious/views/subscription_manager.ecr b/src/invidious/views/subscription_manager.ecr
index 43d14b37..6cddcd6c 100644
--- a/src/invidious/views/subscription_manager.ecr
+++ b/src/invidious/views/subscription_manager.ecr
@@ -37,9 +37,9 @@
<div class="pure-u-2-5"></div>
<div class="pure-u-1-5" style="text-align:right">
<h3 style="padding-right:0.5em">
- <form onsubmit="return false" action="/subscription_ajax?action_remove_subscriptions=1&c=<%= channel.id %>&referer=<%= env.get("current_page") %>" method="post">
+ <form data-onsubmit="return_false" action="/subscription_ajax?action_remove_subscriptions=1&c=<%= channel.id %>&referer=<%= env.get("current_page") %>" method="post">
<input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>">
- <a onclick="remove_subscription(this)" data-ucid="<%= channel.id %>" href="#">
+ <a data-onclick="remove_subscription" data-ucid="<%= channel.id %>" href="#">
<input style="all:unset" type="submit" value="<%= translate(locale, "unsubscribe") %>">
</a>
</form>
@@ -52,32 +52,3 @@
<% end %>
</div>
<% end %>
-
-<script>
-function remove_subscription(target) {
- var row = target.parentNode.parentNode.parentNode.parentNode.parentNode;
- row.style.display = 'none';
- var count = document.getElementById('count');
- count.innerText = count.innerText - 1;
-
- var url = '/subscription_ajax?action_remove_subscriptions=1&redirect=false' +
- '&referer=<%= env.get("current_page") %>' +
- '&c=' + target.getAttribute('data-ucid');
- var xhr = new XMLHttpRequest();
- xhr.responseType = 'json';
- xhr.timeout = 10000;
- xhr.open('POST', url, true);
- xhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');
-
- xhr.onreadystatechange = function() {
- if (xhr.readyState == 4) {
- if (xhr.status != 200) {
- count.innerText = parseInt(count.innerText) + 1;
- row.style.display = '';
- }
- }
- }
-
- xhr.send('csrf_token=<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>');
-}
-</script>
diff --git a/src/invidious/views/subscriptions.ecr b/src/invidious/views/subscriptions.ecr
index ee31d241..af1d4fbc 100644
--- a/src/invidious/views/subscriptions.ecr
+++ b/src/invidious/views/subscriptions.ecr
@@ -45,10 +45,12 @@
<hr>
</div>
-<script>
-var watched_data = {
- csrf_token: '<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>',
-}
+<script id="watched_data" type="application/json">
+<%=
+{
+ "csrf_token" => URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "")
+}.to_pretty_json
+%>
</script>
<script src="/js/watched_widget.js"></script>
diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr
index b7cf2dcb..61cf5c3a 100644
--- a/src/invidious/views/template.ecr
+++ b/src/invidious/views/template.ecr
@@ -111,7 +111,7 @@
<div class="footer">
<div class="pure-g">
<div class="pure-u-1 pure-u-md-1-3">
- <a href="https://github.com/omarroth/invidious">
+ <a href="https://github.com/iv-org/invidious">
<%= translate(locale, "Released under the AGPLv3 by Omar Roth.") %>
</a>
</div>
@@ -140,23 +140,24 @@
</div>
<div class="pure-u-1 pure-u-md-1-3">
<i class="icon ion-logo-github"></i>
- <%= translate(locale, "Current version: ") %> <%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %>
- <i class="icon ion-logo-github"></i>
- <%= CURRENT_BRANCH %>
+ <%= translate(locale, "Current version: ") %> <%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %> @ <%= CURRENT_BRANCH %>
</div>
</div>
</div>
</div>
<div class="pure-u-1 pure-u-md-2-24"></div>
</div>
+ <script src="/js/handlers.js?v=<%= ASSET_COMMIT %>"></script>
<script src="/js/themes.js?v=<%= ASSET_COMMIT %>"></script>
<% if env.get? "user" %>
<script src="/js/sse.js?v=<%= ASSET_COMMIT %>"></script>
- <script>
- var notification_data = {
- upload_text: '<%= HTML.escape(translate(locale, "`x` uploaded a video")) %>',
- live_upload_text: '<%= HTML.escape(translate(locale, "`x` is live")) %>',
- }
+ <script id="notification_data" type="application/json">
+ <%=
+ {
+ "upload_text" => HTML.escape(translate(locale, "`x` uploaded a video")),
+ "live_upload_text" => HTML.escape(translate(locale, "`x` is live"))
+ }.to_pretty_json
+ %>
</script>
<script src="/js/notifications.js?v=<%= ASSET_COMMIT %>"></script>
<% end %>
diff --git a/src/invidious/views/token_manager.ecr b/src/invidious/views/token_manager.ecr
index b626d99c..e48aec2f 100644
--- a/src/invidious/views/token_manager.ecr
+++ b/src/invidious/views/token_manager.ecr
@@ -29,9 +29,9 @@
</div>
<div class="pure-u-1-5" style="text-align:right">
<h3 style="padding-right:0.5em">
- <form onsubmit="return false" action="/token_ajax?action_revoke_token=1&session=<%= token[:session] %>&referer=<%= env.get("current_page") %>" method="post">
+ <form data-onsubmit="return_false" action="/token_ajax?action_revoke_token=1&session=<%= token[:session] %>&referer=<%= env.get("current_page") %>" method="post">
<input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>">
- <a onclick="revoke_token(this)" data-session="<%= token[:session] %>" href="#">
+ <a data-onclick="revoke_token" data-session="<%= token[:session] %>" href="#">
<input style="all:unset" type="submit" value="<%= translate(locale, "revoke") %>">
</a>
</form>
@@ -44,32 +44,3 @@
<% end %>
</div>
<% end %>
-
-<script>
-function revoke_token(target) {
- var row = target.parentNode.parentNode.parentNode.parentNode.parentNode;
- row.style.display = 'none';
- var count = document.getElementById('count');
- count.innerText = count.innerText - 1;
-
- var url = '/token_ajax?action_revoke_token=1&redirect=false' +
- '&referer=<%= env.get("current_page") %>' +
- '&session=' + target.getAttribute('data-session');
- var xhr = new XMLHttpRequest();
- xhr.responseType = 'json';
- xhr.timeout = 10000;
- xhr.open('POST', url, true);
- xhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');
-
- xhr.onreadystatechange = function() {
- if (xhr.readyState == 4) {
- if (xhr.status != 200) {
- count.innerText = parseInt(count.innerText) + 1;
- row.style.display = '';
- }
- }
- }
-
- xhr.send('csrf_token=<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>');
-}
-</script>
diff --git a/src/invidious/views/top.ecr b/src/invidious/views/top.ecr
deleted file mode 100644
index f5db3aaa..00000000
--- a/src/invidious/views/top.ecr
+++ /dev/null
@@ -1,20 +0,0 @@
-<% content_for "header" do %>
-<meta name="description" content="<%= translate(locale, "An alternative front-end to YouTube") %>">
-<title>
- <% if env.get("preferences").as(Preferences).default_home != "Top" %>
- <%= translate(locale, "Top") %> - Invidious
- <% else %>
- Invidious
- <% end %>
-</title>
-<% end %>
-
-<%= rendered "components/feed_menu" %>
-
-<div class="pure-g">
- <% top_videos.each_slice(4) do |slice| %>
- <% slice.each do |item| %>
- <%= rendered "components/item" %>
- <% end %>
- <% end %>
-</div>
diff --git a/src/invidious/views/view_all_playlists.ecr b/src/invidious/views/view_all_playlists.ecr
index 0fa7a325..5ec6aa31 100644
--- a/src/invidious/views/view_all_playlists.ecr
+++ b/src/invidious/views/view_all_playlists.ecr
@@ -6,7 +6,7 @@
<div class="pure-g h-box">
<div class="pure-u-2-3">
- <h3><%= translate(locale, "`x` playlists", %(<span id="count">#{items.size}</span>)) %></h3>
+ <h3><%= translate(locale, "`x` created playlists", %(<span id="count">#{items_created.size}</span>)) %></h3>
</div>
<div class="pure-u-1-3" style="text-align:right">
<h3>
@@ -16,7 +16,21 @@
</div>
<div class="pure-g">
- <% items.each_slice(4) do |slice| %>
+ <% items_created.each_slice(4) do |slice| %>
+ <% slice.each do |item| %>
+ <%= rendered "components/item" %>
+ <% end %>
+ <% end %>
+</div>
+
+<div class="pure-g h-box">
+ <div class="pure-u-1">
+ <h3><%= translate(locale, "`x` saved playlists", %(<span id="count">#{items_saved.size}</span>)) %></h3>
+ </div>
+</div>
+
+<div class="pure-g">
+ <% items_saved.each_slice(4) do |slice| %>
<% slice.each do |item| %>
<%= rendered "components/item" %>
<% end %>
diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr
index df61abc5..9a1e6c32 100644
--- a/src/invidious/views/watch.ecr
+++ b/src/invidious/views/watch.ecr
@@ -3,47 +3,49 @@
<meta name="description" content="<%= HTML.escape(video.short_description) %>">
<meta name="keywords" content="<%= video.keywords.join(",") %>">
<meta property="og:site_name" content="Invidious">
-<meta property="og:url" content="<%= host_url %>/watch?v=<%= video.id %>">
+<meta property="og:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>">
<meta property="og:title" content="<%= HTML.escape(video.title) %>">
<meta property="og:image" content="/vi/<%= video.id %>/maxres.jpg">
-<meta property="og:description" content="<%= HTML.escape(video.short_description) %>">
+<meta property="og:description" content="<%= video.short_description %>">
<meta property="og:type" content="video.other">
-<meta property="og:video:url" content="<%= host_url %>/embed/<%= video.id %>">
-<meta property="og:video:secure_url" content="<%= host_url %>/embed/<%= video.id %>">
+<meta property="og:video:url" content="<%= HOST_URL %>/embed/<%= video.id %>">
+<meta property="og:video:secure_url" content="<%= HOST_URL %>/embed/<%= video.id %>">
<meta property="og:video:type" content="text/html">
<meta property="og:video:width" content="1280">
<meta property="og:video:height" content="720">
<meta name="twitter:card" content="player">
<meta name="twitter:site" content="@omarroth1">
-<meta name="twitter:url" content="<%= host_url %>/watch?v=<%= video.id %>">
+<meta name="twitter:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>">
<meta name="twitter:title" content="<%= HTML.escape(video.title) %>">
-<meta name="twitter:description" content="<%= HTML.escape(video.short_description) %>">
-<meta name="twitter:image" content="<%= host_url %>/vi/<%= video.id %>/maxres.jpg">
-<meta name="twitter:player" content="<%= host_url %>/embed/<%= video.id %>">
+<meta name="twitter:description" content="<%= video.short_description %>">
+<meta name="twitter:image" content="<%= HOST_URL %>/vi/<%= video.id %>/maxres.jpg">
+<meta name="twitter:player" content="<%= HOST_URL %>/embed/<%= video.id %>">
<meta name="twitter:player:width" content="1280">
<meta name="twitter:player:height" content="720">
<%= rendered "components/player_sources" %>
<title><%= HTML.escape(video.title) %> - Invidious</title>
<% end %>
-<script>
-var video_data = {
- id: '<%= video.id %>',
- index: '<%= continuation %>',
- plid: '<%= plid %>',
- length_seconds: <%= video.length_seconds.to_f %>,
- play_next: <%= !rvs.empty? && !plid && params.continue %>,
- next_video: '<%= rvs.select { |rv| rv["id"]? }[0]?.try &.["id"] %>',
- youtube_comments_text: '<%= HTML.escape(translate(locale, "View YouTube comments")) %>',
- reddit_comments_text: '<%= HTML.escape(translate(locale, "View Reddit comments")) %>',
- reddit_permalink_text: '<%= HTML.escape(translate(locale, "View more comments on Reddit")) %>',
- comments_text: '<%= HTML.escape(translate(locale, "View `x` comments", "{commentCount}")) %>',
- hide_replies_text: '<%= HTML.escape(translate(locale, "Hide replies")) %>',
- show_replies_text: '<%= HTML.escape(translate(locale, "Show replies")) %>',
- params: <%= params.to_json %>,
- preferences: <%= preferences.to_json %>,
- premiere_timestamp: <%= video.premiere_timestamp.try &.to_unix || "null" %>
-}
+<script id="video_data" type="application/json">
+<%=
+{
+ "id" => video.id,
+ "index" => continuation,
+ "plid" => plid,
+ "length_seconds" => video.length_seconds.to_f,
+ "play_next" => !video.related_videos.empty? && !plid && params.continue,
+ "next_video" => video.related_videos.select { |rv| rv["id"]? }[0]?.try &.["id"],
+ "youtube_comments_text" => HTML.escape(translate(locale, "View YouTube comments")),
+ "reddit_comments_text" => HTML.escape(translate(locale, "View Reddit comments")),
+ "reddit_permalink_text" => HTML.escape(translate(locale, "View more comments on Reddit")),
+ "comments_text" => HTML.escape(translate(locale, "View `x` comments", "{commentCount}")),
+ "hide_replies_text" => HTML.escape(translate(locale, "Hide replies")),
+ "show_replies_text" => HTML.escape(translate(locale, "Show replies")),
+ "params" => params,
+ "preferences" => preferences,
+ "premiere_timestamp" => video.premiere_timestamp.try &.to_unix
+}.to_pretty_json
+%>
</script>
<div id="player-container" class="h-box">
@@ -70,13 +72,13 @@ var video_data = {
</h3>
<% end %>
- <% if !reason.empty? %>
+ <% if video.reason %>
<h3>
- <%= reason %>
+ <%= video.reason %>
</h3>
- <% elsif video.premiere_timestamp %>
+ <% elsif video.premiere_timestamp.try &.> Time.utc %>
<h3>
- <%= translate(locale, "Premieres in `x`", recode_date((video.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %>
+ <%= video.premiere_timestamp.try { |t| translate(locale, "Premieres in `x`", recode_date((t - Time.utc).ago, locale)) } %>
</h3>
<% end %>
</div>
@@ -84,10 +86,10 @@ var video_data = {
<div class="pure-g">
<div class="pure-u-1 pure-u-lg-1-5">
<div class="h-box">
- <span>
+ <span id="watch-on-youtube">
<a href="https://www.youtube.com/watch?v=<%= video.id %>"><%= translate(locale, "Watch on YouTube") %></a>
</span>
- <p>
+ <p id="annotations">
<% if params.annotations %>
<a href="/watch?<%= env.params.query %>&iv_load_policy=3">
<%= translate(locale, "Hide annotations") %>
@@ -99,26 +101,54 @@ var video_data = {
<% end %>
</p>
+ <% if user %>
+ <% playlists = PG_DB.query_all("SELECT id,title FROM playlists WHERE author = $1 AND id LIKE 'IV%'", user.email, as: {String, String}) %>
+ <% if !playlists.empty? %>
+ <form data-onsubmit="return_false" class="pure-form pure-form-stacked" action="/playlist_ajax" method="post">
+ <div class="pure-control-group">
+ <label for="playlist_id"><%= translate(locale, "Add to playlist: ") %></label>
+ <select style="width:100%" name="playlist_id" id="playlist_id">
+ <% playlists.each do |plid, title| %>
+ <option data-plid="<%= plid %>" value="<%= plid %>"><%= title %></option>
+ <% end %>
+ </select>
+ </div>
+
+ <button data-onclick="add_playlist_video" data-id="<%= video.id %>" type="submit" class="pure-button pure-button-primary">
+ <b><%= translate(locale, "Add to playlist") %></b>
+ </button>
+ </form>
+ <script id="playlist_data" type="application/json">
+ <%=
+ {
+ "csrf_token" => URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "")
+ }.to_pretty_json
+ %>
+ </script>
+ <script src="/js/playlist_widget.js?v=<%= Time.utc.to_unix_ms %>"></script>
+ <% end %>
+ <% end %>
+
<% if CONFIG.dmca_content.includes?(video.id) || CONFIG.disabled?("downloads") %>
- <p><%= translate(locale, "Download is disabled.") %></p>
+ <p id="download"><%= translate(locale, "Download is disabled.") %></p>
<% else %>
<form class="pure-form pure-form-stacked" action="/latest_version" method="get" rel="noopener" target="_blank">
<div class="pure-control-group">
<label for="download_widget"><%= translate(locale, "Download as: ") %></label>
<select style="width:100%" name="download_widget" id="download_widget">
<% fmt_stream.each do |option| %>
- <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["type"].split(";")[0].split("/")[1] %>"}'>
- <%= itag_to_metadata?(option["itag"]).try &.["height"]? || "~240" %>p - <%= option["type"].split(";")[0] %>
+ <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["mimeType"].as_s.split(";")[0].split("/")[1] %>"}'>
+ <%= itag_to_metadata?(option["itag"]).try &.["height"]? || "~240" %>p - <%= option["mimeType"].as_s.split(";")[0] %>
</option>
<% end %>
<% video_streams.each do |option| %>
- <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["type"].split(";")[0].split("/")[1] %>"}'>
- <%= option["quality_label"] %> - <%= option["type"].split(";")[0] %> @ <%= option["fps"] %>fps - video only
+ <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["mimeType"].as_s.split(";")[0].split("/")[1] %>"}'>
+ <%= option["qualityLabel"] %> - <%= option["mimeType"].as_s.split(";")[0] %> @ <%= option["fps"] %>fps - video only
</option>
<% end %>
<% audio_streams.each do |option| %>
- <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["type"].split(";")[0].split("/")[1] %>"}'>
- <%= option["type"].split(";")[0] %> @ <%= option["bitrate"] %>k - audio only
+ <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["mimeType"].as_s.split(";")[0].split("/")[1] %>"}'>
+ <%= option["mimeType"].as_s.split(";")[0] %> @ <%= option["bitrate"]?.try &.as_i./ 1000 %>k - audio only
</option>
<% end %>
<% captions.each do |caption| %>
@@ -135,23 +165,23 @@ var video_data = {
</form>
<% end %>
- <p><i class="icon ion-ios-eye"></i> <%= number_with_separator(video.views) %></p>
- <p><i class="icon ion-ios-thumbs-up"></i> <%= number_with_separator(video.likes) %></p>
- <p><i class="icon ion-ios-thumbs-down"></i> <%= number_with_separator(video.dislikes) %></p>
+ <p id="views"><i class="icon ion-ios-eye"></i> <%= number_with_separator(video.views) %></p>
+ <p id="likes"><i class="icon ion-ios-thumbs-up"></i> <%= number_with_separator(video.likes) %></p>
+ <p id="dislikes"><i class="icon ion-ios-thumbs-down"></i> <%= number_with_separator(video.dislikes) %></p>
<p id="genre"><%= translate(locale, "Genre: ") %>
- <% if video.genre_url.empty? %>
+ <% if !video.genre_url %>
<%= video.genre %>
<% else %>
<a href="<%= video.genre_url %>"><%= video.genre %></a>
<% end %>
</p>
- <% if !video.license.empty? %>
+ <% if video.license %>
<p id="license"><%= translate(locale, "License: ") %><%= video.license %></p>
<% end %>
<p id="family_friendly"><%= translate(locale, "Family friendly? ") %><%= translate_bool(locale, video.is_family_friendly) %></p>
- <p id="wilson"><%= translate(locale, "Wilson score: ") %><%= video.wilson_score.round(4) %></p>
- <p id="rating"><%= translate(locale, "Rating: ") %><%= rating.round(4) %> / 5</p>
- <p id="engagement"><%= translate(locale, "Engagement: ") %><%= engagement.round(2) %>%</p>
+ <p id="wilson"><%= translate(locale, "Wilson score: ") %><%= video.wilson_score %></p>
+ <p id="rating"><%= translate(locale, "Rating: ") %><%= video.average_rating %> / 5</p>
+ <p id="engagement"><%= translate(locale, "Engagement: ") %><%= video.engagement %>%</p>
<% if video.allowed_regions.size != REGIONS.size %>
<p id="allowed_regions">
<% if video.allowed_regions.size < REGIONS.size // 2 %>
@@ -168,8 +198,10 @@ var video_data = {
<div class="h-box">
<a href="/channel/<%= video.ucid %>" style="display:block;width:fit-content;width:-moz-fit-content">
<div class="channel-profile">
- <img src="/ggpht<%= URI.parse(video.author_thumbnail).full_path %>">
- <span><%= video.author %></span>
+ <% if !video.author_thumbnail.empty? %>
+ <img src="/ggpht<%= URI.parse(video.author_thumbnail).full_path %>">
+ <% end %>
+ <span id="channel-name"><%= video.author %></span>
</div>
</a>
@@ -178,9 +210,9 @@ var video_data = {
<% sub_count_text = video.sub_count_text %>
<%= rendered "components/subscribe_widget" %>
- <p>
- <% if video.premiere_timestamp %>
- <b><%= translate(locale, "Premieres `x`", video.premiere_timestamp.not_nil!.to_s("%B %-d, %R UTC")) %></b>
+ <p id="published-date">
+ <% if video.premiere_timestamp.try &.> Time.utc %>
+ <b><%= video.premiere_timestamp.try { |t| translate(locale, "Premieres `x`", t.to_s("%B %-d, %R UTC")) } %></b>
<% else %>
<b><%= translate(locale, "Shared `x`", video.published.to_s("%B %-d, %Y")) %></b>
<% end %>
@@ -214,7 +246,7 @@ var video_data = {
<% if params.related_videos %>
<div class="h-box">
- <% if !rvs.empty? %>
+ <% if !video.related_videos.empty? %>
<div <% if plid %>style="display:none"<% end %>>
<div class="pure-control-group">
<label for="continue"><%= translate(locale, "Play next by default: ") %></label>
@@ -224,7 +256,7 @@ var video_data = {
</div>
<% end %>
- <% rvs.each do |rv| %>
+ <% video.related_videos.each do |rv| %>
<% if rv["id"]? %>
<a href="/watch?v=<%= rv["id"] %>">
<% if !env.get("preferences").as(Preferences).thin_mode %>
@@ -237,15 +269,17 @@ var video_data = {
<h5 class="pure-g">
<div class="pure-u-14-24">
<% if rv["ucid"]? %>
- <b style="width:100%"><a href="/channel/<%= rv["ucid"] %>"><%= rv["author"] %></a></b>
+ <b style="width:100%"><a href="/channel/<%= rv["ucid"] %>"><%= rv["author"]? %></a></b>
<% else %>
- <b style="width:100%"><%= rv["author"] %></b>
+ <b style="width:100%"><%= rv["author"]? %></b>
<% end %>
</div>
<div class="pure-u-10-24" style="text-align:right">
<% if views = rv["short_view_count_text"]?.try &.delete(", views watching") %>
- <b class="width:100%"><%= translate(locale, "`x` views", views) %></b>
+ <% if !views.empty? %>
+ <b class="width:100%"><%= translate(locale, "`x` views", views) %></b>
+ <% end %>
<% end %>
</div>
</h5>