summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/invidious/channels/about.cr4
-rw-r--r--src/invidious/channels/channels.cr2
-rw-r--r--src/invidious/channels/videos.cr6
-rw-r--r--src/invidious/config.cr6
-rw-r--r--src/invidious/helpers/errors.cr6
-rw-r--r--src/invidious/helpers/helpers.cr17
-rw-r--r--src/invidious/jobs/bypass_captcha_job.cr135
-rw-r--r--src/invidious/jobs/statistics_refresh_job.cr12
-rw-r--r--src/invidious/routes/api/v1/misc.cr16
-rw-r--r--src/invidious/routes/feeds.cr25
-rw-r--r--src/invidious/routes/video_playback.cr13
-rw-r--r--src/invidious/search/filters.cr6
-rw-r--r--src/invidious/videos.cr18
-rw-r--r--src/invidious/videos/parser.cr5
-rw-r--r--src/invidious/views/components/item.ecr36
-rw-r--r--src/invidious/views/watch.ecr2
-rw-r--r--src/invidious/yt_backend/connection_pool.cr24
-rw-r--r--src/invidious/yt_backend/extractors.cr4
18 files changed, 144 insertions, 193 deletions
diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr
index 0054f8f2..8b60a728 100644
--- a/src/invidious/channels/about.cr
+++ b/src/invidious/channels/about.cr
@@ -18,8 +18,8 @@ record AboutChannel,
def get_about_info(ucid, locale) : AboutChannel
begin
- # "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"}
- initdata = YoutubeAPI.browse(browse_id: ucid, params: "EgVhYm91dA==")
+ # Fetch channel information from channel home page
+ initdata = YoutubeAPI.browse(browse_id: ucid, params: "")
rescue
raise InfoException.new("Could not get channel info.")
end
diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr
index c3d6124f..be739673 100644
--- a/src/invidious/channels/channels.cr
+++ b/src/invidious/channels/channels.cr
@@ -93,7 +93,7 @@ struct ChannelVideo
def to_tuple
{% begin %}
{
- {{*@type.instance_vars.map(&.name)}}
+ {{@type.instance_vars.map(&.name).splat}}
}
{% end %}
end
diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr
index beb86e08..351790d7 100644
--- a/src/invidious/channels/videos.cr
+++ b/src/invidious/channels/videos.cr
@@ -62,12 +62,6 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
return continuation
end
-# Used in bypass_captcha_job.cr
-def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false)
- continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2)
- return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
-end
-
module Invidious::Channel::Tabs
extend self
diff --git a/src/invidious/config.cr b/src/invidious/config.cr
index 429d9246..09c2168b 100644
--- a/src/invidious/config.cr
+++ b/src/invidious/config.cr
@@ -48,7 +48,7 @@ struct ConfigPreferences
def to_tuple
{% begin %}
{
- {{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}}
+ {{(@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }).splat}}
}
{% end %}
end
@@ -133,10 +133,6 @@ class Config
# Saved cookies in "name1=value1; name2=value2..." format
@[YAML::Field(converter: Preferences::StringToCookies)]
property cookies : HTTP::Cookies = HTTP::Cookies.new
- # Key for Anti-Captcha
- property captcha_key : String? = nil
- # API URL for Anti-Captcha
- property captcha_api_url : String = "https://api.anti-captcha.com"
# Playlist length limit
property playlist_length_limit : Int32 = 500
diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr
index 6e5a975d..21b789bc 100644
--- a/src/invidious/helpers/errors.cr
+++ b/src/invidious/helpers/errors.cr
@@ -3,7 +3,7 @@
# -------------------
macro error_template(*args)
- error_template_helper(env, {{*args}})
+ error_template_helper(env, {{args.splat}})
end
def github_details(summary : String, content : String)
@@ -95,7 +95,7 @@ end
# -------------------
macro error_atom(*args)
- error_atom_helper(env, {{*args}})
+ error_atom_helper(env, {{args.splat}})
end
def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception)
@@ -121,7 +121,7 @@ end
# -------------------
macro error_json(*args)
- error_json_helper(env, {{*args}})
+ error_json_helper(env, {{args.splat}})
end
def error_json_helper(
diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr
index 23ff0da9..6dc9860e 100644
--- a/src/invidious/helpers/helpers.cr
+++ b/src/invidious/helpers/helpers.cr
@@ -208,3 +208,20 @@ def proxy_file(response, env)
IO.copy response.body_io, env.response
end
end
+
+# Fetch the playback requests tracker from the statistics endpoint.
+#
+# Creates a new tracker when unavailable.
+def get_playback_statistic
+ if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]) && tracker.as(Hash).empty?
+ tracker = {
+ "totalRequests" => 0_i64,
+ "successfulRequests" => 0_i64,
+ "ratio" => 0_f64,
+ }
+
+ Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"] = tracker
+ end
+
+ return tracker.as(Hash(String, Int64 | Float64))
+end
diff --git a/src/invidious/jobs/bypass_captcha_job.cr b/src/invidious/jobs/bypass_captcha_job.cr
deleted file mode 100644
index 71f8a938..00000000
--- a/src/invidious/jobs/bypass_captcha_job.cr
+++ /dev/null
@@ -1,135 +0,0 @@
-class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob
- def begin
- loop do
- begin
- random_video = PG_DB.query_one?("select id, ucid from (select id, ucid from channel_videos limit 1000) as s ORDER BY RANDOM() LIMIT 1", as: {id: String, ucid: String})
- if !random_video
- random_video = {id: "zj82_v2R6ts", ucid: "UCK87Lox575O_HCHBWaBSyGA"}
- end
- {"/watch?v=#{random_video["id"]}&gl=US&hl=en&has_verified=1&bpctr=9999999999", produce_channel_videos_url(ucid: random_video["ucid"])}.each do |path|
- response = YT_POOL.client &.get(path)
- if response.body.includes?("To continue with your YouTube experience, please fill out the form below.")
- html = XML.parse_html(response.body)
- form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil!
- site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"]
- s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"]
-
- inputs = {} of String => String
- form.xpath_nodes(%(.//input[@name])).map do |node|
- inputs[node["name"]] = node["value"]
- end
-
- headers = response.cookies.add_request_headers(HTTP::Headers.new)
-
- response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/createTask",
- headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
- "clientKey" => CONFIG.captcha_key,
- "task" => {
- "type" => "NoCaptchaTaskProxyless",
- "websiteURL" => "https://www.youtube.com#{path}",
- "websiteKey" => site_key,
- "recaptchaDataSValue" => s_value,
- },
- }.to_json).body)
-
- raise response["error"].as_s if response["error"]?
- task_id = response["taskId"].as_i
-
- loop do
- sleep 10.seconds
-
- response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/getTaskResult",
- headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
- "clientKey" => CONFIG.captcha_key,
- "taskId" => task_id,
- }.to_json).body)
-
- if response["status"]?.try &.== "ready"
- break
- elsif response["errorId"]?.try &.as_i != 0
- raise response["errorDescription"].as_s
- end
- end
-
- inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
- headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
- response = YT_POOL.client &.post("/das_captcha", headers, form: inputs)
-
- response.cookies
- .select { |cookie| cookie.name != "PREF" }
- .each { |cookie| CONFIG.cookies << cookie }
-
- # Persist cookies between runs
- File.write("config/config.yml", CONFIG.to_yaml)
- elsif response.headers["Location"]?.try &.includes?("/sorry/index")
- location = response.headers["Location"].try { |u| URI.parse(u) }
- headers = HTTP::Headers{":authority" => location.host.not_nil!}
- response = YT_POOL.client &.get(location.request_target, headers)
-
- html = XML.parse_html(response.body)
- form = html.xpath_node(%(//form[@action="index"])).not_nil!
- site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"]
- s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"]
-
- inputs = {} of String => String
- form.xpath_nodes(%(.//input[@name])).map do |node|
- inputs[node["name"]] = node["value"]
- end
-
- captcha_client = HTTPClient.new(URI.parse(CONFIG.captcha_api_url))
- captcha_client.family = CONFIG.force_resolve || Socket::Family::INET
- response = JSON.parse(captcha_client.post("/createTask",
- headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
- "clientKey" => CONFIG.captcha_key,
- "task" => {
- "type" => "NoCaptchaTaskProxyless",
- "websiteURL" => location.to_s,
- "websiteKey" => site_key,
- "recaptchaDataSValue" => s_value,
- },
- }.to_json).body)
-
- captcha_client.close
-
- raise response["error"].as_s if response["error"]?
- task_id = response["taskId"].as_i
-
- loop do
- sleep 10.seconds
-
- response = JSON.parse(captcha_client.post("/getTaskResult",
- headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
- "clientKey" => CONFIG.captcha_key,
- "taskId" => task_id,
- }.to_json).body)
-
- if response["status"]?.try &.== "ready"
- break
- elsif response["errorId"]?.try &.as_i != 0
- raise response["errorDescription"].as_s
- end
- end
-
- inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
- headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
- response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs)
- headers = HTTP::Headers{
- "Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0],
- }
- cookies = HTTP::Cookies.from_client_headers(headers)
-
- cookies.each { |cookie| CONFIG.cookies << cookie }
-
- # Persist cookies between runs
- File.write("config/config.yml", CONFIG.to_yaml)
- end
- end
- rescue ex
- LOGGER.error("BypassCaptchaJob: #{ex.message}")
- ensure
- sleep 1.minute
- Fiber.yield
- end
- end
- end
-end
diff --git a/src/invidious/jobs/statistics_refresh_job.cr b/src/invidious/jobs/statistics_refresh_job.cr
index a113bd77..72d1ce88 100644
--- a/src/invidious/jobs/statistics_refresh_job.cr
+++ b/src/invidious/jobs/statistics_refresh_job.cr
@@ -18,6 +18,13 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
"updatedAt" => Time.utc.to_unix,
"lastChannelRefreshedAt" => 0_i64,
},
+
+ #
+ # "totalRequests" => 0_i64,
+ # "successfulRequests" => 0_i64
+ # "ratio" => 0_i64
+ #
+ "playback" => {} of String => Int64 | Float64,
}
private getter db : DB::Database
@@ -30,7 +37,7 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
loop do
refresh_stats
- sleep 1.minute
+ sleep 10.minute
Fiber.yield
end
end
@@ -56,5 +63,8 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
"updatedAt" => Time.utc.to_unix,
"lastChannelRefreshedAt" => Invidious::Database::Statistics.channel_last_update.try &.to_unix || 0_i64,
}
+
+ # Reset playback requests tracker
+ STATISTICS["playback"] = {} of String => Int64 | Float64
end
end
diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr
index bad47709..12942906 100644
--- a/src/invidious/routes/api/v1/misc.cr
+++ b/src/invidious/routes/api/v1/misc.cr
@@ -6,6 +6,22 @@ module Invidious::Routes::API::V1::Misc
if !CONFIG.statistics_enabled
return {"software" => SOFTWARE}.to_json
else
+ # Calculate playback success rate
+ if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]?)
+ tracker = tracker.as(Hash(String, Int64 | Float64))
+
+ if !tracker.empty?
+ total_requests = tracker["totalRequests"]
+ success_count = tracker["successfulRequests"]
+
+ if total_requests.zero?
+ tracker["ratio"] = 1_i64
+ else
+ tracker["ratio"] = (success_count / (total_requests)).round(2)
+ end
+ end
+ end
+
return Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json
end
end
diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr
index 40bca008..e20a7139 100644
--- a/src/invidious/routes/feeds.cr
+++ b/src/invidious/routes/feeds.cr
@@ -407,14 +407,23 @@ module Invidious::Routes::Feeds
end
spawn do
- rss = XML.parse_html(body)
- rss.xpath_nodes("//feed/entry").each do |entry|
- id = entry.xpath_node("videoid").not_nil!.content
- author = entry.xpath_node("author/name").not_nil!.content
- published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content)
- updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content)
-
- video = get_video(id, force_refresh: true)
+ # TODO: unify this with the other almost identical looking parts in this and channels.cr somehow?
+ namespaces = {
+ "yt" => "http://www.youtube.com/xml/schemas/2015",
+ "default" => "http://www.w3.org/2005/Atom",
+ }
+ rss = XML.parse(body)
+ rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry|
+ id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
+ author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
+ published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content)
+ updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content)
+
+ begin
+ video = get_video(id, force_refresh: true)
+ rescue
+ next # skip this video since it raised an exception (e.g. it is a scheduled live event)
+ end
if CONFIG.enable_user_notifications
# Deliver notifications to `/api/v1/auth/notifications`
diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr
index 9641e01a..ec18f3b8 100644
--- a/src/invidious/routes/video_playback.cr
+++ b/src/invidious/routes/video_playback.cr
@@ -42,7 +42,7 @@ module Invidious::Routes::VideoPlayback
headers["Range"] = "bytes=#{range_for_head}"
end
- client = make_client(URI.parse(host), region)
+ client = make_client(URI.parse(host), region, force_resolve = true)
response = HTTP::Client::Response.new(500)
error = ""
5.times do
@@ -57,7 +57,7 @@ module Invidious::Routes::VideoPlayback
if new_host != host
host = new_host
client.close
- client = make_client(URI.parse(new_host), region)
+ client = make_client(URI.parse(new_host), region, force_resolve = true)
end
url = "#{location.request_target}&host=#{location.host}#{region ? "&region=#{region}" : ""}"
@@ -71,7 +71,7 @@ module Invidious::Routes::VideoPlayback
fvip = "3"
host = "https://r#{fvip}---#{mn}.googlevideo.com"
- client = make_client(URI.parse(host), region)
+ client = make_client(URI.parse(host), region, force_resolve = true)
rescue ex
error = ex.message
end
@@ -80,9 +80,14 @@ module Invidious::Routes::VideoPlayback
# Remove the Range header added previously.
headers.delete("Range") if range_header.nil?
+ playback_statistics = get_playback_statistic()
+ playback_statistics["totalRequests"] += 1
+
if response.status_code >= 400
env.response.content_type = "text/plain"
haltf env, response.status_code
+ else
+ playback_statistics["successfulRequests"] += 1
end
if url.includes? "&file=seg.ts"
@@ -191,7 +196,7 @@ module Invidious::Routes::VideoPlayback
break
else
client.close
- client = make_client(URI.parse(host), region)
+ client = make_client(URI.parse(host), region, force_resolve = true)
end
end
diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr
index c2b5c758..bf968734 100644
--- a/src/invidious/search/filters.cr
+++ b/src/invidious/search/filters.cr
@@ -300,9 +300,9 @@ module Invidious::Search
object["9:varint"] = ((page - 1) * 20).to_i64
end
- # If the object is empty, return an empty string,
- # otherwise encode to protobuf then to base64
- return "" if object.empty?
+ # Prevent censoring of self harm topics
+ # See https://github.com/iv-org/invidious/issues/4398
+ object["30:varint"] = 1.to_i64
return object
.try { |i| Protodec::Any.cast_json(i) }
diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr
index 9fbd1374..a8f02056 100644
--- a/src/invidious/videos.cr
+++ b/src/invidious/videos.cr
@@ -227,8 +227,22 @@ struct Video
info.dig?("streamingData", "hlsManifestUrl").try &.as_s
end
- def dash_manifest_url
- info.dig?("streamingData", "dashManifestUrl").try &.as_s
+ def dash_manifest_url : String?
+ raw_dash_url = info.dig?("streamingData", "dashManifestUrl").try &.as_s
+ return nil if raw_dash_url.nil?
+
+ # Use manifest v5 parameter to reduce file size
+ # See https://github.com/iv-org/invidious/issues/4186
+ dash_url = URI.parse(raw_dash_url)
+ dash_query = dash_url.query || ""
+
+ if dash_query.empty?
+ dash_url.path = "#{dash_url.path}/mpd_version/5"
+ else
+ dash_url.query = "#{dash_query}&mpd_version=5"
+ end
+
+ return dash_url.to_s
end
def genre_url : String?
diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr
index 551ce2cb..77520dbe 100644
--- a/src/invidious/videos/parser.cr
+++ b/src/invidious/videos/parser.cr
@@ -78,6 +78,11 @@ def extract_video_info(video_id : String, proxy_region : String? = nil)
# YouTube may return a different video player response than expected.
# See: https://github.com/TeamNewPipe/NewPipe/issues/8713
# Line to be reverted if one day we solve the video not available issue.
+
+ # Although technically not a call to /videoplayback the fact that YouTube is returning the
+ # wrong video means that we should count it as a failure.
+ get_playback_statistic()["totalRequests"] += 1
+
return {
"version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
"reason" => JSON::Any.new("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. <a href=\"https://github.com/iv-org/invidious/issues/3822\">Click here for more info about the issue.</a>"),
diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr
index 031b46da..6d227cfc 100644
--- a/src/invidious/views/components/item.ecr
+++ b/src/invidious/views/components/item.ecr
@@ -82,11 +82,19 @@
</div>
<div class="video-card-row flexible">
- <div class="flex-left"><a href="/channel/<%= item.ucid %>">
- <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %>
- <%- if author_verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end -%>
- </p>
- </a></div>
+ <div class="flex-left">
+ <% if !item.ucid.to_s.empty? %>
+ <a href="/channel/<%= item.ucid %>">
+ <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %>
+ <%- if author_verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end -%>
+ </p>
+ </a>
+ <% else %>
+ <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %>
+ <%- if author_verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end -%>
+ </p>
+ <% end %>
+ </div>
</div>
<% when Category %>
<% else %>
@@ -160,11 +168,19 @@
</div>
<div class="video-card-row flexible">
- <div class="flex-left"><a href="/channel/<%= item.ucid %>">
- <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %>
- <%- if author_verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end -%>
- </p>
- </a></div>
+ <div class="flex-left">
+ <% if !item.ucid.to_s.empty? %>
+ <a href="/channel/<%= item.ucid %>">
+ <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %>
+ <%- if author_verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end -%>
+ </p>
+ </a>
+ <% else %>
+ <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %>
+ <%- if author_verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end -%>
+ </p>
+ <% end %>
+ </div>
<%= rendered "components/video-context-buttons" %>
</div>
diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr
index 07474896..1b020321 100644
--- a/src/invidious/views/watch.ecr
+++ b/src/invidious/views/watch.ecr
@@ -346,7 +346,7 @@ we're going to need to do it here in order to allow for translations.
<h5 class="pure-g">
<div class="pure-u-14-24">
- <% if rv["ucid"]? %>
+ <% if !rv["ucid"].empty? %>
<b style="width:100%"><a href="/channel/<%= rv["ucid"] %>"><%= rv["author"]? %><% if rv["author_verified"]? == "true" %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %></a></b>
<% else %>
<b style="width:100%"><%= rv["author"]? %><% if rv["author_verified"]? == "true" %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %></b>
diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index e9eb726c..81cfb272 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -1,7 +1,6 @@
def add_yt_headers(request)
- if request.headers["User-Agent"] == "Crystal"
- request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
- end
+ request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal"
+ request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
@@ -27,7 +26,7 @@ struct YoutubeConnectionPool
def client(region = nil, &block)
if region
- conn = make_client(url, region)
+ conn = make_client(url, region, force_resolve = true)
response = yield conn
else
conn = pool.checkout
@@ -37,7 +36,7 @@ struct YoutubeConnectionPool
conn.close
conn = HTTP::Client.new(url)
- conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
+ conn.family = CONFIG.force_resolve
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
response = yield conn
@@ -52,7 +51,7 @@ struct YoutubeConnectionPool
private def build_pool
DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
conn = HTTP::Client.new(url)
- conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
+ conn.family = CONFIG.force_resolve
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
conn
@@ -60,9 +59,14 @@ struct YoutubeConnectionPool
end
end
-def make_client(url : URI, region = nil)
+def make_client(url : URI, region = nil, force_resolve : Bool = false)
client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure)
- client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC
+
+ # Some services do not support IPv6.
+ if force_resolve
+ client.family = CONFIG.force_resolve
+ end
+
client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
client.read_timeout = 10.seconds
client.connect_timeout = 10.seconds
@@ -81,8 +85,8 @@ def make_client(url : URI, region = nil)
return client
end
-def make_client(url : URI, region = nil, &block)
- client = make_client(url, region)
+def make_client(url : URI, region = nil, force_resolve : Bool = false, &block)
+ client = make_client(url, region, force_resolve)
begin
yield client
ensure
diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr
index 56325cf7..0e72957e 100644
--- a/src/invidious/yt_backend/extractors.cr
+++ b/src/invidious/yt_backend/extractors.cr
@@ -822,9 +822,9 @@ module HelperExtractors
end
# Retrieves the ID required for querying the InnerTube browse endpoint.
- # Raises when it's unable to do so
+ # Returns an empty string when it's unable to do so
def self.get_browse_id(container)
- return container.dig("navigationEndpoint", "browseEndpoint", "browseId").as_s
+ return container.dig?("navigationEndpoint", "browseEndpoint", "browseId").try &.as_s || ""
end
end
generated by cgit v1.2.3 (git 2.39.1) at 2025-12-17 17:18:09 +0000