summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/invidious.cr17
-rw-r--r--src/invidious/channels/channels.cr24
-rw-r--r--src/invidious/database/base.cr64
-rw-r--r--src/invidious/database/channels.cr31
-rw-r--r--src/invidious/database/playlists.cr26
-rw-r--r--src/invidious/database/sessions.cr4
-rw-r--r--src/invidious/database/users.cr6
-rw-r--r--src/invidious/jobs/refresh_channels_job.cr2
-rw-r--r--src/invidious/routes/api/v1/authenticated.cr2
-rw-r--r--src/invidious/routes/feeds.cr2
-rw-r--r--src/invidious/routes/playlists.cr16
-rw-r--r--src/invidious/routes/preferences.cr10
-rw-r--r--src/invidious/routes/subscriptions.cr2
-rw-r--r--src/invidious/users.cr2
14 files changed, 107 insertions, 101 deletions
diff --git a/src/invidious.cr b/src/invidious.cr
index 8ba62503..b09f31c2 100644
--- a/src/invidious.cr
+++ b/src/invidious.cr
@@ -112,22 +112,7 @@ OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mo
LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level)
# Check table integrity
-if CONFIG.check_tables
- Invidious::Database.check_enum(PG_DB, "privacy", PlaylistPrivacy)
-
- Invidious::Database.check_table(PG_DB, "channels", InvidiousChannel)
- Invidious::Database.check_table(PG_DB, "channel_videos", ChannelVideo)
- Invidious::Database.check_table(PG_DB, "playlists", InvidiousPlaylist)
- Invidious::Database.check_table(PG_DB, "playlist_videos", PlaylistVideo)
- Invidious::Database.check_table(PG_DB, "nonces", Nonce)
- Invidious::Database.check_table(PG_DB, "session_ids", SessionId)
- Invidious::Database.check_table(PG_DB, "users", User)
- Invidious::Database.check_table(PG_DB, "videos", Video)
-
- if CONFIG.cache_annotations
- Invidious::Database.check_table(PG_DB, "annotations", Annotation)
- end
-end
+Invidious::Database.check_integrity(CONFIG)
# Resolve player dependencies. This is done at compile time.
#
diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr
index 155ec559..6905b6f8 100644
--- a/src/invidious/channels/channels.cr
+++ b/src/invidious/channels/channels.cr
@@ -114,8 +114,9 @@ class ChannelRedirect < Exception
end
end
-def get_batch_channels(channels, refresh = false, pull_all_videos = true, max_threads = 10)
+def get_batch_channels(channels)
finished_channel = Channel(String | Nil).new
+ max_threads = 10
spawn do
active_threads = 0
@@ -130,7 +131,7 @@ def get_batch_channels(channels, refresh = false, pull_all_videos = true, max_th
active_threads += 1
spawn do
begin
- get_channel(ucid, refresh, pull_all_videos)
+ get_channel(ucid)
finished_channel.send(ucid)
rescue ex
finished_channel.send(nil)
@@ -151,23 +152,20 @@ def get_batch_channels(channels, refresh = false, pull_all_videos = true, max_th
return final
end
-def get_channel(id, refresh = true, pull_all_videos = true)
- if channel = Invidious::Database::Channels.select(id)
- if refresh && Time.utc - channel.updated > 10.minutes
- channel = fetch_channel(id, pull_all_videos: pull_all_videos)
- Invidious::Database::Channels.insert(channel, update_on_conflict: true)
- end
- else
- channel = fetch_channel(id, pull_all_videos: pull_all_videos)
- Invidious::Database::Channels.insert(channel)
+def get_channel(id) : InvidiousChannel
+ channel = Invidious::Database::Channels.select(id)
+
+ if channel.nil? || (Time.utc - channel.updated) > 2.days
+ channel = fetch_channel(id, pull_all_videos: false)
+ Invidious::Database::Channels.insert(channel, update_on_conflict: true)
end
return channel
end
-def fetch_channel(ucid, pull_all_videos = true, locale = nil)
+def fetch_channel(ucid, pull_all_videos : Bool)
LOGGER.debug("fetch_channel: #{ucid}")
- LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}, locale = #{locale}")
+ LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}")
LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed")
rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body
diff --git a/src/invidious/database/base.cr b/src/invidious/database/base.cr
index 6e49ea1a..0fb1b6af 100644
--- a/src/invidious/database/base.cr
+++ b/src/invidious/database/base.cr
@@ -3,26 +3,52 @@ require "pg"
module Invidious::Database
extend self
- def check_enum(db, enum_name, struct_type = nil)
+ # Checks table integrity
+ #
+ # Note: config is passed as a parameter to avoid complex
+ # dependencies between different parts of the software.
+ def check_integrity(cfg)
+ return if !cfg.check_tables
+ Invidious::Database.check_enum("privacy", PlaylistPrivacy)
+
+ Invidious::Database.check_table("channels", InvidiousChannel)
+ Invidious::Database.check_table("channel_videos", ChannelVideo)
+ Invidious::Database.check_table("playlists", InvidiousPlaylist)
+ Invidious::Database.check_table("playlist_videos", PlaylistVideo)
+ Invidious::Database.check_table("nonces", Nonce)
+ Invidious::Database.check_table("session_ids", SessionId)
+ Invidious::Database.check_table("users", User)
+ Invidious::Database.check_table("videos", Video)
+
+ if cfg.cache_annotations
+ Invidious::Database.check_table("annotations", Annotation)
+ end
+ end
+
+ #
+ # Table/enum integrity checks
+ #
+
+ def check_enum(enum_name, struct_type = nil)
return # TODO
- if !db.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool)
+ if !PG_DB.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool)
LOGGER.info("check_enum: CREATE TYPE #{enum_name}")
- db.using_connection do |conn|
+ PG_DB.using_connection do |conn|
conn.as(PG::Connection).exec_all(File.read("config/sql/#{enum_name}.sql"))
end
end
end
- def check_table(db, table_name, struct_type = nil)
+ def check_table(table_name, struct_type = nil)
# Create table if it doesn't exist
begin
- db.exec("SELECT * FROM #{table_name} LIMIT 0")
+ PG_DB.exec("SELECT * FROM #{table_name} LIMIT 0")
rescue ex
LOGGER.info("check_table: check_table: CREATE TABLE #{table_name}")
- db.using_connection do |conn|
+ PG_DB.using_connection do |conn|
conn.as(PG::Connection).exec_all(File.read("config/sql/#{table_name}.sql"))
end
end
@@ -30,7 +56,7 @@ module Invidious::Database
return if !struct_type
struct_array = struct_type.type_array
- column_array = get_column_array(db, table_name)
+ column_array = get_column_array(PG_DB, table_name)
column_types = File.read("config/sql/#{table_name}.sql").match(/CREATE TABLE public\.#{table_name}\n\((?<types>[\d\D]*?)\);/)
.try &.["types"].split(",").map(&.strip).reject &.starts_with?("CONSTRAINT")
@@ -41,14 +67,14 @@ module Invidious::Database
if !column_array[i]?
new_column = column_types.select(&.starts_with?(name))[0]
LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
- db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
+ PG_DB.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
next
end
# Column doesn't exist
if !column_array.includes? name
new_column = column_types.select(&.starts_with?(name))[0]
- db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
+ PG_DB.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
end
# Column exists but in the wrong position, rotate
@@ -59,29 +85,29 @@ module Invidious::Database
# There's a column we didn't expect
if !new_column
LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]}")
- db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
+ PG_DB.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
- column_array = get_column_array(db, table_name)
+ column_array = get_column_array(PG_DB, table_name)
next
end
LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
- db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
+ PG_DB.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
LOGGER.info("check_table: UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}")
- db.exec("UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}")
+ PG_DB.exec("UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}")
LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
- db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
+ PG_DB.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
LOGGER.info("check_table: ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}")
- db.exec("ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}")
+ PG_DB.exec("ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}")
- column_array = get_column_array(db, table_name)
+ column_array = get_column_array(PG_DB, table_name)
end
else
LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
- db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
+ PG_DB.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
end
end
end
@@ -91,14 +117,14 @@ module Invidious::Database
column_array.each do |column|
if !struct_array.includes? column
LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE")
- db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE")
+ PG_DB.exec("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE")
end
end
end
def get_column_array(db, table_name)
column_array = [] of String
- db.query("SELECT * FROM #{table_name} LIMIT 0") do |rs|
+ PG_DB.query("SELECT * FROM #{table_name} LIMIT 0") do |rs|
rs.column_count.times do |i|
column = rs.as(PG::ResultSet).field(i)
column_array << column.name
diff --git a/src/invidious/database/channels.cr b/src/invidious/database/channels.cr
index 134cf59d..df44e485 100644
--- a/src/invidious/database/channels.cr
+++ b/src/invidious/database/channels.cr
@@ -35,21 +35,31 @@ module Invidious::Database::Channels
def update_author(id : String, author : String)
request = <<-SQL
UPDATE channels
- SET updated = $1, author = $2, deleted = false
- WHERE id = $3
+ SET updated = now(), author = $1, deleted = false
+ WHERE id = $2
+ SQL
+
+ PG_DB.exec(request, author, id)
+ end
+
+ def update_subscription_time(id : String)
+ request = <<-SQL
+ UPDATE channels
+ SET subscribed = now()
+ WHERE id = $1
SQL
- PG_DB.exec(request, Time.utc, author, id)
+ PG_DB.exec(request, id)
end
def update_mark_deleted(id : String)
request = <<-SQL
UPDATE channels
- SET updated = $1, deleted = true
- WHERE id = $2
+ SET updated = now(), deleted = true
+ WHERE id = $1
SQL
- PG_DB.exec(request, Time.utc, id)
+ PG_DB.exec(request, id)
end
# -------------------
@@ -67,14 +77,13 @@ module Invidious::Database::Channels
def select(ids : Array(String)) : Array(InvidiousChannel)?
return [] of InvidiousChannel if ids.empty?
- values = ids.map { |id| %(('#{id}')) }.join(",")
request = <<-SQL
SELECT * FROM channels
- WHERE id = ANY(VALUES #{values})
+ WHERE id = ANY($1)
SQL
- return PG_DB.query_all(request, as: InvidiousChannel)
+ return PG_DB.query_all(request, ids, as: InvidiousChannel)
end
end
@@ -117,11 +126,11 @@ module Invidious::Database::ChannelVideos
request = <<-SQL
SELECT * FROM channel_videos
- WHERE id IN (#{arg_array(ids)})
+ WHERE id = ANY($1)
ORDER BY published DESC
SQL
- return PG_DB.query_all(request, args: ids, as: ChannelVideo)
+ return PG_DB.query_all(request, ids, as: ChannelVideo)
end
def select_notfications(ucid : String, since : Time) : Array(ChannelVideo)
diff --git a/src/invidious/database/playlists.cr b/src/invidious/database/playlists.cr
index 7a5f61dc..c6754a1e 100644
--- a/src/invidious/database/playlists.cr
+++ b/src/invidious/database/playlists.cr
@@ -59,11 +59,11 @@ module Invidious::Database::Playlists
def update_subscription_time(id : String)
request = <<-SQL
UPDATE playlists
- SET subscribed = $1
- WHERE id = $2
+ SET subscribed = now()
+ WHERE id = $1
SQL
- PG_DB.exec(request, Time.utc, id)
+ PG_DB.exec(request, id)
end
def update_video_added(id : String, index : String | Int64)
@@ -71,11 +71,11 @@ module Invidious::Database::Playlists
UPDATE playlists
SET index = array_append(index, $1),
video_count = cardinality(index) + 1,
- updated = $2
- WHERE id = $3
+ updated = now()
+ WHERE id = $2
SQL
- PG_DB.exec(request, index, Time.utc, id)
+ PG_DB.exec(request, index, id)
end
def update_video_removed(id : String, index : String | Int64)
@@ -83,28 +83,24 @@ module Invidious::Database::Playlists
UPDATE playlists
SET index = array_remove(index, $1),
video_count = cardinality(index) - 1,
- updated = $2
- WHERE id = $3
+ updated = now()
+ WHERE id = $2
SQL
- PG_DB.exec(request, index, Time.utc, id)
+ PG_DB.exec(request, index, id)
end
# -------------------
# Salect
# -------------------
- def select(*, id : String, raise_on_fail : Bool = false) : InvidiousPlaylist?
+ def select(*, id : String) : InvidiousPlaylist?
request = <<-SQL
SELECT * FROM playlists
WHERE id = $1
SQL
- if raise_on_fail
- return PG_DB.query_one(request, id, as: InvidiousPlaylist)
- else
- return PG_DB.query_one?(request, id, as: InvidiousPlaylist)
- end
+ return PG_DB.query_one?(request, id, as: InvidiousPlaylist)
end
def select_all(*, author : String) : Array(InvidiousPlaylist)
diff --git a/src/invidious/database/sessions.cr b/src/invidious/database/sessions.cr
index d5f85dd6..96587082 100644
--- a/src/invidious/database/sessions.cr
+++ b/src/invidious/database/sessions.cr
@@ -10,12 +10,12 @@ module Invidious::Database::SessionIDs
def insert(sid : String, email : String, handle_conflicts : Bool = false)
request = <<-SQL
INSERT INTO session_ids
- VALUES ($1, $2, $3)
+ VALUES ($1, $2, now())
SQL
request += " ON CONFLICT (id) DO NOTHING" if handle_conflicts
- PG_DB.exec(request, sid, email, Time.utc)
+ PG_DB.exec(request, sid, email)
end
# -------------------
diff --git a/src/invidious/database/users.cr b/src/invidious/database/users.cr
index 53724dbf..26be4270 100644
--- a/src/invidious/database/users.cr
+++ b/src/invidious/database/users.cr
@@ -143,11 +143,11 @@ module Invidious::Database::Users
def clear_notifications(user : User)
request = <<-SQL
UPDATE users
- SET notifications = '{}', updated = $1
- WHERE email = $2
+ SET notifications = '{}', updated = now()
+ WHERE email = $1
SQL
- PG_DB.exec(request, Time.utc, user.email)
+ PG_DB.exec(request, user.email)
end
# -------------------
diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr
index 941089c1..55fb8154 100644
--- a/src/invidious/jobs/refresh_channels_job.cr
+++ b/src/invidious/jobs/refresh_channels_job.cr
@@ -30,7 +30,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
spawn do
begin
LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel")
- channel = fetch_channel(id, CONFIG.full_refresh)
+ channel = fetch_channel(id, pull_all_videos: CONFIG.full_refresh)
lim_fibers = max_fibers
diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr
index fda655ef..4d0fe030 100644
--- a/src/invidious/routes/api/v1/authenticated.cr
+++ b/src/invidious/routes/api/v1/authenticated.cr
@@ -92,7 +92,7 @@ module Invidious::Routes::API::V1::Authenticated
ucid = env.params.url["ucid"]
if !user.subscriptions.includes? ucid
- get_channel(ucid, false, false)
+ get_channel(ucid)
Invidious::Database::Users.subscribe_channel(user, ucid)
end
diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr
index fd8c25ce..c323cdf7 100644
--- a/src/invidious/routes/feeds.cr
+++ b/src/invidious/routes/feeds.cr
@@ -362,7 +362,7 @@ module Invidious::Routes::Feeds
end
if ucid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["channel_id"]?
- PG_DB.exec("UPDATE channels SET subscribed = $1 WHERE id = $2", Time.utc, ucid)
+ Invidious::Database::Channels.update_subscription_time(ucid)
elsif plid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["playlist_id"]?
Invidious::Database::Playlists.update_subscription_time(plid)
else
diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr
index d437b79c..1c4f1bef 100644
--- a/src/invidious/routes/playlists.cr
+++ b/src/invidious/routes/playlists.cr
@@ -151,12 +151,8 @@ module Invidious::Routes::Playlists
page = env.params.query["page"]?.try &.to_i?
page ||= 1
- begin
- playlist = Invidious::Database::Playlists.select(id: plid, raise_on_fail: true)
- if !playlist || playlist.author != user.email
- return env.redirect referer
- end
- rescue ex
+ playlist = Invidious::Database::Playlists.select(id: plid)
+ if !playlist || playlist.author != user.email
return env.redirect referer
end
@@ -235,12 +231,8 @@ module Invidious::Routes::Playlists
page = env.params.query["page"]?.try &.to_i?
page ||= 1
- begin
- playlist = Invidious::Database::Playlists.select(id: plid, raise_on_fail: true)
- if !playlist || playlist.author != user.email
- return env.redirect referer
- end
- rescue ex
+ playlist = Invidious::Database::Playlists.select(id: plid)
+ if !playlist || playlist.author != user.email
return env.redirect referer
end
diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr
index faae03bc..9c740cf2 100644
--- a/src/invidious/routes/preferences.cr
+++ b/src/invidious/routes/preferences.cr
@@ -327,7 +327,7 @@ module Invidious::Routes::PreferencesRoute
user.subscriptions += body["subscriptions"].as_a.map(&.as_s)
user.subscriptions.uniq!
- user.subscriptions = get_batch_channels(user.subscriptions, false, false)
+ user.subscriptions = get_batch_channels(user.subscriptions)
Invidious::Database::Users.update_subscriptions(user)
end
@@ -409,7 +409,7 @@ module Invidious::Routes::PreferencesRoute
end
user.subscriptions.uniq!
- user.subscriptions = get_batch_channels(user.subscriptions, false, false)
+ user.subscriptions = get_batch_channels(user.subscriptions)
Invidious::Database::Users.update_subscriptions(user)
when "import_freetube"
@@ -418,7 +418,7 @@ module Invidious::Routes::PreferencesRoute
end
user.subscriptions.uniq!
- user.subscriptions = get_batch_channels(user.subscriptions, false, false)
+ user.subscriptions = get_batch_channels(user.subscriptions)
Invidious::Database::Users.update_subscriptions(user)
when "import_newpipe_subscriptions"
@@ -437,7 +437,7 @@ module Invidious::Routes::PreferencesRoute
end
user.subscriptions.uniq!
- user.subscriptions = get_batch_channels(user.subscriptions, false, false)
+ user.subscriptions = get_batch_channels(user.subscriptions)
Invidious::Database::Users.update_subscriptions(user)
when "import_newpipe"
@@ -456,7 +456,7 @@ module Invidious::Routes::PreferencesRoute
user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map(&.lchop("https://www.youtube.com/channel/"))
user.subscriptions.uniq!
- user.subscriptions = get_batch_channels(user.subscriptions, false, false)
+ user.subscriptions = get_batch_channels(user.subscriptions)
Invidious::Database::Users.update_subscriptions(user)
diff --git a/src/invidious/routes/subscriptions.cr b/src/invidious/routes/subscriptions.cr
index 29152afb..ec8fe67b 100644
--- a/src/invidious/routes/subscriptions.cr
+++ b/src/invidious/routes/subscriptions.cr
@@ -51,7 +51,7 @@ module Invidious::Routes::Subscriptions
case action
when "action_create_subscription_to_channel"
if !user.subscriptions.includes? channel_id
- get_channel(channel_id, false, false)
+ get_channel(channel_id)
Invidious::Database::Users.subscribe_channel(user, channel_id)
end
when "action_remove_subscriptions"
diff --git a/src/invidious/users.cr b/src/invidious/users.cr
index 49074994..a7ee72a9 100644
--- a/src/invidious/users.cr
+++ b/src/invidious/users.cr
@@ -74,7 +74,7 @@ def fetch_user(sid, headers)
end
end
- channels = get_batch_channels(channels, false, false)
+ channels = get_batch_channels(channels)
email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"]))
if email
generated by cgit v1.2.3 (git 2.39.1) at 2025-12-18 23:30:16 +0000