From f75a81c9eeb792c0b99075bc47a1243a23b8700b Mon Sep 17 00:00:00 2001 From: "Féry Mathieu (Mathius)" Date: Mon, 21 Feb 2022 10:53:20 +0100 Subject: Make configurable time between each RefreshChannelsJob --- config/config.example.yml | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'config') diff --git a/config/config.example.yml b/config/config.example.yml index 59cb486b..475f2703 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -314,6 +314,14 @@ https_only: false ## channel_threads: 1 +## +## Time between channel_refresh +## +## Accepted values: a valid time interval (hours:min:seconds) +## Default: 00:30:00 +## +channel_refresh_time: 00:30:00 + ## ## Forcefully dump and re-download the entire list of uploaded ## videos when crawling channel (during subscriptions update). -- cgit v1.2.3 From 18197e7e3eca53176e71b1dfe82dd2de85df175c Mon Sep 17 00:00:00 2001 From: "Féry Mathieu (Mathius)" Date: Mon, 21 Feb 2022 11:13:24 +0100 Subject: Lint description of channel_refresh_time --- config/config.example.yml | 2 +- src/invidious/config.cr | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'config') diff --git a/config/config.example.yml b/config/config.example.yml index 475f2703..5d4cea28 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -315,7 +315,7 @@ https_only: false channel_threads: 1 ## -## Time between channel_refresh +## Time between two jobs for crawling videos from channels ## ## Accepted values: a valid time interval (hours:min:seconds) ## Default: 00:30:00 diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 150f8064..9ffd2cd8 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -58,7 +58,7 @@ class Config property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions) @[YAML::Field(converter: TimeSpanConverter)] - property channel_refresh_time : Time::Span = 30.minutes # Time between channel_refresh + property channel_refresh_time : Time::Span = 30.minutes # Time between two jobs for crawling videos from channels property feed_threads : Int32 = 1 # Number of threads to use for updating feeds property output : String = "STDOUT" # Log file path or STDOUT property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr -- cgit v1.2.3 From dfab62ce48ff1d2227d77c2fc61f7a4ea8da1988 Mon Sep 17 00:00:00 2001 From: "Féry Mathieu (Mathius)" Date: Tue, 22 Feb 2022 00:46:26 +0100 Subject: Rename new property to channel_refresh_interval Follow indications : https://github.com/iv-org/invidious/pull/2915#discussion_r811373503 --- config/config.example.yml | 2 +- docker-compose.yml | 2 +- src/invidious/config.cr | 2 +- src/invidious/jobs/refresh_channels_job.cr | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) (limited to 'config') diff --git a/config/config.example.yml b/config/config.example.yml index 5d4cea28..9519f34a 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -320,7 +320,7 @@ channel_threads: 1 ## Accepted values: a valid time interval (hours:min:seconds) ## Default: 00:30:00 ## -channel_refresh_time: 00:30:00 +channel_refresh_interval: 00:30:00 ## ## Forcefully dump and re-download the entire list of uploaded diff --git a/docker-compose.yml b/docker-compose.yml index 964bb702..ab35a496 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,7 @@ services: INVIDIOUS_CONFIG: | log_level: Info channel_threads: 1 - channel_refresh_time: 00:30:00 + channel_refresh_interval: 00:30:00 check_tables: true feed_threads: 1 db: diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 9ffd2cd8..fc24c1e7 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -58,7 +58,7 @@ class Config property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions) @[YAML::Field(converter: TimeSpanConverter)] - property channel_refresh_time : Time::Span = 30.minutes # Time between two jobs for crawling videos from channels + property channel_refresh_interval : Time::Span = 30.minutes # Time between two jobs for crawling videos from channels property feed_threads : Int32 = 1 # Number of threads to use for updating feeds property output : String = "STDOUT" # Log file path or STDOUT property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr index 3e04d1cd..92681408 100644 --- a/src/invidious/jobs/refresh_channels_job.cr +++ b/src/invidious/jobs/refresh_channels_job.cr @@ -58,8 +58,8 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob end end - LOGGER.debug("RefreshChannelsJob: Done, sleeping for #{CONFIG.channel_refresh_time}") - sleep CONFIG.channel_refresh_time + LOGGER.debug("RefreshChannelsJob: Done, sleeping for #{CONFIG.channel_refresh_interval}") + sleep CONFIG.channel_refresh_interval Fiber.yield end end -- cgit v1.2.3 From fd0ac3a6719b3d6280ce2e784ad370c8d10a6129 Mon Sep 17 00:00:00 2001 From: "Féry Mathieu (Mathius)" Date: Tue, 22 Feb 2022 01:35:35 +0100 Subject: Update management of channel_refresh_interval Follow indications: https://github.com/iv-org/invidious/pull/2915#discussion_r811373503 --- config/config.example.yml | 6 +++--- docker-compose.yml | 2 +- src/invidious/helpers/utils.cr | 18 ++++++++++++++++++ src/invidious/user/preferences.cr | 6 +++--- 4 files changed, 25 insertions(+), 7 deletions(-) (limited to 'config') diff --git a/config/config.example.yml b/config/config.example.yml index 9519f34a..a6440e3d 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -317,10 +317,10 @@ channel_threads: 1 ## ## Time between two jobs for crawling videos from channels ## -## Accepted values: a valid time interval (hours:min:seconds) -## Default: 00:30:00 +## Accepted values: a valid time interval (like 1h30m or 90min) +## Default: 30m ## -channel_refresh_interval: 00:30:00 +channel_refresh_interval: 30min ## ## Forcefully dump and re-download the entire list of uploaded diff --git a/docker-compose.yml b/docker-compose.yml index ab35a496..4fdad921 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,7 @@ services: INVIDIOUS_CONFIG: | log_level: Info channel_threads: 1 - channel_refresh_interval: 00:30:00 + channel_refresh_interval: 30m check_tables: true feed_threads: 1 db: diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 22575c57..53f64f4e 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -53,6 +53,24 @@ def recode_length_seconds(time : Int32) : String end end +def decode_interval(string : String) : Time::Span + rawMinutes = string.try &.to_i32? + + if !rawMinutes + hours = /(?\d+)h/.match(string).try &.["hours"].try &.to_i32 + hours ||= 0 + + minutes = /(?\d+)m(?!s)/.match(string).try &.["minutes"].try &.to_i32 + minutes ||= 0 + + time = Time::Span.new(hours: hours, minutes: minutes) + else + time = Time::Span.new(minutes: rawMinutes) + end + + return time +end + def decode_time(string) time = string.try &.to_f? diff --git a/src/invidious/user/preferences.cr b/src/invidious/user/preferences.cr index c01bdd82..9eeed53f 100644 --- a/src/invidious/user/preferences.cr +++ b/src/invidious/user/preferences.cr @@ -259,15 +259,15 @@ struct Preferences module TimeSpanConverter def self.to_yaml(value : Time::Span, yaml : YAML::Nodes::Builder) - return yaml.scalar recode_length_seconds(value.total_seconds.to_i32) + return yaml.scalar value.total_minutes.to_i32 end def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Time::Span if node.is_a?(YAML::Nodes::Scalar) - return decode_time_span(node.value) + return decode_interval(node.value) else node.raise "Expected scalar, not #{node.class}" end end end -end \ No newline at end of file +end -- cgit v1.2.3 From e60a1836fe077fa00fbfc8e8f9e4401b7151192c Mon Sep 17 00:00:00 2001 From: "Féry Mathieu (Mathius)" Date: Tue, 22 Feb 2022 23:19:59 +0100 Subject: Lint config.example.yml and config.cr Follow lint indications : - https://github.com/iv-org/invidious/pull/2915#discussion_r812396203 - https://github.com/iv-org/invidious/pull/2915#discussion_r812396807 --- config/config.example.yml | 7 ++++--- src/invidious/config.cr | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'config') diff --git a/config/config.example.yml b/config/config.example.yml index a6440e3d..866e8944 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -315,12 +315,13 @@ https_only: false channel_threads: 1 ## -## Time between two jobs for crawling videos from channels +## Time interval between two executions of the job that crawls +## channel videos (subscriptions update). ## -## Accepted values: a valid time interval (like 1h30m or 90min) +## Accepted values: a valid time interval (like 1h30m or 90m) ## Default: 30m ## -channel_refresh_interval: 30min +#channel_refresh_interval: 30m ## ## Forcefully dump and re-download the entire list of uploaded diff --git a/src/invidious/config.cr b/src/invidious/config.cr index cf705d21..bebb9ae5 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -58,7 +58,7 @@ class Config # Number of threads to use for crawling videos from channels (for updating subscriptions) property channel_threads : Int32 = 1 - # Time between two jobs for crawling videos from channels + # Time interval between two executions of the job that crawls channel videos (subscriptions update). @[YAML::Field(converter: Preferences::TimeSpanConverter)] property channel_refresh_interval : Time::Span = 30.minutes # Number of threads to use for updating feeds -- cgit v1.2.3