Make configurable time between each RefreshChannelsJob

This commit is contained in:
Féry Mathieu (Mathius) 2022-02-21 10:53:20 +01:00
parent 85ba04b715
commit f75a81c9ee
No known key found for this signature in database
GPG Key ID: F9CCC80C18A59037
5 changed files with 49 additions and 22 deletions

View File

@ -314,6 +314,14 @@ https_only: false
## ##
channel_threads: 1 channel_threads: 1
##
## Time between channel_refresh
##
## Accepted values: a valid time interval (hours:min:seconds)
## Default: 00:30:00
##
channel_refresh_time: 00:30:00
## ##
## Forcefully dump and re-download the entire list of uploaded ## Forcefully dump and re-download the entire list of uploaded
## videos when crawling channel (during subscriptions update). ## videos when crawling channel (during subscriptions update).

View File

@ -23,7 +23,9 @@ services:
environment: environment:
# Adapted from ./config/config.yml # Adapted from ./config/config.yml
INVIDIOUS_CONFIG: | INVIDIOUS_CONFIG: |
log_level: Info
channel_threads: 1 channel_threads: 1
channel_refresh_time: 00:30:00
check_tables: true check_tables: true
feed_threads: 1 feed_threads: 1
db: db:

View File

@ -56,11 +56,13 @@ end
class Config class Config
include YAML::Serializable include YAML::Serializable
property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions) property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions)
property feed_threads : Int32 = 1 # Number of threads to use for updating feeds @[YAML::Field(converter: TimeSpanConverter)]
property output : String = "STDOUT" # Log file path or STDOUT property channel_refresh_time : Time::Span = 30.minutes # Time between channel_refresh
property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr property feed_threads : Int32 = 1 # Number of threads to use for updating feeds
property db : DBConfig? = nil # Database configuration with separate parameters (username, hostname, etc) property output : String = "STDOUT" # Log file path or STDOUT
property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr
property db : DBConfig? = nil # Database configuration with separate parameters (username, hostname, etc)
@[YAML::Field(converter: Preferences::URIConverter)] @[YAML::Field(converter: Preferences::URIConverter)]
property database_url : URI = URI.parse("") # Database configuration using 12-Factor "Database URL" syntax property database_url : URI = URI.parse("") # Database configuration using 12-Factor "Database URL" syntax

View File

@ -18,23 +18,39 @@ def elapsed_text(elapsed)
"#{(millis * 1000).round(2)}µs" "#{(millis * 1000).round(2)}µs"
end end
def decode_length_seconds(string) module TimeSpanConverter
length_seconds = string.gsub(/[^0-9:]/, "") def self.to_yaml(value : Time::Span, yaml : YAML::Nodes::Builder)
return 0_i32 if length_seconds.empty? return yaml.scalar recode_length_seconds(value.total_seconds.to_i32)
end
length_seconds = length_seconds.split(":").map { |x| x.to_i? || 0 } def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Time::Span
length_seconds = [0] * (3 - length_seconds.size) + length_seconds if node.is_a?(YAML::Nodes::Scalar)
return decode_time_span(node.value)
length_seconds = Time::Span.new( else
hours: length_seconds[0], node.raise "Expected scalar, not #{node.class}"
minutes: length_seconds[1], end
seconds: length_seconds[2] end
).total_seconds.to_i32
return length_seconds
end end
def recode_length_seconds(time) def decode_time_span(string : String) : Time::Span
time_span = string.gsub(/[^0-9:]/, "")
return Time::Span.new(seconds: 0) if time_span.empty?
time_span = time_span.split(":").map { |x| x.to_i? || 0 }
time_span = [0] * (3 - time_span.size) + time_span
return Time::Span.new(
hours: time_span[0],
minutes: time_span[1],
seconds: time_span[2]
)
end
def decode_length_seconds(string : String) : Int32
return decode_time_span(string).total_seconds.to_i32
end
def recode_length_seconds(time : Int32) : String
if time <= 0 if time <= 0
return "" return ""
else else

View File

@ -58,9 +58,8 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
end end
end end
# TODO: make this configurable LOGGER.debug("RefreshChannelsJob: Done, sleeping for #{CONFIG.channel_refresh_time}")
LOGGER.debug("RefreshChannelsJob: Done, sleeping for thirty minutes") sleep CONFIG.channel_refresh_time
sleep 30.minutes
Fiber.yield Fiber.yield
end end
end end