Compare commits

..

1 Commits

Author SHA1 Message Date
Emilien Devos
3c8a14070a limit feeds and delete materialized views 2026-11-03 19:29:12 +01:00
21 changed files with 53 additions and 225 deletions

View File

@@ -66,7 +66,7 @@ jobs:
shell: bash
- name: Install Crystal
uses: crystal-lang/install-crystal@v1.9.5
uses: crystal-lang/install-crystal@v1.9.4
with:
crystal: ${{ matrix.crystal }}
@@ -138,7 +138,7 @@ jobs:
- name: Install Crystal
id: lint_step_install_crystal
uses: crystal-lang/install-crystal@v1.9.5
uses: crystal-lang/install-crystal@v1.9.4
with:
crystal: 1.22.2

View File

@@ -7,48 +7,6 @@
## v2.20261113.0
### Wrap-up
This release adds an AI-made indicator for videos that carry the AI badge and continues support for custom channel handle URLs with tab paths. It also fixes parsing of premiere timestamps on channel videos, corrects the ggpht route, and resolves parsing of published dates and view counts for collab videos.
For instance owners, Invidious now displays a friendly message when the config.yml file is missing. Dependency updates include http_proxy 0.17.0 and athena-negotiation 0.2.0, and CI bumps crystal-lang/install-crystal to 1.9.5.
### New features & important changes
#### For Users
- Added an AI-made indicator for videos that carry the AI badge (#6113)
- Support for custom channel handle URLs with tab paths, e.g. `{INSTANCE}/markrober/streams` (#5811)
#### For instance owners
- Display a friendly message when the config.yml file is missing (#6121)
- Updated http_proxy to 0.17.0 (#6142)
- Updated athena-negotiation to 0.2.0 (#6141)
#### For developers
- Bumped crystal-lang/install-crystal from 1.9.4 to 1.9.5 (#6129)
### Bugs fixed
#### For Users
- Fixed parsing of premiere_timestamp on channel videos (#6140)
- Fixed the ggpht route (#6143)
- Fixed parsing of published date and view count for collab videos (#5812)
### Full list of pull requests merged since the last release (newest first)
* fix: fix ggpht route after commit 3df8616 (https://github.com/iv-org/invidious/pull/6143, by @Fijxu)
* chore(deps): update http_proxy to 0.17.0 (https://github.com/iv-org/invidious/pull/6142, by @Fijxu)
* chore(deps): update athena-negotiation to 0.2.0 (https://github.com/iv-org/invidious/pull/6141, by @Fijxu)
* fix: correctly parse premiere_timestamp on channel videos (https://github.com/iv-org/invidious/pull/6140, by @Fijxu)
* chore(deps): bump crystal-lang/install-crystal from 1.9.4 to 1.9.5 (https://github.com/iv-org/invidious/pull/6129, by @dependabot[bot])
* feat: add AI made indicator for videos that have the AI badge (https://github.com/iv-org/invidious/pull/6113, by @Fijxu)
* chore: Display a nice message when the config.yml file is missing (https://github.com/iv-org/invidious/pull/6121, by @Fijxu)
* Release v2.20261103.0 (https://github.com/iv-org/invidious/pull/6118, by @github-actions[bot])
* fix(parser): parse published date and view count for collab videos (https://github.com/iv-org/invidious/pull/5812, by @radmorecameron)
* feat: Adds support for custom channel handle urls with tab paths (ex: `{INSTANCE}/markrober/streams`) (https://github.com/iv-org/invidious/pull/5811, by @radmorecameron)
## v2.20261103.0
### Wrap-up

View File

@@ -0,0 +1,6 @@
CREATE INDEX channel_videos_ucid_published_idx
ON public.channel_videos
USING btree
(ucid COLLATE pg_catalog."default", published);
DROP INDEX channel_videos_ucid_idx;

View File

@@ -19,12 +19,12 @@ CREATE TABLE IF NOT EXISTS public.channel_videos
GRANT ALL ON TABLE public.channel_videos TO current_user;
-- Index: public.channel_videos_ucid_idx
-- Index: public.channel_videos_ucid_published_idx
-- DROP INDEX public.channel_videos_ucid_idx;
-- DROP INDEX public.channel_videos_ucid_published_idx;
CREATE INDEX IF NOT EXISTS channel_videos_ucid_idx
CREATE INDEX IF NOT EXISTS channel_videos_ucid_published_idx
ON public.channel_videos
USING btree
(ucid COLLATE pg_catalog."default");
(ucid COLLATE pg_catalog."default", published);

View File

@@ -507,6 +507,5 @@
"timeline_parse_error_placeholder_heading": "Unable to parse item",
"timeline_parse_error_placeholder_message": "Invidious encountered an error while trying to parse this item. For more information see below:",
"timeline_parse_error_show_technical_details": "Show technical details",
"dmca_content": "This video cannot be downloaded on this instance due to a DMCA/copyright infringement letter sent to the instance administrator.",
"video_details_ai_made": "AI Made video?:"
"dmca_content": "This video cannot be downloaded on this instance due to a DMCA/copyright infringement letter sent to the instance administrator."
}

View File

@@ -6,7 +6,7 @@ shards:
athena-negotiation:
git: https://github.com/athena-framework/negotiation.git
version: 0.2.0
version: 0.1.1
backtracer:
git: https://github.com/sija/backtracer.cr.git
@@ -22,7 +22,7 @@ shards:
http_proxy:
git: https://github.com/mamantoha/http_proxy.git
version: 0.17.0
version: 0.10.3
kemal:
git: https://github.com/kemalcr/kemal.git

View File

@@ -1,5 +1,5 @@
name: invidious
version: 2.20261113.0-dev
version: 2.20261103.0-dev
authors:
- Invidious team <contact@invidious.io>
@@ -29,10 +29,10 @@ dependencies:
version: ~> 0.1.5
athena-negotiation:
github: athena-framework/negotiation
version: ~> 0.2.0
version: ~> 0.1.1
http_proxy:
github: mamantoha/http_proxy
version: ~> 0.17.0
version: ~> 0.10.3
development_dependencies:
spectator:

View File

@@ -130,14 +130,6 @@ Kemal.config.extra_options do |parser|
exit
end
end
parser.on("-f THREADS", "--feed-threads=THREADS", "Number of threads for refreshing feeds (default: #{CONFIG.feed_threads})") do |number|
begin
CONFIG.feed_threads = number.to_i
rescue
puts "THREADS must be integer"
exit
end
end
parser.on("-o OUTPUT", "--output=OUTPUT", "Redirect output (default: #{CONFIG.output})") do |output|
CONFIG.output = output
end
@@ -189,10 +181,6 @@ if CONFIG.channel_threads > 0
Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB)
end
if CONFIG.feed_threads > 0
Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB)
end
if CONFIG.statistics_enabled
Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, SOFTWARE)
end

View File

@@ -101,8 +101,6 @@ class Config
# Time interval between two executions of the job that crawls channel videos (subscriptions update).
@[YAML::Field(converter: Preferences::TimeSpanConverter)]
property channel_refresh_interval : Time::Span = 30.minutes
# Number of threads to use for updating feeds
property feed_threads : Int32 = 1
# Log file path or STDOUT
property output : String = "STDOUT"
# Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr
@@ -228,16 +226,7 @@ class Config
env_config_yaml = "INVIDIOUS_CONFIG"
config_file = ENV.has_key?(env_config_file) ? ENV.fetch(env_config_file) : "config/config.yml"
config_yaml = if ENV.has_key?(env_config_yaml)
ENV.fetch(env_config_yaml)
else
begin
File.read(config_file)
rescue ex
puts "Config: Failed to load configuration file: #{ex.message}"
exit(1)
end
end
config_yaml = ENV.has_key?(env_config_yaml) ? ENV.fetch(env_config_yaml) : File.read(config_file)
config = Config.from_yaml(config_yaml)

View File

@@ -94,20 +94,6 @@ def decode_date(string : String)
return Time.utc(string.to_i, 1, 1)
end
# Try to parse "Premieres 6/29/27, 9:00 PM"
# For some reason, the Innertube response contains a '\u202F'
# character in the separation between the time and the period
# so the raw string returned by Innertube is something like this:
# "Premieres 6/29/27, 9:00\u202FPM". Instead of using a space character
# it uses the '\u202F' character.
#
# Raw: "Premieres 6/29/27, 9:00 PM"
begin
string2 = string.lchop("Premieres ").gsub('\u202F', " ")
return Time.parse(string2, "%m/%d/%y, %l:%M %p", Time::Location.local)
rescue
end
# Try to parse as format Jul 10, 2000
begin
return Time.parse(string, "%b %-d, %Y", Time::Location.local)

View File

@@ -1,75 +0,0 @@
class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
private getter db : DB::Database
def initialize(@db)
end
def begin
max_fibers = CONFIG.feed_threads
active_fibers = 0
active_channel = ::Channel(Bool).new
loop do
db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs|
rs.each do
email = rs.read(String)
view_name = "subscriptions_#{sha256(email)}"
if active_fibers >= max_fibers
if active_channel.receive
active_fibers -= 1
end
end
active_fibers += 1
spawn do
begin
# Drop outdated views
column_array = Invidious::Database.get_column_array(db, view_name)
ChannelVideo.type_array.each_with_index do |name, i|
if name != column_array[i]?
LOGGER.info("RefreshFeedsJob: DROP MATERIALIZED VIEW #{view_name}")
db.exec("DROP MATERIALIZED VIEW #{view_name}")
raise "view does not exist"
end
end
if !db.query_one("SELECT pg_get_viewdef('#{view_name}')", as: String).includes? "WHERE ((cv.ucid = ANY (u.subscriptions))"
LOGGER.info("RefreshFeedsJob: Materialized view #{view_name} is out-of-date, recreating...")
db.exec("DROP MATERIALIZED VIEW #{view_name}")
end
db.exec("REFRESH MATERIALIZED VIEW #{view_name}")
db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
rescue ex
# Rename old views
begin
legacy_view_name = "subscriptions_#{sha256(email)[0..7]}"
db.exec("SELECT * FROM #{legacy_view_name} LIMIT 0")
LOGGER.info("RefreshFeedsJob: RENAME MATERIALIZED VIEW #{legacy_view_name}")
db.exec("ALTER MATERIALIZED VIEW #{legacy_view_name} RENAME TO #{view_name}")
rescue ex
begin
# While iterating through, we may have an email stored from a deleted account
if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool)
LOGGER.info("RefreshFeedsJob: CREATE #{view_name}")
db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{Invidious::User::Users::MATERIALIZED_VIEW_SQL.call(email)}")
db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
end
rescue ex
LOGGER.error("RefreshFeedJobs: REFRESH #{email} : #{ex.message}")
end
end
end
active_channel.send(true)
end
end
end
sleep 5.seconds
Fiber.yield
end
end
end

View File

@@ -64,7 +64,6 @@ module Invidious::JSONify::APIv1
json.field "liveNow", video.live_now
json.field "isPostLiveDvr", video.post_live_dvr
json.field "isUpcoming", video.upcoming?
json.field "aiMade", video.ai_made
if video.premiere_timestamp
json.field "premiereTimestamp", video.premiere_timestamp.try &.to_unix

View File

@@ -123,10 +123,8 @@ module Invidious::Routes::Account
return Errors.error_template(400, ex)
end
view_name = "subscriptions_#{sha256(user.email)}"
Invidious::Database::Users.delete(user)
Invidious::Database::SessionIDs.delete(email: user.email)
PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}")
env.request.cookies.each do |cookie|
cookie.expires = Time.utc(1990, 1, 1)

View File

@@ -125,9 +125,6 @@ module Invidious::Routes::Login
Invidious::Database::Users.insert(user)
Invidious::Database::SessionIDs.insert(sid, email)
view_name = "subscriptions_#{sha256(user.email)}"
PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{Invidious::User::Users::MATERIALIZED_VIEW_SQL.call(user.email)}")
if alt = CONFIG.alternative_domains.index(host)
env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.alternative_domains[alt], sid)
else

View File

@@ -222,8 +222,7 @@ module Invidious::Routing
end
def register_image_routes
get "/ggpht/:id", Routes::Images, :ggpht
get "/ggpht/ytc/:id", Routes::Images, :ggpht
get "/ggpht/*", Routes::Images, :ggpht
options "/sb/:authority/:id/:storyboard/:index", Routes::Images, :options_storyboard
get "/sb/:authority/:id/:storyboard/:index", Routes::Images, :get_storyboard
get "/s_p/:id/:name", Routes::Images, :s_p_image

View File

@@ -36,19 +36,19 @@ module Invidious::Search::Processors
# Search inside of user subscriptions
def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo)
view_name = "subscriptions_#{sha256(user.email)}"
return PG_DB.query_all(<<-SQL,
SELECT id,title,published,updated,ucid,author,length_seconds
FROM (
SELECT *,
to_tsvector(#{view_name}.title) ||
to_tsvector(#{view_name}.author)
as document
FROM #{view_name}
) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;
SELECT cv.*,
to_tsvector(cv.title) ||
to_tsvector(cv.author) AS document
FROM channel_videos cv
JOIN users ON cv.ucid = any(users.subscriptions)
WHERE users.email = $1 AND published > now() - interval '1 month'
ORDER BY published
) v_search WHERE v_search.document @@ plainto_tsquery($2) LIMIT 20 OFFSET $3;
SQL
query.text, (query.page - 1) * 20,
user.email, query.text, (query.page - 1) * 20,
as: ChannelVideo
)
end

View File

@@ -2,8 +2,6 @@ require "crypto/bcrypt/password"
module Invidious::User::Users
extend self
# Materialized views may not be defined using bound parameters (`$1` as used elsewhere)
MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" }
def create_user(sid, email, password)
password = Crypto::Bcrypt::Password.create(password, cost: 10)
@@ -29,7 +27,6 @@ module Invidious::User::Users
offset = (page - 1) * limit
notifications = Invidious::Database::Users.select_notifications(user)
view_name = "subscriptions_#{sha256(user.email)}"
if user.preferences.notifications_only && !notifications.empty?
# Only show notifications
@@ -54,11 +51,19 @@ module Invidious::User::Users
# Show latest video from a channel that a user hasn't watched
# "unseen_only" isn't really correct here, more accurate would be "unwatched_only"
videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY ($1::text[]) ORDER BY ucid, published DESC", user.watched, as: ChannelVideo)
videos = PG_DB.query_all("SELECT DISTINCT ON (cv.ucid) cv.* " \
"FROM channel_videos cv " \
"JOIN users ON cv.ucid = any(users.subscriptions) " \
"WHERE users.email = $1 AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' " \
"ORDER BY ucid, published DESC", user.email, as: ChannelVideo)
else
# Show latest video from each channel
videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo)
videos = PG_DB.query_all("SELECT DISTINCT ON (cv.ucid) cv.* " \
"FROM channel_videos cv " \
"JOIN users ON cv.ucid = any(users.subscriptions) " \
"WHERE users.email = $1 AND published > now() - interval '1 month' " \
"ORDER BY ucid, published DESC", user.email, as: ChannelVideo)
end
videos.sort_by!(&.published).reverse!
@@ -66,11 +71,19 @@ module Invidious::User::Users
if user.preferences.unseen_only
# Only show unwatched
videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY ($1::text[]) ORDER BY published DESC LIMIT $2 OFFSET $3", user.watched, limit, offset, as: ChannelVideo)
videos = PG_DB.query_all("SELECT cv.* " \
"FROM channel_videos cv " \
"JOIN users ON cv.ucid = any(users.subscriptions) " \
"WHERE users.email = $1 AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' " \
"ORDER BY published DESC LIMIT $2 OFFSET $3", user.email, limit, offset, as: ChannelVideo)
else
# Sort subscriptions as normal
videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
videos = PG_DB.query_all("SELECT cv.* " \
"FROM channel_videos cv " \
"JOIN users ON cv.ucid = any(users.subscriptions) " \
"WHERE users.email = $1 AND published > now() - interval '1 month' " \
"ORDER BY published DESC LIMIT $2 OFFSET $3", user.email, limit, offset, as: ChannelVideo)
end
end

View File

@@ -234,10 +234,6 @@ module Invidious::Videos::Parser
keywords = video_details["keywords"]?
.try &.as_a.map &.as_s || [] of String
ai_made = video_primary_renderer
.try &.dig?("badges")
.try { |badges| badges.as_a.any? { |badge| badge.dig?("metadataBadgeRenderer", "label").try(&.as_s) == "AI" } } || false
# Related videos
LOGGER.debug("extract_video_info: parsing related videos...")
@@ -458,7 +454,6 @@ module Invidious::Videos::Parser
"authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""),
"authorVerified" => JSON::Any.new(author_verified || false),
"subCountText" => JSON::Any.new(subs_text || "-"),
"aiMade" => JSON::Any.new(ai_made),
}
return params

View File

@@ -15,7 +15,7 @@ struct Video
# NOTE: don't forget to bump this number if any change is made to
# the `params` structure in videos/parser.cr!!!
#
SCHEMA_VERSION = 4
SCHEMA_VERSION = 3
property id : String
@@ -297,7 +297,6 @@ struct Video
getset_bool authorVerified
getset_bool isFamilyFriendly
getset_bool isListed
getset_bool aiMade
predicate_bool upcoming, isUpcoming
end

View File

@@ -202,11 +202,6 @@ we're going to need to do it here in order to allow for translations.
<p id="views"><i class="icon ion-ios-eye"></i> <%= number_with_separator(video.views) %></p>
<p id="likes"><i class="icon ion-ios-thumbs-up"></i> <%= number_with_separator(video.likes) %></p>
<p id="dislikes" style="display: none; visibility: hidden;"></p>
<% if video.ai_made %>
<p id="ai_made"><%= I18n.translate(locale, "video_details_ai_made") %>
<%= I18n.translate_bool(locale, video.ai_made) %>
</p>
<% end %>
<p id="genre"><%= I18n.translate(locale, "Genre: ") %>
<% if !video.genre_url %>
<%= video.genre %>

View File

@@ -712,17 +712,9 @@ private module Parsers
title = metadata.dig("title", "content").as_s
published = nil
view_count_text = nil
premiere_timestamp = nil
# Contains the views of the video and the published time of the video.
if (metadata_rows = metadata.dig("metadata", "contentMetadataViewModel", "metadataRows").try &.as_a)
metadata_rows.each do |row|
# To filter members only videos, they do not have metadataParts available.
# This videos are not hidden from the channel videos list.
badges = row.dig?("badges").try &.as_a
if badges.try &.any? { |item| item.dig?("badgeViewModel", "badgeStyle").try &.as_s == "BADGE_MEMBERS_ONLY" }
next
end
metadata_parts = row.dig("metadataParts").try &.as_a
if view_count_text.nil?
@@ -733,12 +725,8 @@ private module Parsers
published = metadata_parts.try &.find { |item| item["icon"]?.nil? && item.dig?("text", "content").try &.as_s.includes?("ago") }
.try { |item| decode_date(item.dig("text", "content").as_s) }
end
if premiere_timestamp.nil?
premiere_timestamp = metadata_parts.try &.find { |item| item["icon"]?.nil? && item.dig?("text", "content").try &.as_s.includes?("Premieres") }
.try { |item| decode_date(item.dig("text", "content").as_s) }
end
break if !view_count_text.nil? && !published.nil? && !premiere_timestamp.nil?
break if !view_count_text.nil? && !published.nil?
end
end
@@ -757,7 +745,7 @@ private module Parsers
views: view_count,
description_html: "",
length_seconds: length_seconds || 0,
premiere_timestamp: premiere_timestamp,
premiere_timestamp: Time.unix(0),
author_verified: false,
author_thumbnail: nil,
badges: VideoBadges::None,
@@ -813,13 +801,7 @@ private module Parsers
.compact_map(&.dig?("thumbnailOverlayBadgeViewModel", "thumbnailBadges").try &.as_a)
.flatten
.find(nil, &.dig?("thumbnailBadgeViewModel", "text").try do |node|
# Playlists that contain MORE THAN ONE video, use "videos" for their video count.
# "text" => "8 videos"
#
# Playlists that contain ONLY ONE video, use "video" for their video count.
# "text" => "1 video"
#
{"episodes", "videos", "video", "lessons"}.any? { |str| node.as_s.ends_with?(str) }
{"episodes", "videos", "lessons"}.any? { |str| node.as_s.ends_with?(str) }
end)
.try &.dig("thumbnailBadgeViewModel", "text").as_s.to_i(strict: false)