limit feeds and delete materialized views

config/migrate-scripts/migrate-db-8bc91ce.sh

@@ -0,0 +1,6 @@
+CREATE INDEX channel_videos_ucid_published_idx
+  ON public.channel_videos
+  USING btree
+  (ucid COLLATE pg_catalog."default", published);
+
+DROP INDEX channel_videos_ucid_idx;

config/sql/channel_videos.sql

@@ -19,12 +19,12 @@ CREATE TABLE IF NOT EXISTS public.channel_videos
 
 GRANT ALL ON TABLE public.channel_videos TO current_user;
 
--- Index: public.channel_videos_ucid_idx
+-- Index: public.channel_videos_ucid_published_idx
 
--- DROP INDEX public.channel_videos_ucid_idx;
+-- DROP INDEX public.channel_videos_ucid_published_idx;
 
-CREATE INDEX IF NOT EXISTS channel_videos_ucid_idx
+CREATE INDEX IF NOT EXISTS channel_videos_ucid_published_idx
   ON public.channel_videos
   USING btree
-  (ucid COLLATE pg_catalog."default");
+  (ucid COLLATE pg_catalog."default", published);
 

locales/en-US.json

@@ -507,6 +507,5 @@
     "timeline_parse_error_placeholder_heading": "Unable to parse item",
     "timeline_parse_error_placeholder_message": "Invidious encountered an error while trying to parse this item. For more information see below:",
     "timeline_parse_error_show_technical_details": "Show technical details",
-    "dmca_content": "This video cannot be downloaded on this instance due to a DMCA/copyright infringement letter sent to the instance administrator.",
+    "dmca_content": "This video cannot be downloaded on this instance due to a DMCA/copyright infringement letter sent to the instance administrator."
-    "video_details_ai_made": "AI Made video?:"
 }

src/invidious.cr

@@ -130,14 +130,6 @@ Kemal.config.extra_options do |parser|
       exit
     end
   end
-  parser.on("-f THREADS", "--feed-threads=THREADS", "Number of threads for refreshing feeds (default: #{CONFIG.feed_threads})") do |number|
-    begin
-      CONFIG.feed_threads = number.to_i
-    rescue
-      puts "THREADS must be integer"
-      exit
-    end
-  end
   parser.on("-o OUTPUT", "--output=OUTPUT", "Redirect output (default: #{CONFIG.output})") do |output|
     CONFIG.output = output
   end
@@ -189,10 +181,6 @@ if CONFIG.channel_threads > 0
   Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB)
 end
 
-if CONFIG.feed_threads > 0
-  Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB)
-end
-
 if CONFIG.statistics_enabled
   Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, SOFTWARE)
 end

src/invidious/config.cr

@@ -101,8 +101,6 @@ class Config
   # Time interval between two executions of the job that crawls channel videos (subscriptions update).
   @[YAML::Field(converter: Preferences::TimeSpanConverter)]
   property channel_refresh_interval : Time::Span = 30.minutes
-  # Number of threads to use for updating feeds
-  property feed_threads : Int32 = 1
   # Log file path or STDOUT
   property output : String = "STDOUT"
   # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr
@@ -228,16 +226,7 @@ class Config
     env_config_yaml = "INVIDIOUS_CONFIG"
 
     config_file = ENV.has_key?(env_config_file) ? ENV.fetch(env_config_file) : "config/config.yml"
-    config_yaml = if ENV.has_key?(env_config_yaml)
+    config_yaml = ENV.has_key?(env_config_yaml) ? ENV.fetch(env_config_yaml) : File.read(config_file)
-                    ENV.fetch(env_config_yaml)
-                  else
-                    begin
-                      File.read(config_file)
-                    rescue ex
-                      puts "Config: Failed to load configuration file: #{ex.message}"
-                      exit(1)
-                    end
-                  end
 
     config = Config.from_yaml(config_yaml)
 

src/invidious/jobs/refresh_feeds_job.cr

@@ -1,75 +0,0 @@
-class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
-  private getter db : DB::Database
-
-  def initialize(@db)
-  end
-
-  def begin
-    max_fibers = CONFIG.feed_threads
-    active_fibers = 0
-    active_channel = ::Channel(Bool).new
-
-    loop do
-      db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs|
-        rs.each do
-          email = rs.read(String)
-          view_name = "subscriptions_#{sha256(email)}"
-
-          if active_fibers >= max_fibers
-            if active_channel.receive
-              active_fibers -= 1
-            end
-          end
-
-          active_fibers += 1
-          spawn do
-            begin
-              # Drop outdated views
-              column_array = Invidious::Database.get_column_array(db, view_name)
-              ChannelVideo.type_array.each_with_index do |name, i|
-                if name != column_array[i]?
-                  LOGGER.info("RefreshFeedsJob: DROP MATERIALIZED VIEW #{view_name}")
-                  db.exec("DROP MATERIALIZED VIEW #{view_name}")
-                  raise "view does not exist"
-                end
-              end
-
-              if !db.query_one("SELECT pg_get_viewdef('#{view_name}')", as: String).includes? "WHERE ((cv.ucid = ANY (u.subscriptions))"
-                LOGGER.info("RefreshFeedsJob: Materialized view #{view_name} is out-of-date, recreating...")
-                db.exec("DROP MATERIALIZED VIEW #{view_name}")
-              end
-
-              db.exec("REFRESH MATERIALIZED VIEW #{view_name}")
-              db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
-            rescue ex
-              # Rename old views
-              begin
-                legacy_view_name = "subscriptions_#{sha256(email)[0..7]}"
-
-                db.exec("SELECT * FROM #{legacy_view_name} LIMIT 0")
-                LOGGER.info("RefreshFeedsJob: RENAME MATERIALIZED VIEW #{legacy_view_name}")
-                db.exec("ALTER MATERIALIZED VIEW #{legacy_view_name} RENAME TO #{view_name}")
-              rescue ex
-                begin
-                  # While iterating through, we may have an email stored from a deleted account
-                  if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool)
-                    LOGGER.info("RefreshFeedsJob: CREATE #{view_name}")
-                    db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{Invidious::User::Users::MATERIALIZED_VIEW_SQL.call(email)}")
-                    db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
-                  end
-                rescue ex
-                  LOGGER.error("RefreshFeedJobs: REFRESH #{email} : #{ex.message}")
-                end
-              end
-            end
-
-            active_channel.send(true)
-          end
-        end
-      end
-
-      sleep 5.seconds
-      Fiber.yield
-    end
-  end
-end

src/invidious/jsonify/api_v1/video_json.cr

@@ -64,7 +64,6 @@ module Invidious::JSONify::APIv1
       json.field "liveNow", video.live_now
       json.field "isPostLiveDvr", video.post_live_dvr
       json.field "isUpcoming", video.upcoming?
-      json.field "aiMade", video.ai_made
 
       if video.premiere_timestamp
         json.field "premiereTimestamp", video.premiere_timestamp.try &.to_unix

src/invidious/routes/account.cr

@@ -123,10 +123,8 @@ module Invidious::Routes::Account
       return Errors.error_template(400, ex)
     end
 
-    view_name = "subscriptions_#{sha256(user.email)}"
     Invidious::Database::Users.delete(user)
     Invidious::Database::SessionIDs.delete(email: user.email)
-    PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}")
 
     env.request.cookies.each do |cookie|
       cookie.expires = Time.utc(1990, 1, 1)

src/invidious/routes/login.cr

@@ -125,9 +125,6 @@ module Invidious::Routes::Login
         Invidious::Database::Users.insert(user)
         Invidious::Database::SessionIDs.insert(sid, email)
 
-        view_name = "subscriptions_#{sha256(user.email)}"
-        PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{Invidious::User::Users::MATERIALIZED_VIEW_SQL.call(user.email)}")
-
         if alt = CONFIG.alternative_domains.index(host)
           env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.alternative_domains[alt], sid)
         else

src/invidious/search/processors.cr

@@ -36,19 +36,19 @@ module Invidious::Search::Processors
 
   # Search inside of user subscriptions
   def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo)
-    view_name = "subscriptions_#{sha256(user.email)}"
-
     return PG_DB.query_all(<<-SQL,
       SELECT id,title,published,updated,ucid,author,length_seconds
       FROM (
-        SELECT *,
+        SELECT cv.*,
-        to_tsvector(#{view_name}.title) ||
+        to_tsvector(cv.title) ||
-        to_tsvector(#{view_name}.author)
+        to_tsvector(cv.author) AS document
-        as document
+        FROM channel_videos cv
-        FROM #{view_name}
+        JOIN users ON cv.ucid = any(users.subscriptions)
-      ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;
+        WHERE users.email = $1 AND published > now() - interval '1 month'
+        ORDER BY published
+      ) v_search WHERE v_search.document @@ plainto_tsquery($2) LIMIT 20 OFFSET $3;
       SQL
-      query.text, (query.page - 1) * 20,
+      user.email, query.text, (query.page - 1) * 20,
       as: ChannelVideo
     )
   end

src/invidious/user/users.cr

@@ -2,8 +2,6 @@ require "crypto/bcrypt/password"
 
 module Invidious::User::Users
   extend self
-  # Materialized views may not be defined using bound parameters (`$1` as used elsewhere)
-  MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" }
 
   def create_user(sid, email, password)
     password = Crypto::Bcrypt::Password.create(password, cost: 10)
@@ -29,7 +27,6 @@ module Invidious::User::Users
     offset = (page - 1) * limit
 
     notifications = Invidious::Database::Users.select_notifications(user)
-    view_name = "subscriptions_#{sha256(user.email)}"
 
     if user.preferences.notifications_only && !notifications.empty?
       # Only show notifications
@@ -54,11 +51,19 @@ module Invidious::User::Users
           # Show latest video from a channel that a user hasn't watched
           # "unseen_only" isn't really correct here, more accurate would be "unwatched_only"
 
-          videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY ($1::text[]) ORDER BY ucid, published DESC", user.watched, as: ChannelVideo)
+          videos = PG_DB.query_all("SELECT DISTINCT ON (cv.ucid) cv.* " \
+                                   "FROM channel_videos cv " \
+                                   "JOIN users ON cv.ucid = any(users.subscriptions) " \
+                                   "WHERE users.email = $1 AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' " \
+                                   "ORDER BY ucid, published DESC", user.email, as: ChannelVideo)
         else
           # Show latest video from each channel
 
-          videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo)
+          videos = PG_DB.query_all("SELECT DISTINCT ON (cv.ucid) cv.* " \
+                                   "FROM channel_videos cv " \
+                                   "JOIN users ON cv.ucid = any(users.subscriptions) " \
+                                   "WHERE users.email = $1 AND published > now() - interval '1 month' " \
+                                   "ORDER BY ucid, published DESC", user.email, as: ChannelVideo)
         end
 
         videos.sort_by!(&.published).reverse!
@@ -66,11 +71,19 @@ module Invidious::User::Users
         if user.preferences.unseen_only
           # Only show unwatched
 
-          videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY ($1::text[]) ORDER BY published DESC LIMIT $2 OFFSET $3", user.watched, limit, offset, as: ChannelVideo)
+          videos = PG_DB.query_all("SELECT cv.* " \
+                                   "FROM channel_videos cv " \
+                                   "JOIN users ON cv.ucid = any(users.subscriptions) " \
+                                   "WHERE users.email = $1 AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' " \
+                                   "ORDER BY published DESC LIMIT $2 OFFSET $3", user.email, limit, offset, as: ChannelVideo)
         else
           # Sort subscriptions as normal
 
-          videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
+          videos = PG_DB.query_all("SELECT cv.* " \
+                                   "FROM channel_videos cv " \
+                                   "JOIN users ON cv.ucid = any(users.subscriptions) " \
+                                   "WHERE users.email = $1 AND published > now() - interval '1 month' " \
+                                   "ORDER BY published DESC LIMIT $2 OFFSET $3", user.email, limit, offset, as: ChannelVideo)
         end
       end
 

src/invidious/videos/parser.cr

@@ -234,10 +234,6 @@ module Invidious::Videos::Parser
     keywords = video_details["keywords"]?
       .try &.as_a.map &.as_s || [] of String
 
-    ai_made = video_primary_renderer
-      .try &.dig?("badges")
-        .try { |badges| badges.as_a.any? { |badge| badge.dig?("metadataBadgeRenderer", "label").try(&.as_s) == "AI" } } || false
-
     # Related videos
 
     LOGGER.debug("extract_video_info: parsing related videos...")
@@ -458,7 +454,6 @@ module Invidious::Videos::Parser
       "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""),
       "authorVerified"  => JSON::Any.new(author_verified || false),
       "subCountText"    => JSON::Any.new(subs_text || "-"),
-      "aiMade"          => JSON::Any.new(ai_made),
     }
 
     return params

src/invidious/videos/videos.cr

@@ -15,7 +15,7 @@ struct Video
   # NOTE: don't forget to bump this number if any change is made to
   # the `params` structure in videos/parser.cr!!!
   #
-  SCHEMA_VERSION = 4
+  SCHEMA_VERSION = 3
 
   property id : String
 
@@ -297,7 +297,6 @@ struct Video
   getset_bool authorVerified
   getset_bool isFamilyFriendly
   getset_bool isListed
-  getset_bool aiMade
   predicate_bool upcoming, isUpcoming
 end
 

src/invidious/views/watch.ecr

@@ -202,11 +202,6 @@ we're going to need to do it here in order to allow for translations.
             <p id="views"><i class="icon ion-ios-eye"></i> <%= number_with_separator(video.views) %></p>
             <p id="likes"><i class="icon ion-ios-thumbs-up"></i> <%= number_with_separator(video.likes) %></p>
             <p id="dislikes" style="display: none; visibility: hidden;"></p>
-            <% if video.ai_made %>
-                <p id="ai_made"><%= I18n.translate(locale, "video_details_ai_made") %>
-                    <%= I18n.translate_bool(locale, video.ai_made) %>
-                </p>
-            <% end %>
             <p id="genre"><%= I18n.translate(locale, "Genre: ") %>
                 <% if !video.genre_url %>
                     <%= video.genre %>

src/invidious/yt_backend/extractors.cr

@@ -715,13 +715,6 @@ private module Parsers
         # Contains the views of the video and the published time of the video.
         if (metadata_rows = metadata.dig("metadata", "contentMetadataViewModel", "metadataRows").try &.as_a)
           metadata_rows.each do |row|
-            # To filter members only videos, they do not have metadataParts available.
-            # This videos are not hidden from the channel videos list.
-            badges = row.dig?("badges").try &.as_a
-            if badges.try &.any? { |item| item.dig?("badgeViewModel", "badgeStyle").try &.as_s == "BADGE_MEMBERS_ONLY" }
-              next
-            end
-
             metadata_parts = row.dig("metadataParts").try &.as_a
 
             if view_count_text.nil?