From 8ec992a8a31742a82de38a0aa5eeb509362da9b4 Mon Sep 17 00:00:00 2001 From: matthewmcgarvey Date: Wed, 9 Feb 2022 00:50:32 -0600 Subject: [PATCH] Add custom migration implementation --- src/invidious.cr | 3 ++ src/invidious/migration.cr | 38 +++++++++++++++ .../migrations/0000_create_channels_table.cr | 30 ++++++++++++ .../migrations/0001_create_videos_table.cr | 28 +++++++++++ .../0002_create_channel_videos_table.cr | 35 ++++++++++++++ .../migrations/0003_create_users_table.cr | 34 ++++++++++++++ .../0004_create_session_ids_table.cr | 28 +++++++++++ .../migrations/0005_create_nonces_table.cr | 27 +++++++++++ .../0006_create_annotations_table.cr | 20 ++++++++ .../migrations/0007_create_playlists_table.cr | 47 +++++++++++++++++++ .../0008_create_playlist_videos_table.cr | 27 +++++++++++ src/invidious/migrator.cr | 41 ++++++++++++++++ 12 files changed, 358 insertions(+) create mode 100644 src/invidious/migration.cr create mode 100644 src/invidious/migrations/0000_create_channels_table.cr create mode 100644 src/invidious/migrations/0001_create_videos_table.cr create mode 100644 src/invidious/migrations/0002_create_channel_videos_table.cr create mode 100644 src/invidious/migrations/0003_create_users_table.cr create mode 100644 src/invidious/migrations/0004_create_session_ids_table.cr create mode 100644 src/invidious/migrations/0005_create_nonces_table.cr create mode 100644 src/invidious/migrations/0006_create_annotations_table.cr create mode 100644 src/invidious/migrations/0007_create_playlists_table.cr create mode 100644 src/invidious/migrations/0008_create_playlist_videos_table.cr create mode 100644 src/invidious/migrator.cr diff --git a/src/invidious.cr b/src/invidious.cr index 1ff70905..6ec5f3a5 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -34,6 +34,7 @@ require "./invidious/channels/*" require "./invidious/user/*" require "./invidious/routes/**" require "./invidious/jobs/**" +require "./invidious/migrations/*" CONFIG = Config.load HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32) @@ -111,6 +112,8 @@ end OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mode: "a") LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) +# Run migrations +Invidious::Migrator.new(PG_DB).migrate # Check table integrity Invidious::Database.check_integrity(CONFIG) diff --git a/src/invidious/migration.cr b/src/invidious/migration.cr new file mode 100644 index 00000000..a4eec1c5 --- /dev/null +++ b/src/invidious/migration.cr @@ -0,0 +1,38 @@ +abstract class Invidious::Migration + macro inherited + Invidious::Migrator.migrations << self + end + + @@version : Int64? + + def self.version(version : Int32 | Int64) + @@version = version.to_i64 + end + + getter? completed = false + + def initialize(@db : DB::Database) + end + + abstract def up(conn : DB::Connection) + + def migrate + # migrator already ignores completed migrations + # but this is an extra check to make sure a migration doesn't run twice + return if completed? + + @db.transaction do |txn| + up(txn.connection) + track(txn.connection) + @completed = true + end + end + + def version : Int64 + @@version.not_nil! + end + + private def track(conn : DB::Connection) + conn.exec("INSERT INTO #{Invidious::Migrator::MIGRATIONS_TABLE}(version) VALUES ($1)", version) + end +end diff --git a/src/invidious/migrations/0000_create_channels_table.cr b/src/invidious/migrations/0000_create_channels_table.cr new file mode 100644 index 00000000..1f8f18e2 --- /dev/null +++ b/src/invidious/migrations/0000_create_channels_table.cr @@ -0,0 +1,30 @@ +module Invidious::Migrations + class CreateChannelsTable < Migration + version 0 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.channels + ( + id text NOT NULL, + author text, + updated timestamp with time zone, + deleted boolean, + subscribed timestamp with time zone, + CONSTRAINT channels_id_key UNIQUE (id) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.channels TO current_user; + SQL + + conn.exec <<-SQL + CREATE INDEX IF NOT EXISTS channels_id_idx + ON public.channels + USING btree + (id COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/migrations/0001_create_videos_table.cr b/src/invidious/migrations/0001_create_videos_table.cr new file mode 100644 index 00000000..cdc9993f --- /dev/null +++ b/src/invidious/migrations/0001_create_videos_table.cr @@ -0,0 +1,28 @@ +module Invidious::Migrations + class CreateVideosTable < Migration + version 1 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE UNLOGGED TABLE IF NOT EXISTS public.videos + ( + id text NOT NULL, + info text, + updated timestamp with time zone, + CONSTRAINT videos_pkey PRIMARY KEY (id) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.videos TO current_user; + SQL + + conn.exec <<-SQL + CREATE UNIQUE INDEX IF NOT EXISTS id_idx + ON public.videos + USING btree + (id COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/migrations/0002_create_channel_videos_table.cr b/src/invidious/migrations/0002_create_channel_videos_table.cr new file mode 100644 index 00000000..737abad4 --- /dev/null +++ b/src/invidious/migrations/0002_create_channel_videos_table.cr @@ -0,0 +1,35 @@ +module Invidious::Migrations + class CreateChannelVideosTable < Migration + version 2 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.channel_videos + ( + id text NOT NULL, + title text, + published timestamp with time zone, + updated timestamp with time zone, + ucid text, + author text, + length_seconds integer, + live_now boolean, + premiere_timestamp timestamp with time zone, + views bigint, + CONSTRAINT channel_videos_id_key UNIQUE (id) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.channel_videos TO current_user; + SQL + + conn.exec <<-SQL + CREATE INDEX IF NOT EXISTS channel_videos_ucid_idx + ON public.channel_videos + USING btree + (ucid COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/migrations/0003_create_users_table.cr b/src/invidious/migrations/0003_create_users_table.cr new file mode 100644 index 00000000..d91cca8d --- /dev/null +++ b/src/invidious/migrations/0003_create_users_table.cr @@ -0,0 +1,34 @@ +module Invidious::Migrations + class CreateUsersTable < Migration + version 3 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.users + ( + updated timestamp with time zone, + notifications text[], + subscriptions text[], + email text NOT NULL, + preferences text, + password text, + token text, + watched text[], + feed_needs_update boolean, + CONSTRAINT users_email_key UNIQUE (email) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.users TO current_user; + SQL + + conn.exec <<-SQL + CREATE UNIQUE INDEX IF NOT EXISTS email_unique_idx + ON public.users + USING btree + (lower(email) COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/migrations/0004_create_session_ids_table.cr b/src/invidious/migrations/0004_create_session_ids_table.cr new file mode 100644 index 00000000..9ef00f78 --- /dev/null +++ b/src/invidious/migrations/0004_create_session_ids_table.cr @@ -0,0 +1,28 @@ +module Invidious::Migrations + class CreateSessionIdsTable < Migration + version 4 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.session_ids + ( + id text NOT NULL, + email text, + issued timestamp with time zone, + CONSTRAINT session_ids_pkey PRIMARY KEY (id) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.session_ids TO current_user; + SQL + + conn.exec <<-SQL + CREATE INDEX IF NOT EXISTS session_ids_id_idx + ON public.session_ids + USING btree + (id COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/migrations/0005_create_nonces_table.cr b/src/invidious/migrations/0005_create_nonces_table.cr new file mode 100644 index 00000000..4b1220e6 --- /dev/null +++ b/src/invidious/migrations/0005_create_nonces_table.cr @@ -0,0 +1,27 @@ +module Invidious::Migrations + class CreateNoncesTable < Migration + version 5 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.nonces + ( + nonce text, + expire timestamp with time zone, + CONSTRAINT nonces_id_key UNIQUE (nonce) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.nonces TO current_user; + SQL + + conn.exec <<-SQL + CREATE INDEX IF NOT EXISTS nonces_nonce_idx + ON public.nonces + USING btree + (nonce COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/migrations/0006_create_annotations_table.cr b/src/invidious/migrations/0006_create_annotations_table.cr new file mode 100644 index 00000000..86f21dd9 --- /dev/null +++ b/src/invidious/migrations/0006_create_annotations_table.cr @@ -0,0 +1,20 @@ +module Invidious::Migrations + class CreateAnnotationsTable < Migration + version 6 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.annotations + ( + id text NOT NULL, + annotations xml, + CONSTRAINT annotations_id_key UNIQUE (id) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.annotations TO current_user; + SQL + end + end +end diff --git a/src/invidious/migrations/0007_create_playlists_table.cr b/src/invidious/migrations/0007_create_playlists_table.cr new file mode 100644 index 00000000..81217365 --- /dev/null +++ b/src/invidious/migrations/0007_create_playlists_table.cr @@ -0,0 +1,47 @@ +module Invidious::Migrations + class CreatePlaylistsTable < Migration + version 7 + + def up(conn : DB::Connection) + conn.exec <<-SQL + DO + $$ + BEGIN + IF NOT EXISTS (SELECT * + FROM pg_type typ + INNER JOIN pg_namespace nsp ON nsp.oid = typ.typnamespace + WHERE nsp.nspname = 'public' + AND typ.typname = 'privacy') THEN + CREATE TYPE public.privacy AS ENUM + ( + 'Public', + 'Unlisted', + 'Private' + ); + END IF; + END; + $$ + LANGUAGE plpgsql; + SQL + + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.playlists + ( + title text, + id text primary key, + author text, + description text, + video_count integer, + created timestamptz, + updated timestamptz, + privacy privacy, + index int8[] + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON public.playlists TO current_user; + SQL + end + end +end diff --git a/src/invidious/migrations/0008_create_playlist_videos_table.cr b/src/invidious/migrations/0008_create_playlist_videos_table.cr new file mode 100644 index 00000000..80fa6b5f --- /dev/null +++ b/src/invidious/migrations/0008_create_playlist_videos_table.cr @@ -0,0 +1,27 @@ +module Invidious::Migrations + class CreatePlaylistVideosTable < Migration + version 8 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.playlist_videos + ( + title text, + id text, + author text, + ucid text, + length_seconds integer, + published timestamptz, + plid text references playlists(id), + index int8, + live_now boolean, + PRIMARY KEY (index,plid) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.playlist_videos TO current_user; + SQL + end + end +end diff --git a/src/invidious/migrator.cr b/src/invidious/migrator.cr new file mode 100644 index 00000000..dc6880b9 --- /dev/null +++ b/src/invidious/migrator.cr @@ -0,0 +1,41 @@ +class Invidious::Migrator + MIGRATIONS_TABLE = "invidious_migrations" + + class_getter migrations = [] of Invidious::Migration.class + + def initialize(@db : DB::Database) + end + + def migrate + run_migrations = load_run_migrations + migrations = load_migrations.sort_by(&.version) + migrations_to_run = migrations.reject { |migration| run_migrations.includes?(migration.version) } + if migrations.empty? + puts "No migrations to run." + return + end + + migrations_to_run.each do |migration| + puts "Running migration: #{migration.class.name}" + migration.migrate + end + end + + private def load_migrations : Array(Invidious::Migration) + self.class.migrations.map(&.new(@db)) + end + + private def load_run_migrations : Array(Int64) + create_migrations_table + @db.query_all("SELECT version FROM #{MIGRATIONS_TABLE}", as: Int64) + end + + private def create_migrations_table + @db.exec <<-SQL + CREATE TABLE IF NOT EXISTS #{MIGRATIONS_TABLE} ( + id bigserial PRIMARY KEY, + version bigint NOT NULL + ) + SQL + end +end