Fix warnings with latest version of Crystal

src/invidious.cr

@@ -1203,7 +1203,7 @@ post "/playlist_ajax" do |env|
       end
     end
 
-    playlist_video = PlaylistVideo.new(
+    playlist_video = PlaylistVideo.new({
       title:          video.title,
       id:             video.id,
       author:         video.author,
@@ -1212,8 +1212,8 @@ post "/playlist_ajax" do |env|
       published:      video.published,
       plid:           playlist_id,
       live_now:       video.live_now,
-      index: Random::Secure.rand(0_i64..Int64::MAX)
+      index:          Random::Secure.rand(0_i64..Int64::MAX),
-    )
+    })
 
     video_array = playlist_video.to_a
     args = arg_array(video_array)
@@ -1839,8 +1839,8 @@ post "/login" do |env|
       sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
       user, sid = create_user(sid, email, password)
       user_array = user.to_a
+      user_array[4] = user_array[4].to_json # User preferences
 
-      user_array[4] = user_array[4].to_json
       args = arg_array(user_array)
 
       PG_DB.exec("INSERT INTO users VALUES (#{args})", args: user_array)
@@ -2519,7 +2519,7 @@ post "/data_control" do |env|
   if user
     user = user.as(User)
 
-    # TODO: Find better way to prevent timeout
+    # TODO: Find a way to prevent browser timeout
 
     HTTP::FormData.parse(env.request) do |part|
       body = part.body.gets_to_end
@@ -2546,7 +2546,7 @@ post "/data_control" do |env|
         end
 
         if body["preferences"]?
-          user.preferences = Preferences.from_json(body["preferences"].to_json, user.preferences)
+          user.preferences = Preferences.from_json(body["preferences"].to_json)
           PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", user.preferences.to_json, user.email)
         end
 
@@ -2573,7 +2573,7 @@ post "/data_control" do |env|
                 next
               end
 
-              playlist_video = PlaylistVideo.new(
+              playlist_video = PlaylistVideo.new({
                 title:          video.title,
                 id:             video.id,
                 author:         video.author,
@@ -2582,8 +2582,8 @@ post "/data_control" do |env|
                 published:      video.published,
                 plid:           playlist.id,
                 live_now:       video.live_now,
-                index: Random::Secure.rand(0_i64..Int64::MAX)
+                index:          Random::Secure.rand(0_i64..Int64::MAX),
-              )
+              })
 
               video_array = playlist_video.to_a
               args = arg_array(video_array)
@@ -3154,7 +3154,7 @@ get "/feed/channel/:ucid" do |env|
     description_html = entry.xpath_node("group/description").not_nil!.to_s
     views = entry.xpath_node("group/community/statistics").not_nil!.["views"].to_i64
 
-    SearchVideo.new(
+    SearchVideo.new({
       title:              title,
       id:                 video_id,
       author:             author,
@@ -3166,8 +3166,8 @@ get "/feed/channel/:ucid" do |env|
       live_now:           false,
       paid:               false,
       premium:            false,
-      premiere_timestamp: nil
+      premiere_timestamp: nil,
-    )
+    })
   end
 
   XML.build(indent: "  ", encoding: "UTF-8") do |xml|
@@ -3397,7 +3397,7 @@ post "/feed/webhook/:token" do |env|
       }.to_json
       PG_DB.exec("NOTIFY notifications, E'#{payload}'")
 
-      video = ChannelVideo.new(
+      video = ChannelVideo.new({
         id:                 id,
         title:              video.title,
         published:          published,
@@ -3408,7 +3408,7 @@ post "/feed/webhook/:token" do |env|
         live_now:           video.live_now,
         premiere_timestamp: video.premiere_timestamp,
         views:              video.views,
-      )
+      })
 
       PG_DB.query_all("UPDATE users SET feed_needs_update = true, notifications = array_append(notifications, $1) \
         WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)",
@@ -4666,7 +4666,7 @@ post "/api/v1/auth/preferences" do |env|
   user = env.get("user").as(User)
 
   begin
-    preferences = Preferences.from_json(env.request.body || "{}", user.preferences)
+    preferences = Preferences.from_json(env.request.body || "{}")
   rescue
     preferences = user.preferences
   end
@@ -4920,7 +4920,7 @@ post "/api/v1/auth/playlists/:plid/videos" do |env|
     next error_message
   end
 
-  playlist_video = PlaylistVideo.new(
+  playlist_video = PlaylistVideo.new({
     title:          video.title,
     id:             video.id,
     author:         video.author,
@@ -4929,8 +4929,8 @@ post "/api/v1/auth/playlists/:plid/videos" do |env|
     published:      video.published,
     plid:           plid,
     live_now:       video.live_now,
-    index: Random::Secure.rand(0_i64..Int64::MAX)
+    index:          Random::Secure.rand(0_i64..Int64::MAX),
-  )
+  })
 
   video_array = playlist_video.to_a
   args = arg_array(video_array)

src/invidious/channels.cr

@@ -1,14 +1,27 @@
 struct InvidiousChannel
-  db_mapping({
+  include DB::Serializable
-    id:         String,
+
-    author:     String,
+  property id : String
-    updated:    Time,
+  property author : String
-    deleted:    Bool,
+  property updated : Time
-    subscribed: Time?,
+  property deleted : Bool
-  })
+  property subscribed : Time?
 end
 
 struct ChannelVideo
+  include DB::Serializable
+
+  property id : String
+  property title : String
+  property published : Time
+  property updated : Time
+  property ucid : String
+  property author : String
+  property length_seconds : Int32 = 0
+  property live_now : Bool = false
+  property premiere_timestamp : Time? = nil
+  property views : Int64? = nil
+
   def to_json(locale, json : JSON::Builder)
     json.object do
       json.field "type", "shortVideo"
@@ -84,49 +97,36 @@ struct ChannelVideo
       end
     end
   end
-
-  db_mapping({
-    id:                 String,
-    title:              String,
-    published:          Time,
-    updated:            Time,
-    ucid:               String,
-    author:             String,
-    length_seconds:     {type: Int32, default: 0},
-    live_now:           {type: Bool, default: false},
-    premiere_timestamp: {type: Time?, default: nil},
-    views:              {type: Int64?, default: nil},
-  })
 end
 
 struct AboutRelatedChannel
-  db_mapping({
+  include DB::Serializable
-    ucid:             String,
+
-    author:           String,
+  property ucid : String
-    author_url:       String,
+  property author : String
-    author_thumbnail: String,
+  property author_url : String
-  })
+  property author_thumbnail : String
 end
 
 # TODO: Refactor into either SearchChannel or InvidiousChannel
 struct AboutChannel
-  db_mapping({
+  include DB::Serializable
-    ucid:               String,
+
-    author:             String,
+  property ucid : String
-    auto_generated:     Bool,
+  property author : String
-    author_url:         String,
+  property auto_generated : Bool
-    author_thumbnail:   String,
+  property author_url : String
-    banner:             String?,
+  property author_thumbnail : String
-    description_html:   String,
+  property banner : String?
-    paid:               Bool,
+  property description_html : String
-    total_views:        Int64,
+  property paid : Bool
-    sub_count:          Int32,
+  property total_views : Int64
-    joined:             Time,
+  property sub_count : Int32
-    is_family_friendly: Bool,
+  property joined : Time
-    allowed_regions:    Array(String),
+  property is_family_friendly : Bool
-    related_channels:   Array(AboutRelatedChannel),
+  property allowed_regions : Array(String)
-    tabs:               Array(String),
+  property related_channels : Array(AboutRelatedChannel)
-  })
+  property tabs : Array(String)
 end
 
 class ChannelRedirect < Exception
@@ -248,7 +248,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
 
     premiere_timestamp = channel_video.try &.premiere_timestamp
 
-    video = ChannelVideo.new(
+    video = ChannelVideo.new({
       id:                 video_id,
       title:              title,
       published:          published,
@@ -259,7 +259,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
       live_now:           live_now,
       premiere_timestamp: premiere_timestamp,
       views:              views,
-    )
+    })
 
     emails = db.query_all("UPDATE users SET notifications = array_append(notifications, $1) \
       WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications) RETURNING email",
@@ -298,7 +298,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
       videos = extract_videos(initial_data.as_h, author, ucid)
 
       count = videos.size
-      videos = videos.map { |video| ChannelVideo.new(
+      videos = videos.map { |video| ChannelVideo.new({
         id:                 video.id,
         title:              video.title,
         published:          video.published,
@@ -308,8 +308,8 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
         length_seconds:     video.length_seconds,
         live_now:           video.live_now,
         premiere_timestamp: video.premiere_timestamp,
-        views: video.views
+        views:              video.views,
-      ) }
+      }) }
 
       videos.each do |video|
         ids << video.id
@@ -352,7 +352,13 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
     db.exec("DELETE FROM channel_videos * WHERE NOT id = ANY ('{#{ids.map { |id| %("#{id}") }.join(",")}}') AND ucid = $1", ucid)
   end
 
-  channel = InvidiousChannel.new(ucid, author, Time.utc, false, nil)
+  channel = InvidiousChannel.new({
+    id:         ucid,
+    author:     author,
+    updated:    Time.utc,
+    deleted:    false,
+    subscribed: nil,
+  })
 
   return channel
 end
@@ -396,11 +402,11 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "
       "2:string" => ucid,
       "3:base64" => {
         "2:string"  => "videos",
-        "6:varint":   2_i64,
+        "6:varint"  => 2_i64,
-        "7:varint":   1_i64,
+        "7:varint"  => 1_i64,
-        "12:varint":  1_i64,
+        "12:varint" => 1_i64,
-        "13:string":  "",
+        "13:string" => "",
-        "23:varint":  0_i64,
+        "23:varint" => 0_i64,
       },
     },
   }
@@ -445,11 +451,11 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated
       "2:string" => ucid,
       "3:base64" => {
         "2:string"  => "playlists",
-        "6:varint":   2_i64,
+        "6:varint"  => 2_i64,
-        "7:varint":   1_i64,
+        "7:varint"  => 1_i64,
-        "12:varint":  1_i64,
+        "12:varint" => 1_i64,
-        "13:string":  "",
+        "13:string" => "",
-        "23:varint":  0_i64,
+        "23:varint" => 0_i64,
       },
     },
   }
@@ -849,12 +855,12 @@ def get_about_info(ucid, locale)
     related_author_thumbnail = node.xpath_node(%q(.//img)).try &.["data-thumb"]
     related_author_thumbnail ||= ""
 
-    AboutRelatedChannel.new(
+    AboutRelatedChannel.new({
       ucid:             related_id,
       author:           related_title,
       author_url:       related_author_url,
       author_thumbnail: related_author_thumbnail,
-    )
+    })
   end
 
   joined = about.xpath_node(%q(//span[contains(., "Joined")]))
@@ -876,7 +882,7 @@ def get_about_info(ucid, locale)
 
   tabs = about.xpath_nodes(%q(//ul[@id="channel-navigation-menu"]/li/a/span)).map { |node| node.content.downcase }
 
-  AboutChannel.new(
+  AboutChannel.new({
     ucid:               ucid,
     author:             author,
     auto_generated:     auto_generated,
@@ -891,8 +897,8 @@ def get_about_info(ucid, locale)
     is_family_friendly: is_family_friendly,
     allowed_regions:    allowed_regions,
     related_channels:   related_channels,
-    tabs: tabs
+    tabs:               tabs,
-  )
+  })
 end
 
 def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")

src/invidious/comments.cr

@@ -1,11 +1,23 @@
 class RedditThing
-  JSON.mapping({
+  include JSON::Serializable
-    kind: String,
+
-    data: RedditComment | RedditLink | RedditMore | RedditListing,
+  property kind : String
-  })
+  property data : RedditComment | RedditLink | RedditMore | RedditListing
 end
 
 class RedditComment
+  include JSON::Serializable
+
+  property author : String
+  property body_html : String
+  property replies : RedditThing | String
+  property score : Int32
+  property depth : Int32
+  property permalink : String
+
+  @[JSON::Field(converter: RedditComment::TimeConverter)]
+  property created_utc : Time
+
   module TimeConverter
     def self.from_json(value : JSON::PullParser) : Time
       Time.unix(value.read_float.to_i)
@@ -15,46 +27,33 @@ class RedditComment
       json.number(value.to_unix)
     end
   end
-
-  JSON.mapping({
-    author:      String,
-    body_html:   String,
-    replies:     RedditThing | String,
-    score:       Int32,
-    depth:       Int32,
-    permalink:   String,
-    created_utc: {
-      type:      Time,
-      converter: RedditComment::TimeConverter,
-    },
-  })
 end
 
 struct RedditLink
-  JSON.mapping({
+  include JSON::Serializable
-    author:       String,
+
-    score:        Int32,
+  property author : String
-    subreddit:    String,
+  property score : Int32
-    num_comments: Int32,
+  property subreddit : String
-    id:           String,
+  property num_comments : Int32
-    permalink:    String,
+  property id : String
-    title:        String,
+  property permalink : String
-  })
+  property title : String
 end
 
 struct RedditMore
-  JSON.mapping({
+  include JSON::Serializable
-    children: Array(String),
+
-    count:    Int32,
+  property children : Array(String)
-    depth:    Int32,
+  property count : Int32
-  })
+  property depth : Int32
 end
 
 class RedditListing
-  JSON.mapping({
+  include JSON::Serializable
-    children: Array(RedditThing),
+
-    modhash:  String,
+  property children : Array(RedditThing)
-  })
+  property modhash : String
 end
 
 def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, sort_by = "top")

src/invidious/helpers/helpers.cr

@@ -1,219 +1,100 @@
 require "./macros"
 
 struct Nonce
-  db_mapping({
+  include DB::Serializable
-    nonce:  String,
+
-    expire: Time,
+  property nonce : String
-  })
+  property expire : Time
 end
 
 struct SessionId
-  db_mapping({
+  include DB::Serializable
-    id:     String,
+
-    email:  String,
+  property id : String
-    issued: String,
+  property email : String
-  })
+  property issued : String
 end
 
 struct Annotation
-  db_mapping({
+  include DB::Serializable
-    id:          String,
+
-    annotations: String,
+  property id : String
-  })
+  property annotations : String
 end
 
 struct ConfigPreferences
-  module StringToArray
+  include YAML::Serializable
-    def self.to_json(value : Array(String), json : JSON::Builder)
-      json.array do
-        value.each do |element|
-          json.string element
-        end
-      end
-    end
 
-    def self.from_json(value : JSON::PullParser) : Array(String)
+  property annotations : Bool = false
-      begin
+  property annotations_subscribed : Bool = false
-        result = [] of String
+  property autoplay : Bool = false
-        value.read_array do
+  property captions : Array(String) = ["", "", ""]
-          result << HTML.escape(value.read_string[0, 100])
+  property comments : Array(String) = ["youtube", ""]
-        end
+  property continue : Bool = false
-      rescue ex
+  property continue_autoplay : Bool = true
-        result = [HTML.escape(value.read_string[0, 100]), ""]
+  property dark_mode : String = ""
-      end
+  property latest_only : Bool = false
+  property listen : Bool = false
+  property local : Bool = false
+  property locale : String = "en-US"
+  property max_results : Int32 = 40
+  property notifications_only : Bool = false
+  property player_style : String = "invidious"
+  property quality : String = "hd720"
+  property default_home : String = "Popular"
+  property feed_menu : Array(String) = ["Popular", "Trending", "Subscriptions", "Playlists"]
+  property related_videos : Bool = true
+  property sort : String = "published"
+  property speed : Float32 = 1.0_f32
+  property thin_mode : Bool = false
+  property unseen_only : Bool = false
+  property video_loop : Bool = false
+  property volume : Int32 = 100
 
-      result
+  def to_tuple
+    {% begin %}
+      {
+        {{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}}
+      }
+    {% end %}
   end
-
-    def self.to_yaml(value : Array(String), yaml : YAML::Nodes::Builder)
-      yaml.sequence do
-        value.each do |element|
-          yaml.scalar element
-        end
-      end
-    end
-
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(String)
-      begin
-        unless node.is_a?(YAML::Nodes::Sequence)
-          node.raise "Expected sequence, not #{node.class}"
-        end
-
-        result = [] of String
-        node.nodes.each do |item|
-          unless item.is_a?(YAML::Nodes::Scalar)
-            node.raise "Expected scalar, not #{item.class}"
-          end
-
-          result << HTML.escape(item.value[0, 100])
-        end
-      rescue ex
-        if node.is_a?(YAML::Nodes::Scalar)
-          result = [HTML.escape(node.value[0, 100]), ""]
-        else
-          result = ["", ""]
-        end
-      end
-
-      result
-    end
-  end
-
-  module BoolToString
-    def self.to_json(value : String, json : JSON::Builder)
-      json.string value
-    end
-
-    def self.from_json(value : JSON::PullParser) : String
-      begin
-        result = value.read_string
-
-        if result.empty?
-          CONFIG.default_user_preferences.dark_mode
-        else
-          result
-        end
-      rescue ex
-        if value.read_bool
-          "dark"
-        else
-          "light"
-        end
-      end
-    end
-
-    def self.to_yaml(value : String, yaml : YAML::Nodes::Builder)
-      yaml.scalar value
-    end
-
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String
-      unless node.is_a?(YAML::Nodes::Scalar)
-        node.raise "Expected scalar, not #{node.class}"
-      end
-
-      case node.value
-      when "true"
-        "dark"
-      when "false"
-        "light"
-      when ""
-        CONFIG.default_user_preferences.dark_mode
-      else
-        node.value
-      end
-    end
-  end
-
-  yaml_mapping({
-    annotations:            {type: Bool, default: false},
-    annotations_subscribed: {type: Bool, default: false},
-    autoplay:               {type: Bool, default: false},
-    captions:               {type: Array(String), default: ["", "", ""], converter: StringToArray},
-    comments:               {type: Array(String), default: ["youtube", ""], converter: StringToArray},
-    continue:               {type: Bool, default: false},
-    continue_autoplay:      {type: Bool, default: true},
-    dark_mode:              {type: String, default: "", converter: BoolToString},
-    latest_only:            {type: Bool, default: false},
-    listen:                 {type: Bool, default: false},
-    local:                  {type: Bool, default: false},
-    locale:                 {type: String, default: "en-US"},
-    max_results:            {type: Int32, default: 40},
-    notifications_only:     {type: Bool, default: false},
-    player_style:           {type: String, default: "invidious"},
-    quality:                {type: String, default: "hd720"},
-    default_home:           {type: String, default: "Popular"},
-    feed_menu:              {type: Array(String), default: ["Popular", "Trending", "Subscriptions", "Playlists"]},
-    related_videos:         {type: Bool, default: true},
-    sort:                   {type: String, default: "published"},
-    speed:                  {type: Float32, default: 1.0_f32},
-    thin_mode:              {type: Bool, default: false},
-    unseen_only:            {type: Bool, default: false},
-    video_loop:             {type: Bool, default: false},
-    volume:                 {type: Int32, default: 100},
-  })
 end
 
 struct Config
-  module ConfigPreferencesConverter
+  include YAML::Serializable
-    def self.to_yaml(value : Preferences, yaml : YAML::Nodes::Builder)
-      value.to_yaml(yaml)
-    end
 
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Preferences
+  property channel_threads : Int32                 # Number of threads to use for crawling videos from channels (for updating subscriptions)
-      Preferences.new(*ConfigPreferences.new(ctx, node).to_tuple)
+  property feed_threads : Int32                    # Number of threads to use for updating feeds
-    end
+  property db : DBConfig                           # Database configuration
-  end
+  property full_refresh : Bool                     # Used for crawling channels: threads should check all videos uploaded by a channel
+  property https_only : Bool?                      # Used to tell Invidious it is behind a proxy, so links to resources should be https://
+  property hmac_key : String?                      # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
+  property domain : String?                        # Domain to be used for links to resources on the site where an absolute URL is required
+  property use_pubsub_feeds : Bool | Int32 = false # Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
+  property captcha_enabled : Bool = true
+  property login_enabled : Bool = true
+  property registration_enabled : Bool = true
+  property statistics_enabled : Bool = false
+  property admins : Array(String) = [] of String
+  property external_port : Int32? = nil
+  property default_user_preferences : ConfigPreferences
+  property dmca_content : Array(String) = [] of String    # For compliance with DMCA, disables download widget using list of video IDs
+  property check_tables : Bool = false                    # Check table integrity, automatically try to add any missing columns, create tables, etc.
+  property cache_annotations : Bool = false               # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards
+  property banner : String? = nil                         # Optional banner to be displayed along top of page for announcements, etc.
+  property hsts : Bool? = true                            # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely
+  property disable_proxy : Bool? | Array(String)? = false # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local'
 
-  module FamilyConverter
+  @[YAML::Field(converter: Preferences::FamilyConverter)]
-    def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder)
+  property force_resolve : Socket::Family = Socket::Family::UNSPEC # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
-      case value
+  property port : Int32 = 3000                                     # Port to listen for connections (overrided by command line argument)
-      when Socket::Family::UNSPEC
+  property host_binding : String = "0.0.0.0"                       # Host to bind (overrided by command line argument)
-        yaml.scalar nil
+  property pool_size : Int32 = 100                                 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
-      when Socket::Family::INET
+  property admin_email : String = "omarroth@protonmail.com"        # Email for bug reports
-        yaml.scalar "ipv4"
-      when Socket::Family::INET6
-        yaml.scalar "ipv6"
-      when Socket::Family::UNIX
-        raise "Invalid socket family #{value}"
-      end
-    end
 
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family
+  @[YAML::Field(converter: Preferences::StringToCookies)]
-      if node.is_a?(YAML::Nodes::Scalar)
+  property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format
-        case node.value.downcase
+  property captcha_key : String? = nil                 # Key for Anti-Captcha
-        when "ipv4"
-          Socket::Family::INET
-        when "ipv6"
-          Socket::Family::INET6
-        else
-          Socket::Family::UNSPEC
-        end
-      else
-        node.raise "Expected scalar, not #{node.class}"
-      end
-    end
-  end
-
-  module StringToCookies
-    def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder)
-      (value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml)
-    end
-
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies
-      unless node.is_a?(YAML::Nodes::Scalar)
-        node.raise "Expected scalar, not #{node.class}"
-      end
-
-      cookies = HTTP::Cookies.new
-      node.value.split(";").each do |cookie|
-        next if cookie.strip.empty?
-        name, value = cookie.split("=", 2)
-        cookies << HTTP::Cookie.new(name.strip, value.strip)
-      end
-
-      cookies
-    end
-  end
 
   def disabled?(option)
     case disabled = CONFIG.disable_proxy
@@ -229,50 +110,16 @@ struct Config
       return false
     end
   end
-
-  YAML.mapping({
-    channel_threads:          Int32,                                # Number of threads to use for crawling videos from channels (for updating subscriptions)
-    feed_threads:             Int32,                                # Number of threads to use for updating feeds
-    db:                       DBConfig,                             # Database configuration
-    full_refresh:             Bool,                                 # Used for crawling channels: threads should check all videos uploaded by a channel
-    https_only:               Bool?,                                # Used to tell Invidious it is behind a proxy, so links to resources should be https://
-    hmac_key:                 String?,                              # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
-    domain:                   String?,                              # Domain to be used for links to resources on the site where an absolute URL is required
-    use_pubsub_feeds:         {type: Bool | Int32, default: false}, # Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
-    captcha_enabled:          {type: Bool, default: true},
-    login_enabled:            {type: Bool, default: true},
-    registration_enabled:     {type: Bool, default: true},
-    statistics_enabled:       {type: Bool, default: false},
-    admins:                   {type: Array(String), default: [] of String},
-    external_port:            {type: Int32?, default: nil},
-    default_user_preferences: {type: Preferences,
-                               default: Preferences.new(*ConfigPreferences.from_yaml("").to_tuple),
-                               converter: ConfigPreferencesConverter,
-    },
-    dmca_content:      {type: Array(String), default: [] of String},                                        # For compliance with DMCA, disables download widget using list of video IDs
-    check_tables:      {type: Bool, default: false},                                                        # Check table integrity, automatically try to add any missing columns, create tables, etc.
-    cache_annotations: {type: Bool, default: false},                                                        # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards
-    banner:            {type: String?, default: nil},                                                       # Optional banner to be displayed along top of page for announcements, etc.
-    hsts:              {type: Bool?, default: true},                                                        # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely
-    disable_proxy:     {type: Bool? | Array(String)?, default: false},                                      # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local'
-    force_resolve:     {type: Socket::Family, default: Socket::Family::UNSPEC, converter: FamilyConverter}, # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
-    port:              {type: Int32, default: 3000},                                                        # Port to listen for connections (overrided by command line argument)
-    host_binding:      {type: String, default: "0.0.0.0"},                                                  # Host to bind (overrided by command line argument)
-    pool_size:         {type: Int32, default: 100},                                                         # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
-    admin_email:       {type: String, default: "omarroth@protonmail.com"},                                  # Email for bug reports
-    cookies:           {type: HTTP::Cookies, default: HTTP::Cookies.new, converter: StringToCookies},       # Saved cookies in "name1=value1; name2=value2..." format
-    captcha_key:       {type: String?, default: nil},                                                       # Key for Anti-Captcha
-  })
 end
 
 struct DBConfig
-  yaml_mapping({
+  include YAML::Serializable
-    user:     String,
+
-    password: String,
+  property user : String
-    host:     String,
+  property password : String
-    port:     Int32,
+  property host : String
-    dbname:   String,
+  property port : Int32
-  })
+  property dbname : String
 end
 
 def login_req(f_req)
@@ -365,7 +212,7 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
               end
             end
 
-            items << SearchVideo.new(
+            items << SearchVideo.new({
               title:              title,
               id:                 video_id,
               author:             author,
@@ -377,8 +224,8 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
               live_now:           live_now,
               paid:               paid,
               premium:            premium,
-              premiere_timestamp: premiere_timestamp
+              premiere_timestamp: premiere_timestamp,
-            )
+            })
           elsif i = item["channelRenderer"]?
             author = i["title"]["simpleText"]?.try &.as_s || author_fallback || ""
             author_id = i["channelId"]?.try &.as_s || author_id_fallback || ""
@@ -391,7 +238,7 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
             video_count = i["videoCountText"]?.try &.["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0
             description_html = i["descriptionSnippet"]?.try { |t| parse_content(t) } || ""
 
-            items << SearchChannel.new(
+            items << SearchChannel.new({
               author:           author,
               ucid:             author_id,
               author_thumbnail: author_thumbnail,
@@ -399,7 +246,7 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
               video_count:      video_count,
               description_html: description_html,
               auto_generated:   auto_generated,
-            )
+            })
           elsif i = item["gridPlaylistRenderer"]?
             title = i["title"]["runs"].as_a[0]?.try &.["text"].as_s || ""
             plid = i["playlistId"]?.try &.as_s || ""
@@ -407,15 +254,15 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
             video_count = i["videoCountText"]["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0
             playlist_thumbnail = i["thumbnail"]["thumbnails"][0]?.try &.["url"]?.try &.as_s || ""
 
-            items << SearchPlaylist.new(
+            items << SearchPlaylist.new({
               title:       title,
               id:          plid,
               author:      author_fallback || "",
               ucid:        author_id_fallback || "",
               video_count: video_count,
               videos:      [] of SearchPlaylistVideo,
-              thumbnail: playlist_thumbnail
+              thumbnail:   playlist_thumbnail,
-            )
+            })
           elsif i = item["playlistRenderer"]?
             title = i["title"]["simpleText"]?.try &.as_s || ""
             plid = i["playlistId"]?.try &.as_s || ""
@@ -432,24 +279,24 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
               v_title = v["title"]["simpleText"]?.try &.as_s || ""
               v_id = v["videoId"]?.try &.as_s || ""
               v_length_seconds = v["lengthText"]?.try &.["simpleText"]?.try { |t| decode_length_seconds(t.as_s) } || 0
-              SearchPlaylistVideo.new(
+              SearchPlaylistVideo.new({
                 title:          v_title,
                 id:             v_id,
-                length_seconds: v_length_seconds
+                length_seconds: v_length_seconds,
-              )
+              })
             end || [] of SearchPlaylistVideo
 
             # TODO: i["publishedTimeText"]?
 
-            items << SearchPlaylist.new(
+            items << SearchPlaylist.new({
               title:       title,
               id:          plid,
               author:      author,
               ucid:        author_id,
               video_count: video_count,
               videos:      videos,
-              thumbnail: playlist_thumbnail
+              thumbnail:   playlist_thumbnail,
-            )
+            })
           elsif i = item["radioRenderer"]? # Mix
             # TODO
           elsif i = item["showRenderer"]? # Show
@@ -465,6 +312,7 @@ end
 
 def check_enum(db, logger, enum_name, struct_type = nil)
   return # TODO
+
   if !db.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool)
     logger.puts("CREATE TYPE #{enum_name}")
 
@@ -488,7 +336,7 @@ def check_table(db, logger, table_name, struct_type = nil)
 
   return if !struct_type
 
-  struct_array = struct_type.to_type_tuple
+  struct_array = struct_type.type_array
   column_array = get_column_array(db, table_name)
   column_types = File.read("config/sql/#{table_name}.sql").match(/CREATE TABLE public\.#{table_name}\n\((?<types>[\d\D]*?)\);/)
     .try &.["types"].split(",").map { |line| line.strip }.reject &.starts_with?("CONSTRAINT")

src/invidious/helpers/jobs.cr

@@ -67,7 +67,7 @@ def refresh_feeds(db, logger, config)
             begin
               # Drop outdated views
               column_array = get_column_array(db, view_name)
-              ChannelVideo.to_type_tuple.each_with_index do |name, i|
+              ChannelVideo.type_array.each_with_index do |name, i|
                 if name != column_array[i]?
                   logger.puts("DROP MATERIALIZED VIEW #{view_name}")
                   db.exec("DROP MATERIALIZED VIEW #{view_name}")

src/invidious/helpers/macros.cr

@@ -1,43 +1,51 @@
-macro db_mapping(mapping)
+module DB::Serializable
-  def initialize({{*mapping.keys.map { |id| "@#{id}".id }}})
+  macro included
+    {% verbatim do %}
+      macro finished
+        def self.type_array
+          \{{ @type.instance_vars
+            .reject { |var| var.annotation(::DB::Field) && var.annotation(::DB::Field)[:ignore] }
+            .map { |name| name.stringify }
+          }}
+        end
+
+        def initialize(tuple)
+          \{% for var in @type.instance_vars %}
+            \{% ann = var.annotation(::DB::Field) %}
+            \{% if ann && ann[:ignore] %}
+            \{% else %}
+              @\{{var.name}} = tuple[:\{{var.name.id}}]
+            \{% end %}
+          \{% end %}
         end
 
         def to_a
-      return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ]
+          \{{ @type.instance_vars
+            .reject { |var| var.annotation(::DB::Field) && var.annotation(::DB::Field)[:ignore] }
+            .map { |name| name }
+          }}
+        end
+      end
+    {% end %}
+  end
 end
 
-  def self.to_type_tuple
+module JSON::Serializable
-      return { {{*mapping.keys.map { |id| "#{id}" }}} }
+  macro included
+    {% verbatim do %}
+      macro finished
+        def initialize(tuple)
+          \{% for var in @type.instance_vars %}
+            \{% ann = var.annotation(::JSON::Field) %}
+            \{% if ann && ann[:ignore] %}
+            \{% else %}
+              @\{{var.name}} = tuple[:\{{var.name.id}}]
+            \{% end %}
+          \{% end %}
         end
-
-  DB.mapping( {{mapping}} )
       end
-
+    {% end %}
-macro json_mapping(mapping)
-  def initialize({{*mapping.keys.map { |id| "@#{id}".id }}})
   end
-
-  def to_a
-      return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ]
-  end
-
-  patched_json_mapping( {{mapping}} )
-  YAML.mapping( {{mapping}} )
-end
-
-macro yaml_mapping(mapping)
-  def initialize({{*mapping.keys.map { |id| "@#{id}".id }}})
-  end
-
-  def to_a
-      return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ]
-  end
-
-  def to_tuple
-      return { {{*mapping.keys.map { |id| "@#{id}".id }}} }
-  end
-
-  YAML.mapping({{mapping}})
 end
 
 macro templated(filename, template = "template")

src/invidious/helpers/patch_mapping.cr

@@ -1,166 +0,0 @@
-# Overloads https://github.com/crystal-lang/crystal/blob/0.28.0/src/json/from_json.cr#L24
-def Object.from_json(string_or_io, default) : self
-  parser = JSON::PullParser.new(string_or_io)
-  new parser, default
-end
-
-# Adds configurable 'default'
-macro patched_json_mapping(_properties_, strict = false)
-  {% for key, value in _properties_ %}
-    {% _properties_[key] = {type: value} unless value.is_a?(HashLiteral) || value.is_a?(NamedTupleLiteral) %}
-  {% end %}
-
-  {% for key, value in _properties_ %}
-    {% _properties_[key][:key_id] = key.id.gsub(/\?$/, "") %}
-  {% end %}
-
-  {% for key, value in _properties_ %}
-    @{{value[:key_id]}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }}
-
-    {% if value[:setter] == nil ? true : value[:setter] %}
-      def {{value[:key_id]}}=(_{{value[:key_id]}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }})
-        @{{value[:key_id]}} = _{{value[:key_id]}}
-      end
-    {% end %}
-
-    {% if value[:getter] == nil ? true : value[:getter] %}
-      def {{key.id}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }}
-        @{{value[:key_id]}}
-      end
-    {% end %}
-
-    {% if value[:presence] %}
-      @{{value[:key_id]}}_present : Bool = false
-
-      def {{value[:key_id]}}_present?
-        @{{value[:key_id]}}_present
-      end
-    {% end %}
-  {% end %}
-
-  def initialize(%pull : ::JSON::PullParser, default = nil)
-    {% for key, value in _properties_ %}
-      %var{key.id} = nil
-      %found{key.id} = false
-    {% end %}
-
-    %location = %pull.location
-    begin
-      %pull.read_begin_object
-    rescue exc : ::JSON::ParseException
-      raise ::JSON::MappingError.new(exc.message, self.class.to_s, nil, *%location, exc)
-    end
-    until %pull.kind.end_object?
-      %key_location = %pull.location
-      key = %pull.read_object_key
-      case key
-      {% for key, value in _properties_ %}
-        when {{value[:key] || value[:key_id].stringify}}
-          %found{key.id} = true
-          begin
-            %var{key.id} =
-              {% if value[:nilable] || value[:default] != nil %} %pull.read_null_or { {% end %}
-
-              {% if value[:root] %}
-                %pull.on_key!({{value[:root]}}) do
-              {% end %}
-
-              {% if value[:converter] %}
-                {{value[:converter]}}.from_json(%pull)
-              {% elsif value[:type].is_a?(Path) || value[:type].is_a?(Generic) %}
-                {{value[:type]}}.new(%pull)
-              {% else %}
-                ::Union({{value[:type]}}).new(%pull)
-              {% end %}
-
-              {% if value[:root] %}
-                end
-              {% end %}
-
-            {% if value[:nilable] || value[:default] != nil %} } {% end %}
-          rescue exc : ::JSON::ParseException
-            raise ::JSON::MappingError.new(exc.message, self.class.to_s, {{value[:key] || value[:key_id].stringify}}, *%key_location, exc)
-          end
-      {% end %}
-      else
-        {% if strict %}
-          raise ::JSON::MappingError.new("Unknown JSON attribute: #{key}", self.class.to_s, nil, *%key_location, nil)
-        {% else %}
-          %pull.skip
-        {% end %}
-      end
-    end
-    %pull.read_next
-
-    {% for key, value in _properties_ %}
-      {% unless value[:nilable] || value[:default] != nil %}
-        if %var{key.id}.nil? && !%found{key.id} && !::Union({{value[:type]}}).nilable?
-          raise ::JSON::MappingError.new("Missing JSON attribute: {{(value[:key] || value[:key_id]).id}}", self.class.to_s, nil, *%location, nil)
-        end
-      {% end %}
-
-      {% if value[:nilable] %}
-        {% if value[:default] != nil %}
-          @{{value[:key_id]}} = %found{key.id} ? %var{key.id} : (default.responds_to?(:{{value[:key_id]}}) ? default.{{value[:key_id]}} : {{value[:default]}})
-        {% else %}
-          @{{value[:key_id]}} = %var{key.id}
-        {% end %}
-      {% elsif value[:default] != nil %}
-        @{{value[:key_id]}} = %var{key.id}.nil? ? (default.responds_to?(:{{value[:key_id]}}) ? default.{{value[:key_id]}} : {{value[:default]}}) : %var{key.id}
-      {% else %}
-        @{{value[:key_id]}} = (%var{key.id}).as({{value[:type]}})
-      {% end %}
-
-      {% if value[:presence] %}
-        @{{value[:key_id]}}_present = %found{key.id}
-      {% end %}
-    {% end %}
-  end
-
-  def to_json(json : ::JSON::Builder)
-    json.object do
-      {% for key, value in _properties_ %}
-        _{{value[:key_id]}} = @{{value[:key_id]}}
-
-        {% unless value[:emit_null] %}
-          unless _{{value[:key_id]}}.nil?
-        {% end %}
-
-          json.field({{value[:key] || value[:key_id].stringify}}) do
-            {% if value[:root] %}
-              {% if value[:emit_null] %}
-                if _{{value[:key_id]}}.nil?
-                  nil.to_json(json)
-                else
-              {% end %}
-
-              json.object do
-                json.field({{value[:root]}}) do
-            {% end %}
-
-            {% if value[:converter] %}
-              if _{{value[:key_id]}}
-                {{ value[:converter] }}.to_json(_{{value[:key_id]}}, json)
-              else
-                nil.to_json(json)
-              end
-            {% else %}
-              _{{value[:key_id]}}.to_json(json)
-            {% end %}
-
-            {% if value[:root] %}
-              {% if value[:emit_null] %}
-                end
-              {% end %}
-                end
-              end
-            {% end %}
-          end
-
-        {% unless value[:emit_null] %}
-          end
-        {% end %}
-      {% end %}
-    end
-  end
-end

src/invidious/mixes.cr

@@ -1,21 +1,21 @@
 struct MixVideo
-  db_mapping({
+  include DB::Serializable
-    title:          String,
+
-    id:             String,
+  property title : String
-    author:         String,
+  property id : String
-    ucid:           String,
+  property author : String
-    length_seconds: Int32,
+  property ucid : String
-    index:          Int32,
+  property length_seconds : Int32
-    rdid:           String,
+  property index : Int32
-  })
+  property rdid : String
 end
 
 struct Mix
-  db_mapping({
+  include DB::Serializable
-    title:  String,
+
-    id:     String,
+  property title : String
-    videos: Array(MixVideo),
+  property id : String
-  })
+  property videos : Array(MixVideo)
 end
 
 def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
@@ -48,23 +48,22 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
 
     id = item["videoId"].as_s
     title = item["title"]?.try &.["simpleText"].as_s
-    if !title
+    next if !title
-      next
+
-    end
     author = item["longBylineText"]["runs"][0]["text"].as_s
     ucid = item["longBylineText"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"].as_s
     length_seconds = decode_length_seconds(item["lengthText"]["simpleText"].as_s)
     index = item["navigationEndpoint"]["watchEndpoint"]["index"].as_i
 
-    videos << MixVideo.new(
+    videos << MixVideo.new({
-      title,
+      title:          title,
-      id,
+      id:             id,
-      author,
+      author:         author,
-      ucid,
+      ucid:           ucid,
-      length_seconds,
+      length_seconds: length_seconds,
-      index,
+      index:          index,
-      rdid
+      rdid:           rdid,
-    )
+    })
   end
 
   if !cookies
@@ -74,7 +73,11 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
 
   videos.uniq! { |video| video.id }
   videos = videos.first(50)
-  return Mix.new(mix_title, rdid, videos)
+  return Mix.new({
+    title:  mix_title,
+    id:     rdid,
+    videos: videos,
+  })
 end
 
 def template_mix(mix)

src/invidious/playlists.cr

@@ -1,4 +1,16 @@
 struct PlaylistVideo
+  include DB::Serializable
+
+  property title : String
+  property id : String
+  property author : String
+  property ucid : String
+  property length_seconds : Int32
+  property published : Time
+  property plid : String
+  property index : Int64
+  property live_now : Bool
+
   def to_xml(auto_generated, xml : XML::Builder)
     xml.element("entry") do
       xml.element("id") { xml.text "yt:video:#{self.id}" }
@@ -78,21 +90,22 @@ struct PlaylistVideo
       end
     end
   end
-
-  db_mapping({
-    title:          String,
-    id:             String,
-    author:         String,
-    ucid:           String,
-    length_seconds: Int32,
-    published:      Time,
-    plid:           String,
-    index:          Int64,
-    live_now:       Bool,
-  })
 end
 
 struct Playlist
+  include DB::Serializable
+
+  property title : String
+  property id : String
+  property author : String
+  property author_thumbnail : String
+  property ucid : String
+  property description : String
+  property video_count : Int32
+  property views : Int64
+  property updated : Time
+  property thumbnail : String?
+
   def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
     json.object do
       json.field "type", "playlist"
@@ -147,19 +160,6 @@ struct Playlist
     end
   end
 
-  db_mapping({
-    title:            String,
-    id:               String,
-    author:           String,
-    author_thumbnail: String,
-    ucid:             String,
-    description:      String,
-    video_count:      Int32,
-    views:            Int64,
-    updated:          Time,
-    thumbnail:        String?,
-  })
-
   def privacy
     PlaylistPrivacy::Public
   end
@@ -176,6 +176,29 @@ enum PlaylistPrivacy
 end
 
 struct InvidiousPlaylist
+  include DB::Serializable
+
+  property title : String
+  property id : String
+  property author : String
+  property description : String = ""
+  property video_count : Int32
+  property created : Time
+  property updated : Time
+
+  @[DB::Field(converter: InvidiousPlaylist::PlaylistPrivacyConverter)]
+  property privacy : PlaylistPrivacy = PlaylistPrivacy::Private
+  property index : Array(Int64)
+
+  @[DB::Field(ignore: true)]
+  property thumbnail_id : String?
+
+  module PlaylistPrivacyConverter
+    def self.from_rs(rs)
+      return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8))))
+    end
+  end
+
   def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
     json.object do
       json.field "type", "invidiousPlaylist"
@@ -216,26 +239,6 @@ struct InvidiousPlaylist
     end
   end
 
-  property thumbnail_id
-
-  module PlaylistPrivacyConverter
-    def self.from_rs(rs)
-      return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8))))
-    end
-  end
-
-  db_mapping({
-    title:       String,
-    id:          String,
-    author:      String,
-    description: {type: String, default: ""},
-    video_count: Int32,
-    created:     Time,
-    updated:     Time,
-    privacy:     {type: PlaylistPrivacy, default: PlaylistPrivacy::Private, converter: PlaylistPrivacyConverter},
-    index:       Array(Int64),
-  })
-
   def thumbnail
     @thumbnail_id ||= PG_DB.query_one?("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 1", self.id, self.index, as: String) || "-----------"
     "/vi/#{@thumbnail_id}/mqdefault.jpg"
@@ -261,7 +264,7 @@ end
 def create_playlist(db, title, privacy, user)
   plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}"
 
-  playlist = InvidiousPlaylist.new(
+  playlist = InvidiousPlaylist.new({
     title:       title.byte_slice(0, 150),
     id:          plid,
     author:      user.email,
@@ -271,7 +274,7 @@ def create_playlist(db, title, privacy, user)
     updated:     Time.utc,
     privacy:     privacy,
     index:       [] of Int64,
-  )
+  })
 
   playlist_array = playlist.to_a
   args = arg_array(playlist_array)
@@ -282,7 +285,7 @@ def create_playlist(db, title, privacy, user)
 end
 
 def subscribe_playlist(db, user, playlist)
-  playlist = InvidiousPlaylist.new(
+  playlist = InvidiousPlaylist.new({
     title:       playlist.title.byte_slice(0, 150),
     id:          playlist.id,
     author:      user.email,
@@ -292,7 +295,7 @@ def subscribe_playlist(db, user, playlist)
     updated:     playlist.updated,
     privacy:     PlaylistPrivacy::Private,
     index:       [] of Int64,
-  )
+  })
 
   playlist_array = playlist.to_a
   args = arg_array(playlist_array)
@@ -393,7 +396,7 @@ def fetch_playlist(plid, locale)
   author = author_info["title"]["runs"][0]["text"]?.try &.as_s || ""
   ucid = author_info["title"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"]?.try &.as_s || ""
 
-  return Playlist.new(
+  return Playlist.new({
     title:            title,
     id:               plid,
     author:           author,
@@ -403,8 +406,8 @@ def fetch_playlist(plid, locale)
     video_count:      video_count,
     views:            views,
     updated:          updated,
-    thumbnail: thumbnail
+    thumbnail:        thumbnail,
-  )
+  })
 end
 
 def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil)
@@ -471,7 +474,7 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
         length_seconds = 0
       end
 
-      videos << PlaylistVideo.new(
+      videos << PlaylistVideo.new({
         title:          title,
         id:             video_id,
         author:         author,
@@ -480,8 +483,8 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
         published:      Time.utc,
         plid:           plid,
         live_now:       live,
-        index: index - 1
+        index:          index - 1,
-      )
+      })
     end
   end
 

src/invidious/search.cr

@@ -1,4 +1,19 @@
 struct SearchVideo
+  include DB::Serializable
+
+  property title : String
+  property id : String
+  property author : String
+  property ucid : String
+  property published : Time
+  property views : Int64
+  property description_html : String
+  property length_seconds : Int32
+  property live_now : Bool
+  property paid : Bool
+  property premium : Bool
+  property premiere_timestamp : Time?
+
   def to_xml(auto_generated, query_params, xml : XML::Builder)
     query_params["v"] = self.id
 
@@ -99,32 +114,27 @@ struct SearchVideo
   def is_upcoming
     premiere_timestamp ? true : false
   end
-
-  db_mapping({
-    title:              String,
-    id:                 String,
-    author:             String,
-    ucid:               String,
-    published:          Time,
-    views:              Int64,
-    description_html:   String,
-    length_seconds:     Int32,
-    live_now:           Bool,
-    paid:               Bool,
-    premium:            Bool,
-    premiere_timestamp: Time?,
-  })
 end
 
 struct SearchPlaylistVideo
-  db_mapping({
+  include DB::Serializable
-    title:          String,
+
-    id:             String,
+  property title : String
-    length_seconds: Int32,
+  property id : String
-  })
+  property length_seconds : Int32
 end
 
 struct SearchPlaylist
+  include DB::Serializable
+
+  property title : String
+  property id : String
+  property author : String
+  property ucid : String
+  property video_count : Int32
+  property videos : Array(SearchPlaylistVideo)
+  property thumbnail : String?
+
   def to_json(locale, json : JSON::Builder)
     json.object do
       json.field "type", "playlist"
@@ -164,19 +174,19 @@ struct SearchPlaylist
       end
     end
   end
-
-  db_mapping({
-    title:       String,
-    id:          String,
-    author:      String,
-    ucid:        String,
-    video_count: Int32,
-    videos:      Array(SearchPlaylistVideo),
-    thumbnail:   String?,
-  })
 end
 
 struct SearchChannel
+  include DB::Serializable
+
+  property author : String
+  property ucid : String
+  property author_thumbnail : String
+  property subscriber_count : Int32
+  property video_count : Int32
+  property description_html : String
+  property auto_generated : Bool
+
   def to_json(locale, json : JSON::Builder)
     json.object do
       json.field "type", "channel"
@@ -216,16 +226,6 @@ struct SearchChannel
       end
     end
   end
-
-  db_mapping({
-    author:           String,
-    ucid:             String,
-    author_thumbnail: String,
-    subscriber_count: Int32,
-    video_count:      Int32,
-    description_html: String,
-    auto_generated:   Bool,
-  })
 end
 
 alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist

src/invidious/users.cr

@@ -4,6 +4,20 @@ require "crypto/bcrypt/password"
 MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" }
 
 struct User
+  include DB::Serializable
+
+  property updated : Time
+  property notifications : Array(String)
+  property subscriptions : Array(String)
+  property email : String
+
+  @[DB::Field(converter: User::PreferencesConverter)]
+  property preferences : Preferences
+  property password : String?
+  property token : String
+  property watched : Array(String)
+  property feed_needs_update : Bool?
+
   module PreferencesConverter
     def self.from_rs(rs)
       begin
@@ -13,31 +27,78 @@ struct User
       end
     end
   end
-
-  db_mapping({
-    updated:       Time,
-    notifications: Array(String),
-    subscriptions: Array(String),
-    email:         String,
-    preferences:   {
-      type:      Preferences,
-      converter: PreferencesConverter,
-    },
-    password:          String?,
-    token:             String,
-    watched:           Array(String),
-    feed_needs_update: Bool?,
-  })
 end
 
 struct Preferences
-  module ProcessString
+  include JSON::Serializable
+  include YAML::Serializable
+
+  property annotations : Bool = CONFIG.default_user_preferences.annotations
+  property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed
+  property autoplay : Bool = CONFIG.default_user_preferences.autoplay
+
+  @[JSON::Field(converter: Preferences::StringToArray)]
+  @[YAML::Field(converter: Preferences::StringToArray)]
+  property captions : Array(String) = CONFIG.default_user_preferences.captions
+
+  @[JSON::Field(converter: Preferences::StringToArray)]
+  @[YAML::Field(converter: Preferences::StringToArray)]
+  property comments : Array(String) = CONFIG.default_user_preferences.comments
+  property continue : Bool = CONFIG.default_user_preferences.continue
+  property continue_autoplay : Bool = CONFIG.default_user_preferences.continue_autoplay
+
+  @[JSON::Field(converter: Preferences::BoolToString)]
+  @[YAML::Field(converter: Preferences::BoolToString)]
+  property dark_mode : String = CONFIG.default_user_preferences.dark_mode
+  property latest_only : Bool = CONFIG.default_user_preferences.latest_only
+  property listen : Bool = CONFIG.default_user_preferences.listen
+  property local : Bool = CONFIG.default_user_preferences.local
+
+  @[JSON::Field(converter: Preferences::ProcessString)]
+  property locale : String = CONFIG.default_user_preferences.locale
+
+  @[JSON::Field(converter: Preferences::ClampInt)]
+  property max_results : Int32 = CONFIG.default_user_preferences.max_results
+  property notifications_only : Bool = CONFIG.default_user_preferences.notifications_only
+
+  @[JSON::Field(converter: Preferences::ProcessString)]
+  property player_style : String = CONFIG.default_user_preferences.player_style
+
+  @[JSON::Field(converter: Preferences::ProcessString)]
+  property quality : String = CONFIG.default_user_preferences.quality
+  property default_home : String = CONFIG.default_user_preferences.default_home
+  property feed_menu : Array(String) = CONFIG.default_user_preferences.feed_menu
+  property related_videos : Bool = CONFIG.default_user_preferences.related_videos
+
+  @[JSON::Field(converter: Preferences::ProcessString)]
+  property sort : String = CONFIG.default_user_preferences.sort
+  property speed : Float32 = CONFIG.default_user_preferences.speed
+  property thin_mode : Bool = CONFIG.default_user_preferences.thin_mode
+  property unseen_only : Bool = CONFIG.default_user_preferences.unseen_only
+  property video_loop : Bool = CONFIG.default_user_preferences.video_loop
+  property volume : Int32 = CONFIG.default_user_preferences.volume
+
+  module BoolToString
     def self.to_json(value : String, json : JSON::Builder)
       json.string value
     end
 
     def self.from_json(value : JSON::PullParser) : String
-      HTML.escape(value.read_string[0, 100])
+      begin
+        result = value.read_string
+
+        if result.empty?
+          CONFIG.default_user_preferences.dark_mode
+        else
+          result
+        end
+      rescue ex
+        if value.read_bool
+          "dark"
+        else
+          "light"
+        end
+      end
     end
 
     def self.to_yaml(value : String, yaml : YAML::Nodes::Builder)
@@ -45,7 +106,20 @@ struct Preferences
     end
 
     def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String
-      HTML.escape(node.value[0, 100])
+      unless node.is_a?(YAML::Nodes::Scalar)
+        node.raise "Expected scalar, not #{node.class}"
+      end
+
+      case node.value
+      when "true"
+        "dark"
+      when "false"
+        "light"
+      when ""
+        CONFIG.default_user_preferences.dark_mode
+      else
+        node.value
+      end
     end
   end
 
@@ -67,33 +141,130 @@ struct Preferences
     end
   end
 
-  json_mapping({
+  module FamilyConverter
-    annotations:            {type: Bool, default: CONFIG.default_user_preferences.annotations},
+    def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder)
-    annotations_subscribed: {type: Bool, default: CONFIG.default_user_preferences.annotations_subscribed},
+      case value
-    autoplay:               {type: Bool, default: CONFIG.default_user_preferences.autoplay},
+      when Socket::Family::UNSPEC
-    captions:               {type: Array(String), default: CONFIG.default_user_preferences.captions, converter: ConfigPreferences::StringToArray},
+        yaml.scalar nil
-    comments:               {type: Array(String), default: CONFIG.default_user_preferences.comments, converter: ConfigPreferences::StringToArray},
+      when Socket::Family::INET
-    continue:               {type: Bool, default: CONFIG.default_user_preferences.continue},
+        yaml.scalar "ipv4"
-    continue_autoplay:      {type: Bool, default: CONFIG.default_user_preferences.continue_autoplay},
+      when Socket::Family::INET6
-    dark_mode:              {type: String, default: CONFIG.default_user_preferences.dark_mode, converter: ConfigPreferences::BoolToString},
+        yaml.scalar "ipv6"
-    latest_only:            {type: Bool, default: CONFIG.default_user_preferences.latest_only},
+      when Socket::Family::UNIX
-    listen:                 {type: Bool, default: CONFIG.default_user_preferences.listen},
+        raise "Invalid socket family #{value}"
-    local:                  {type: Bool, default: CONFIG.default_user_preferences.local},
+      end
-    locale:                 {type: String, default: CONFIG.default_user_preferences.locale, converter: ProcessString},
+    end
-    max_results:            {type: Int32, default: CONFIG.default_user_preferences.max_results, converter: ClampInt},
+
-    notifications_only:     {type: Bool, default: CONFIG.default_user_preferences.notifications_only},
+    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family
-    player_style:           {type: String, default: CONFIG.default_user_preferences.player_style, converter: ProcessString},
+      if node.is_a?(YAML::Nodes::Scalar)
-    quality:                {type: String, default: CONFIG.default_user_preferences.quality, converter: ProcessString},
+        case node.value.downcase
-    default_home:           {type: String, default: CONFIG.default_user_preferences.default_home},
+        when "ipv4"
-    feed_menu:              {type: Array(String), default: CONFIG.default_user_preferences.feed_menu},
+          Socket::Family::INET
-    related_videos:         {type: Bool, default: CONFIG.default_user_preferences.related_videos},
+        when "ipv6"
-    sort:                   {type: String, default: CONFIG.default_user_preferences.sort, converter: ProcessString},
+          Socket::Family::INET6
-    speed:                  {type: Float32, default: CONFIG.default_user_preferences.speed},
+        else
-    thin_mode:              {type: Bool, default: CONFIG.default_user_preferences.thin_mode},
+          Socket::Family::UNSPEC
-    unseen_only:            {type: Bool, default: CONFIG.default_user_preferences.unseen_only},
+        end
-    video_loop:             {type: Bool, default: CONFIG.default_user_preferences.video_loop},
+      else
-    volume:                 {type: Int32, default: CONFIG.default_user_preferences.volume},
+        node.raise "Expected scalar, not #{node.class}"
-  })
+      end
+    end
+  end
+
+  module ProcessString
+    def self.to_json(value : String, json : JSON::Builder)
+      json.string value
+    end
+
+    def self.from_json(value : JSON::PullParser) : String
+      HTML.escape(value.read_string[0, 100])
+    end
+
+    def self.to_yaml(value : String, yaml : YAML::Nodes::Builder)
+      yaml.scalar value
+    end
+
+    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String
+      HTML.escape(node.value[0, 100])
+    end
+  end
+
+  module StringToArray
+    def self.to_json(value : Array(String), json : JSON::Builder)
+      json.array do
+        value.each do |element|
+          json.string element
+        end
+      end
+    end
+
+    def self.from_json(value : JSON::PullParser) : Array(String)
+      begin
+        result = [] of String
+        value.read_array do
+          result << HTML.escape(value.read_string[0, 100])
+        end
+      rescue ex
+        result = [HTML.escape(value.read_string[0, 100]), ""]
+      end
+
+      result
+    end
+
+    def self.to_yaml(value : Array(String), yaml : YAML::Nodes::Builder)
+      yaml.sequence do
+        value.each do |element|
+          yaml.scalar element
+        end
+      end
+    end
+
+    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(String)
+      begin
+        unless node.is_a?(YAML::Nodes::Sequence)
+          node.raise "Expected sequence, not #{node.class}"
+        end
+
+        result = [] of String
+        node.nodes.each do |item|
+          unless item.is_a?(YAML::Nodes::Scalar)
+            node.raise "Expected scalar, not #{item.class}"
+          end
+
+          result << HTML.escape(item.value[0, 100])
+        end
+      rescue ex
+        if node.is_a?(YAML::Nodes::Scalar)
+          result = [HTML.escape(node.value[0, 100]), ""]
+        else
+          result = ["", ""]
+        end
+      end
+
+      result
+    end
+  end
+
+  module StringToCookies
+    def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder)
+      (value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml)
+    end
+
+    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies
+      unless node.is_a?(YAML::Nodes::Scalar)
+        node.raise "Expected scalar, not #{node.class}"
+      end
+
+      cookies = HTTP::Cookies.new
+      node.value.split(";").each do |cookie|
+        next if cookie.strip.empty?
+        name, value = cookie.split("=", 2)
+        cookies << HTTP::Cookie.new(name.strip, value.strip)
+      end
+
+      cookies
+    end
+  end
 end
 
 def get_user(sid, headers, db, refresh = true)
@@ -103,8 +274,7 @@ def get_user(sid, headers, db, refresh = true)
     if refresh && Time.utc - user.updated > 1.minute
       user, sid = fetch_user(sid, headers, db)
       user_array = user.to_a
-
+      user_array[4] = user_array[4].to_json # User preferences
-      user_array[4] = user_array[4].to_json
       args = arg_array(user_array)
 
       db.exec("INSERT INTO users VALUES (#{args}) \
@@ -122,8 +292,7 @@ def get_user(sid, headers, db, refresh = true)
   else
     user, sid = fetch_user(sid, headers, db)
     user_array = user.to_a
-
+    user_array[4] = user_array[4].to_json # User preferences
-    user_array[4] = user_array[4].to_json
     args = arg_array(user.to_a)
 
     db.exec("INSERT INTO users VALUES (#{args}) \
@@ -166,7 +335,17 @@ def fetch_user(sid, headers, db)
 
   token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
 
-  user = User.new(Time.utc, [] of String, channels, email, CONFIG.default_user_preferences, nil, token, [] of String, true)
+  user = User.new({
+    updated:           Time.utc,
+    notifications:     [] of String,
+    subscriptions:     channels,
+    email:             email,
+    preferences:       Preferences.new(CONFIG.default_user_preferences.to_tuple),
+    password:          nil,
+    token:             token,
+    watched:           [] of String,
+    feed_needs_update: true,
+  })
   return user, sid
 end
 
@@ -174,7 +353,17 @@ def create_user(sid, email, password)
   password = Crypto::Bcrypt::Password.create(password, cost: 10)
   token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
 
-  user = User.new(Time.utc, [] of String, [] of String, email, CONFIG.default_user_preferences, password.to_s, token, [] of String, true)
+  user = User.new({
+    updated:           Time.utc,
+    notifications:     [] of String,
+    subscriptions:     [] of String,
+    email:             email,
+    preferences:       Preferences.new(CONFIG.default_user_preferences.to_tuple),
+    password:          password.to_s,
+    token:             token,
+    watched:           [] of String,
+    feed_needs_update: true,
+  })
 
   return user, sid
 end
@@ -281,48 +470,6 @@ def subscribe_ajax(channel_id, action, env_headers)
   end
 end
 
-# TODO: Playlist stub, sync with YouTube for Google accounts
-# def playlist_ajax(video_ids, source_playlist_id, name, privacy, action, env_headers)
-#   headers = HTTP::Headers.new
-#   headers["Cookie"] = env_headers["Cookie"]
-#
-#   html = YT_POOL.client &.get("/view_all_playlists", headers)
-#
-#   cookies = HTTP::Cookies.from_headers(headers)
-#   html.cookies.each do |cookie|
-#     if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name
-#       if cookies[cookie.name]?
-#         cookies[cookie.name] = cookie
-#       else
-#         cookies << cookie
-#       end
-#     end
-#   end
-#   headers = cookies.add_request_headers(headers)
-#
-#   if match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[^"]+)"/)
-#     session_token = match["session_token"]
-#
-#     headers["content-type"] = "application/x-www-form-urlencoded"
-#
-#     post_req = {
-#       video_ids:          [] of String,
-#       source_playlist_id: "",
-#       n:                  name,
-#       p:                  privacy,
-#       session_token:      session_token,
-#     }
-#     post_url = "/playlist_ajax?#{action}=1"
-#
-#     response = client.post(post_url, headers, form: post_req)
-#     if response.status_code == 200
-#       return JSON.parse(response.body)["result"]["playlistId"].as_s
-#     else
-#       return nil
-#     end
-#   end
-# end
-
 def get_subscription_feed(db, user, max_results = 40, page = 1)
   limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE)
   offset = (page - 1) * limit

src/invidious/videos.cr

@@ -222,30 +222,50 @@ VIDEO_FORMATS = {
 }
 
 struct VideoPreferences
-  json_mapping({
+  include JSON::Serializable
-    annotations:        Bool,
+
-    autoplay:           Bool,
+  property annotations : Bool
-    comments:           Array(String),
+  property autoplay : Bool
-    continue:           Bool,
+  property comments : Array(String)
-    continue_autoplay:  Bool,
+  property continue : Bool
-    controls:           Bool,
+  property continue_autoplay : Bool
-    listen:             Bool,
+  property controls : Bool
-    local:              Bool,
+  property listen : Bool
-    preferred_captions: Array(String),
+  property local : Bool
-    player_style:       String,
+  property preferred_captions : Array(String)
-    quality:            String,
+  property player_style : String
-    raw:                Bool,
+  property quality : String
-    region:             String?,
+  property raw : Bool
-    related_videos:     Bool,
+  property region : String?
-    speed:              (Float32 | Float64),
+  property related_videos : Bool
-    video_end:          (Float64 | Int32),
+  property speed : Float32 | Float64
-    video_loop:         Bool,
+  property video_end : Float64 | Int32
-    video_start:        (Float64 | Int32),
+  property video_loop : Bool
-    volume:             Int32,
+  property video_start : Float64 | Int32
-  })
+  property volume : Int32
 end
 
 struct Video
+  include DB::Serializable
+
+  property id : String
+
+  @[DB::Field(converter: Video::JSONConverter)]
+  property info : Hash(String, JSON::Any)
+  property updated : Time
+
+  @[DB::Field(ignore: true)]
+  property captions : Array(Caption)?
+
+  @[DB::Field(ignore: true)]
+  property adaptive_fmts : Array(Hash(String, JSON::Any))?
+
+  @[DB::Field(ignore: true)]
+  property fmt_stream : Array(Hash(String, JSON::Any))?
+
+  @[DB::Field(ignore: true)]
+  property description : String?
+
   module JSONConverter
     def self.from_rs(rs)
       JSON.parse(rs.read(String)).as_h
@@ -552,6 +572,7 @@ struct Video
 
   def fmt_stream
     return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream
+
     fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
     fmt_stream.each do |fmt|
       if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
@@ -751,30 +772,20 @@ struct Video
   def session_token : String?
     info["sessionToken"]?.try &.as_s?
   end
-
-  db_mapping({
-    id:      String,
-    info:    {type: Hash(String, JSON::Any), converter: Video::JSONConverter},
-    updated: Time,
-  })
-
-  @captions : Array(Caption)?
-  @adaptive_fmts : Array(Hash(String, JSON::Any))?
-  @fmt_stream : Array(Hash(String, JSON::Any))?
-end
-
-struct Caption
-  json_mapping({
-    name:         CaptionName,
-    baseUrl:      String,
-    languageCode: String,
-  })
 end
 
 struct CaptionName
-  json_mapping({
+  include JSON::Serializable
-    simpleText: String,
+
-  })
+  property simpleText : String
+end
+
+struct Caption
+  include JSON::Serializable
+
+  property name : CaptionName
+  property baseUrl : String
+  property languageCode : String
 end
 
 class VideoRedirect < Exception
@@ -990,7 +1001,12 @@ def fetch_video(id, region)
 
   raise info["reason"]?.try &.as_s || "" if !info["videoDetails"]?
 
-  video = Video.new(id, info, Time.utc)
+  video = Video.new({
+    id:      id,
+    info:    info,
+    updated: Time.utc,
+  })
+
   return video
 end
 
@@ -1097,7 +1113,7 @@ def process_video_params(query, preferences)
   controls ||= 1
   controls = controls >= 1
 
-  params = VideoPreferences.new(
+  params = VideoPreferences.new({
     annotations:        annotations,
     autoplay:           autoplay,
     comments:           comments,
@@ -1117,7 +1133,7 @@ def process_video_params(query, preferences)
     video_loop:         video_loop,
     video_start:        video_start,
     volume:             volume,
-  )
+  })
 
   return params
 end