Mirrored_Repos/invidious
1
0
Fork 0
mirror of https://github.com/iv-org/invidious.git synced 2025-11-04 14:41:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix warnings with latest version of Crystal

Browse Source
This commit is contained in:
Omar Roth
2020-07-26 10:58:50 -04:00
parent 92f337c67e
commit 452d1e8307
12 changed files with 843 additions and 979 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
src/invidious.cr
View File

@@ -1203,17 +1203,17 @@ post "/playlist_ajax" do |env|
end
end
playlist_video = PlaylistVideo.new(
title: video.title,
id: video.id,
author: video.author,
ucid: video.ucid,
playlist_video = PlaylistVideo.new({
title: video.title,
id: video.id,
author: video.author,
ucid: video.ucid,
length_seconds: video.length_seconds,
published: video.published,
plid: playlist_id,
live_now: video.live_now,
index: Random::Secure.rand(0_i64..Int64::MAX)
)
published: video.published,
plid: playlist_id,
live_now: video.live_now,
index: Random::Secure.rand(0_i64..Int64::MAX),
})
video_array = playlist_video.to_a
args = arg_array(video_array)
@@ -1839,8 +1839,8 @@ post "/login" do |env|
sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
user, sid = create_user(sid, email, password)
user_array = user.to_a
user_array[4] = user_array[4].to_json # User preferences
user_array[4] = user_array[4].to_json
args = arg_array(user_array)
PG_DB.exec("INSERT INTO users VALUES (#{args})", args: user_array)
@@ -2519,7 +2519,7 @@ post "/data_control" do |env|
if user
user = user.as(User)
# TODO: Find better way to prevent timeout
# TODO: Find a way to prevent browser timeout
HTTP::FormData.parse(env.request) do |part|
body = part.body.gets_to_end
@@ -2546,7 +2546,7 @@ post "/data_control" do |env|
end
if body["preferences"]?
user.preferences = Preferences.from_json(body["preferences"].to_json, user.preferences)
user.preferences = Preferences.from_json(body["preferences"].to_json)
PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", user.preferences.to_json, user.email)
end
@@ -2573,17 +2573,17 @@ post "/data_control" do |env|
next
end
playlist_video = PlaylistVideo.new(
title: video.title,
id: video.id,
author: video.author,
ucid: video.ucid,
playlist_video = PlaylistVideo.new({
title: video.title,
id: video.id,
author: video.author,
ucid: video.ucid,
length_seconds: video.length_seconds,
published: video.published,
plid: playlist.id,
live_now: video.live_now,
index: Random::Secure.rand(0_i64..Int64::MAX)
)
published: video.published,
plid: playlist.id,
live_now: video.live_now,
index: Random::Secure.rand(0_i64..Int64::MAX),
})
video_array = playlist_video.to_a
args = arg_array(video_array)
@@ -3154,20 +3154,20 @@ get "/feed/channel/:ucid" do |env|
description_html = entry.xpath_node("group/description").not_nil!.to_s
views = entry.xpath_node("group/community/statistics").not_nil!.["views"].to_i64
SearchVideo.new(
title: title,
id: video_id,
author: author,
ucid: ucid,
published: published,
views: views,
description_html: description_html,
length_seconds: 0,
live_now: false,
paid: false,
premium: false,
premiere_timestamp: nil
)
SearchVideo.new({
title: title,
id: video_id,
author: author,
ucid: ucid,
published: published,
views: views,
description_html: description_html,
length_seconds: 0,
live_now: false,
paid: false,
premium: false,
premiere_timestamp: nil,
})
end
XML.build(indent: " ", encoding: "UTF-8") do |xml|
@@ -3397,18 +3397,18 @@ post "/feed/webhook/:token" do |env|
}.to_json
PG_DB.exec("NOTIFY notifications, E'#{payload}'")
video = ChannelVideo.new(
id: id,
title: video.title,
published: published,
updated: updated,
ucid: video.ucid,
author: author,
length_seconds: video.length_seconds,
live_now: video.live_now,
video = ChannelVideo.new({
id: id,
title: video.title,
published: published,
updated: updated,
ucid: video.ucid,
author: author,
length_seconds: video.length_seconds,
live_now: video.live_now,
premiere_timestamp: video.premiere_timestamp,
views: video.views,
)
views: video.views,
})
PG_DB.query_all("UPDATE users SET feed_needs_update = true, notifications = array_append(notifications, $1) \
WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)",
@@ -4666,7 +4666,7 @@ post "/api/v1/auth/preferences" do |env|
user = env.get("user").as(User)
begin
preferences = Preferences.from_json(env.request.body || "{}", user.preferences)
preferences = Preferences.from_json(env.request.body || "{}")
rescue
preferences = user.preferences
end
@@ -4920,17 +4920,17 @@ post "/api/v1/auth/playlists/:plid/videos" do |env|
next error_message
end
playlist_video = PlaylistVideo.new(
title: video.title,
id: video.id,
author: video.author,
ucid: video.ucid,
playlist_video = PlaylistVideo.new({
title: video.title,
id: video.id,
author: video.author,
ucid: video.ucid,
length_seconds: video.length_seconds,
published: video.published,
plid: plid,
live_now: video.live_now,
index: Random::Secure.rand(0_i64..Int64::MAX)
)
published: video.published,
plid: plid,
live_now: video.live_now,
index: Random::Secure.rand(0_i64..Int64::MAX),
})
video_array = playlist_video.to_a
args = arg_array(video_array)

204
src/invidious/channels.cr
View File

@@ -1,14 +1,27 @@
struct InvidiousChannel
db_mapping({
id: String,
author: String,
updated: Time,
deleted: Bool,
subscribed: Time?,
})
include DB::Serializable
property id : String
property author : String
property updated : Time
property deleted : Bool
property subscribed : Time?
end
struct ChannelVideo
include DB::Serializable
property id : String
property title : String
property published : Time
property updated : Time
property ucid : String
property author : String
property length_seconds : Int32 = 0
property live_now : Bool = false
property premiere_timestamp : Time? = nil
property views : Int64? = nil
def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "shortVideo"
@@ -84,49 +97,36 @@ struct ChannelVideo
end
end
end
db_mapping({
id: String,
title: String,
published: Time,
updated: Time,
ucid: String,
author: String,
length_seconds: {type: Int32, default: 0},
live_now: {type: Bool, default: false},
premiere_timestamp: {type: Time?, default: nil},
views: {type: Int64?, default: nil},
})
end
struct AboutRelatedChannel
db_mapping({
ucid: String,
author: String,
author_url: String,
author_thumbnail: String,
})
include DB::Serializable
property ucid : String
property author : String
property author_url : String
property author_thumbnail : String
end
# TODO: Refactor into either SearchChannel or InvidiousChannel
struct AboutChannel
db_mapping({
ucid: String,
author: String,
auto_generated: Bool,
author_url: String,
author_thumbnail: String,
banner: String?,
description_html: String,
paid: Bool,
total_views: Int64,
sub_count: Int32,
joined: Time,
is_family_friendly: Bool,
allowed_regions: Array(String),
related_channels: Array(AboutRelatedChannel),
tabs: Array(String),
})
include DB::Serializable
property ucid : String
property author : String
property auto_generated : Bool
property author_url : String
property author_thumbnail : String
property banner : String?
property description_html : String
property paid : Bool
property total_views : Int64
property sub_count : Int32
property joined : Time
property is_family_friendly : Bool
property allowed_regions : Array(String)
property related_channels : Array(AboutRelatedChannel)
property tabs : Array(String)
end
class ChannelRedirect < Exception
@@ -248,18 +248,18 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
premiere_timestamp = channel_video.try &.premiere_timestamp
video = ChannelVideo.new(
id: video_id,
title: title,
published: published,
updated: Time.utc,
ucid: ucid,
author: author,
length_seconds: length_seconds,
live_now: live_now,
video = ChannelVideo.new({
id: video_id,
title: title,
published: published,
updated: Time.utc,
ucid: ucid,
author: author,
length_seconds: length_seconds,
live_now: live_now,
premiere_timestamp: premiere_timestamp,
views: views,
)
views: views,
})
emails = db.query_all("UPDATE users SET notifications = array_append(notifications, $1) \
WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications) RETURNING email",
@@ -298,18 +298,18 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
videos = extract_videos(initial_data.as_h, author, ucid)
count = videos.size
videos = videos.map { |video| ChannelVideo.new(
id: video.id,
title: video.title,
published: video.published,
updated: Time.utc,
ucid: video.ucid,
author: video.author,
length_seconds: video.length_seconds,
live_now: video.live_now,
videos = videos.map { |video| ChannelVideo.new({
id: video.id,
title: video.title,
published: video.published,
updated: Time.utc,
ucid: video.ucid,
author: video.author,
length_seconds: video.length_seconds,
live_now: video.live_now,
premiere_timestamp: video.premiere_timestamp,
views: video.views
) }
views: video.views,
}) }
videos.each do |video|
ids << video.id
@@ -352,7 +352,13 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
db.exec("DELETE FROM channel_videos * WHERE NOT id = ANY ('{#{ids.map { |id| %("#{id}") }.join(",")}}') AND ucid = $1", ucid)
end
channel = InvidiousChannel.new(ucid, author, Time.utc, false, nil)
channel = InvidiousChannel.new({
id: ucid,
author: author,
updated: Time.utc,
deleted: false,
subscribed: nil,
})
return channel
end
@@ -395,12 +401,12 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "
"80226972:embedded" => {
"2:string" => ucid,
"3:base64" => {
"2:string" => "videos",
"6:varint": 2_i64,
"7:varint": 1_i64,
"12:varint": 1_i64,
"13:string": "",
"23:varint": 0_i64,
"2:string" => "videos",
"6:varint" => 2_i64,
"7:varint" => 1_i64,
"12:varint" => 1_i64,
"13:string" => "",
"23:varint" => 0_i64,
},
},
}
@@ -444,12 +450,12 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated
"80226972:embedded" => {
"2:string" => ucid,
"3:base64" => {
"2:string" => "playlists",
"6:varint": 2_i64,
"7:varint": 1_i64,
"12:varint": 1_i64,
"13:string": "",
"23:varint": 0_i64,
"2:string" => "playlists",
"6:varint" => 2_i64,
"7:varint" => 1_i64,
"12:varint" => 1_i64,
"13:string" => "",
"23:varint" => 0_i64,
},
},
}
@@ -849,12 +855,12 @@ def get_about_info(ucid, locale)
related_author_thumbnail = node.xpath_node(%q(.//img)).try &.["data-thumb"]
related_author_thumbnail ||= ""
AboutRelatedChannel.new(
ucid: related_id,
author: related_title,
author_url: related_author_url,
AboutRelatedChannel.new({
ucid: related_id,
author: related_title,
author_url: related_author_url,
author_thumbnail: related_author_thumbnail,
)
})
end
joined = about.xpath_node(%q(//span[contains(., "Joined")]))
@@ -876,23 +882,23 @@ def get_about_info(ucid, locale)
tabs = about.xpath_nodes(%q(//ul[@id="channel-navigation-menu"]/li/a/span)).map { |node| node.content.downcase }
AboutChannel.new(
ucid: ucid,
author: author,
auto_generated: auto_generated,
author_url: author_url,
author_thumbnail: author_thumbnail,
banner: banner,
description_html: description_html,
paid: paid,
total_views: total_views,
sub_count: sub_count,
joined: joined,
AboutChannel.new({
ucid: ucid,
author: author,
auto_generated: auto_generated,
author_url: author_url,
author_thumbnail: author_thumbnail,
banner: banner,
description_html: description_html,
paid: paid,
total_views: total_views,
sub_count: sub_count,
joined: joined,
is_family_friendly: is_family_friendly,
allowed_regions: allowed_regions,
related_channels: related_channels,
tabs: tabs
)
allowed_regions: allowed_regions,
related_channels: related_channels,
tabs: tabs,
})
end
def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")

69
src/invidious/comments.cr
View File

@@ -1,11 +1,23 @@
class RedditThing
JSON.mapping({
kind: String,
data: RedditComment | RedditLink | RedditMore | RedditListing,
})
include JSON::Serializable
property kind : String
property data : RedditComment | RedditLink | RedditMore | RedditListing
end
class RedditComment
include JSON::Serializable
property author : String
property body_html : String
property replies : RedditThing | String
property score : Int32
property depth : Int32
property permalink : String
@[JSON::Field(converter: RedditComment::TimeConverter)]
property created_utc : Time
module TimeConverter
def self.from_json(value : JSON::PullParser) : Time
Time.unix(value.read_float.to_i)
@@ -15,46 +27,33 @@ class RedditComment
json.number(value.to_unix)
end
end
JSON.mapping({
author: String,
body_html: String,
replies: RedditThing | String,
score: Int32,
depth: Int32,
permalink: String,
created_utc: {
type: Time,
converter: RedditComment::TimeConverter,
},
})
end
struct RedditLink
JSON.mapping({
author: String,
score: Int32,
subreddit: String,
num_comments: Int32,
id: String,
permalink: String,
title: String,
})
include JSON::Serializable
property author : String
property score : Int32
property subreddit : String
property num_comments : Int32
property id : String
property permalink : String
property title : String
end
struct RedditMore
JSON.mapping({
children: Array(String),
count: Int32,
depth: Int32,
})
include JSON::Serializable
property children : Array(String)
property count : Int32
property depth : Int32
end
class RedditListing
JSON.mapping({
children: Array(RedditThing),
modhash: String,
})
include JSON::Serializable
property children : Array(RedditThing)
property modhash : String
end
def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, sort_by = "top")

404
src/invidious/helpers/helpers.cr
View File

@@ -1,219 +1,100 @@
require "./macros"
struct Nonce
db_mapping({
nonce: String,
expire: Time,
})
include DB::Serializable
property nonce : String
property expire : Time
end
struct SessionId
db_mapping({
id: String,
email: String,
issued: String,
})
include DB::Serializable
property id : String
property email : String
property issued : String
end
struct Annotation
db_mapping({
id: String,
annotations: String,
})
include DB::Serializable
property id : String
property annotations : String
end
struct ConfigPreferences
module StringToArray
def self.to_json(value : Array(String), json : JSON::Builder)
json.array do
value.each do |element|
json.string element
end
end
end
include YAML::Serializable
def self.from_json(value : JSON::PullParser) : Array(String)
begin
result = [] of String
value.read_array do
result << HTML.escape(value.read_string[0, 100])
end
rescue ex
result = [HTML.escape(value.read_string[0, 100]), ""]
end
property annotations : Bool = false
property annotations_subscribed : Bool = false
property autoplay : Bool = false
property captions : Array(String) = ["", "", ""]
property comments : Array(String) = ["youtube", ""]
property continue : Bool = false
property continue_autoplay : Bool = true
property dark_mode : String = ""
property latest_only : Bool = false
property listen : Bool = false
property local : Bool = false
property locale : String = "en-US"
property max_results : Int32 = 40
property notifications_only : Bool = false
property player_style : String = "invidious"
property quality : String = "hd720"
property default_home : String = "Popular"
property feed_menu : Array(String) = ["Popular", "Trending", "Subscriptions", "Playlists"]
property related_videos : Bool = true
property sort : String = "published"
property speed : Float32 = 1.0_f32
property thin_mode : Bool = false
property unseen_only : Bool = false
property video_loop : Bool = false
property volume : Int32 = 100
result
end
def self.to_yaml(value : Array(String), yaml : YAML::Nodes::Builder)
yaml.sequence do
value.each do |element|
yaml.scalar element
end
end
end
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(String)
begin
unless node.is_a?(YAML::Nodes::Sequence)
node.raise "Expected sequence, not #{node.class}"
end
result = [] of String
node.nodes.each do |item|
unless item.is_a?(YAML::Nodes::Scalar)
node.raise "Expected scalar, not #{item.class}"
end
result << HTML.escape(item.value[0, 100])
end
rescue ex
if node.is_a?(YAML::Nodes::Scalar)
result = [HTML.escape(node.value[0, 100]), ""]
else
result = ["", ""]
end
end
result
end
def to_tuple
{% begin %}
{
{{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}}
}
{% end %}
end
module BoolToString
def self.to_json(value : String, json : JSON::Builder)
json.string value
end
def self.from_json(value : JSON::PullParser) : String
begin
result = value.read_string
if result.empty?
CONFIG.default_user_preferences.dark_mode
else
result
end
rescue ex
if value.read_bool
"dark"
else
"light"
end
end
end
def self.to_yaml(value : String, yaml : YAML::Nodes::Builder)
yaml.scalar value
end
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String
unless node.is_a?(YAML::Nodes::Scalar)
node.raise "Expected scalar, not #{node.class}"
end
case node.value
when "true"
"dark"
when "false"
"light"
when ""
CONFIG.default_user_preferences.dark_mode
else
node.value
end
end
end
yaml_mapping({
annotations: {type: Bool, default: false},
annotations_subscribed: {type: Bool, default: false},
autoplay: {type: Bool, default: false},
captions: {type: Array(String), default: ["", "", ""], converter: StringToArray},
comments: {type: Array(String), default: ["youtube", ""], converter: StringToArray},
continue: {type: Bool, default: false},
continue_autoplay: {type: Bool, default: true},
dark_mode: {type: String, default: "", converter: BoolToString},
latest_only: {type: Bool, default: false},
listen: {type: Bool, default: false},
local: {type: Bool, default: false},
locale: {type: String, default: "en-US"},
max_results: {type: Int32, default: 40},
notifications_only: {type: Bool, default: false},
player_style: {type: String, default: "invidious"},
quality: {type: String, default: "hd720"},
default_home: {type: String, default: "Popular"},
feed_menu: {type: Array(String), default: ["Popular", "Trending", "Subscriptions", "Playlists"]},
related_videos: {type: Bool, default: true},
sort: {type: String, default: "published"},
speed: {type: Float32, default: 1.0_f32},
thin_mode: {type: Bool, default: false},
unseen_only: {type: Bool, default: false},
video_loop: {type: Bool, default: false},
volume: {type: Int32, default: 100},
})
end
struct Config
module ConfigPreferencesConverter
def self.to_yaml(value : Preferences, yaml : YAML::Nodes::Builder)
value.to_yaml(yaml)
end
include YAML::Serializable
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Preferences
Preferences.new(*ConfigPreferences.new(ctx, node).to_tuple)
end
end
property channel_threads : Int32 # Number of threads to use for crawling videos from channels (for updating subscriptions)
property feed_threads : Int32 # Number of threads to use for updating feeds
property db : DBConfig # Database configuration
property full_refresh : Bool # Used for crawling channels: threads should check all videos uploaded by a channel
property https_only : Bool? # Used to tell Invidious it is behind a proxy, so links to resources should be https://
property hmac_key : String? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
property domain : String? # Domain to be used for links to resources on the site where an absolute URL is required
property use_pubsub_feeds : Bool | Int32 = false # Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
property captcha_enabled : Bool = true
property login_enabled : Bool = true
property registration_enabled : Bool = true
property statistics_enabled : Bool = false
property admins : Array(String) = [] of String
property external_port : Int32? = nil
property default_user_preferences : ConfigPreferences
property dmca_content : Array(String) = [] of String # For compliance with DMCA, disables download widget using list of video IDs
property check_tables : Bool = false # Check table integrity, automatically try to add any missing columns, create tables, etc.
property cache_annotations : Bool = false # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards
property banner : String? = nil # Optional banner to be displayed along top of page for announcements, etc.
property hsts : Bool? = true # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely
property disable_proxy : Bool? | Array(String)? = false # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local'
module FamilyConverter
def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder)
case value
when Socket::Family::UNSPEC
yaml.scalar nil
when Socket::Family::INET
yaml.scalar "ipv4"
when Socket::Family::INET6
yaml.scalar "ipv6"
when Socket::Family::UNIX
raise "Invalid socket family #{value}"
end
end
@[YAML::Field(converter: Preferences::FamilyConverter)]
property force_resolve : Socket::Family = Socket::Family::UNSPEC # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
property port : Int32 = 3000 # Port to listen for connections (overrided by command line argument)
property host_binding : String = "0.0.0.0" # Host to bind (overrided by command line argument)
property pool_size : Int32 = 100 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
property admin_email : String = "omarroth@protonmail.com" # Email for bug reports
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family
if node.is_a?(YAML::Nodes::Scalar)
case node.value.downcase
when "ipv4"
Socket::Family::INET
when "ipv6"
Socket::Family::INET6
else
Socket::Family::UNSPEC
end
else
node.raise "Expected scalar, not #{node.class}"
end
end
end
module StringToCookies
def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder)
(value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml)
end
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies
unless node.is_a?(YAML::Nodes::Scalar)
node.raise "Expected scalar, not #{node.class}"
end
cookies = HTTP::Cookies.new
node.value.split(";").each do |cookie|
next if cookie.strip.empty?
name, value = cookie.split("=", 2)
cookies << HTTP::Cookie.new(name.strip, value.strip)
end
cookies
end
end
@[YAML::Field(converter: Preferences::StringToCookies)]
property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format
property captcha_key : String? = nil # Key for Anti-Captcha
def disabled?(option)
case disabled = CONFIG.disable_proxy
@@ -229,50 +110,16 @@ struct Config
return false
end
end
YAML.mapping({
channel_threads: Int32, # Number of threads to use for crawling videos from channels (for updating subscriptions)
feed_threads: Int32, # Number of threads to use for updating feeds
db: DBConfig, # Database configuration
full_refresh: Bool, # Used for crawling channels: threads should check all videos uploaded by a channel
https_only: Bool?, # Used to tell Invidious it is behind a proxy, so links to resources should be https://
hmac_key: String?, # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
domain: String?, # Domain to be used for links to resources on the site where an absolute URL is required
use_pubsub_feeds: {type: Bool | Int32, default: false}, # Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
captcha_enabled: {type: Bool, default: true},
login_enabled: {type: Bool, default: true},
registration_enabled: {type: Bool, default: true},
statistics_enabled: {type: Bool, default: false},
admins: {type: Array(String), default: [] of String},
external_port: {type: Int32?, default: nil},
default_user_preferences: {type: Preferences,
default: Preferences.new(*ConfigPreferences.from_yaml("").to_tuple),
converter: ConfigPreferencesConverter,
},
dmca_content: {type: Array(String), default: [] of String}, # For compliance with DMCA, disables download widget using list of video IDs
check_tables: {type: Bool, default: false}, # Check table integrity, automatically try to add any missing columns, create tables, etc.
cache_annotations: {type: Bool, default: false}, # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards
banner: {type: String?, default: nil}, # Optional banner to be displayed along top of page for announcements, etc.
hsts: {type: Bool?, default: true}, # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely
disable_proxy: {type: Bool? | Array(String)?, default: false}, # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local'
force_resolve: {type: Socket::Family, default: Socket::Family::UNSPEC, converter: FamilyConverter}, # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
port: {type: Int32, default: 3000}, # Port to listen for connections (overrided by command line argument)
host_binding: {type: String, default: "0.0.0.0"}, # Host to bind (overrided by command line argument)
pool_size: {type: Int32, default: 100}, # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
admin_email: {type: String, default: "omarroth@protonmail.com"}, # Email for bug reports
cookies: {type: HTTP::Cookies, default: HTTP::Cookies.new, converter: StringToCookies}, # Saved cookies in "name1=value1; name2=value2..." format
captcha_key: {type: String?, default: nil}, # Key for Anti-Captcha
})
end
struct DBConfig
yaml_mapping({
user: String,
password: String,
host: String,
port: Int32,
dbname: String,
})
include YAML::Serializable
property user : String
property password : String
property host : String
property port : Int32
property dbname : String
end
def login_req(f_req)
@@ -365,20 +212,20 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
end
end
items << SearchVideo.new(
title: title,
id: video_id,
author: author,
ucid: author_id,
published: published,
views: view_count,
description_html: description_html,
length_seconds: length_seconds,
live_now: live_now,
paid: paid,
premium: premium,
premiere_timestamp: premiere_timestamp
)
items << SearchVideo.new({
title: title,
id: video_id,
author: author,
ucid: author_id,
published: published,
views: view_count,
description_html: description_html,
length_seconds: length_seconds,
live_now: live_now,
paid: paid,
premium: premium,
premiere_timestamp: premiere_timestamp,
})
elsif i = item["channelRenderer"]?
author = i["title"]["simpleText"]?.try &.as_s || author_fallback || ""
author_id = i["channelId"]?.try &.as_s || author_id_fallback || ""
@@ -391,15 +238,15 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
video_count = i["videoCountText"]?.try &.["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0
description_html = i["descriptionSnippet"]?.try { |t| parse_content(t) } || ""
items << SearchChannel.new(
author: author,
ucid: author_id,
items << SearchChannel.new({
author: author,
ucid: author_id,
author_thumbnail: author_thumbnail,
subscriber_count: subscriber_count,
video_count: video_count,
video_count: video_count,
description_html: description_html,
auto_generated: auto_generated,
)
auto_generated: auto_generated,
})
elsif i = item["gridPlaylistRenderer"]?
title = i["title"]["runs"].as_a[0]?.try &.["text"].as_s || ""
plid = i["playlistId"]?.try &.as_s || ""
@@ -407,15 +254,15 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
video_count = i["videoCountText"]["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0
playlist_thumbnail = i["thumbnail"]["thumbnails"][0]?.try &.["url"]?.try &.as_s || ""
items << SearchPlaylist.new(
title: title,
id: plid,
author: author_fallback || "",
ucid: author_id_fallback || "",
items << SearchPlaylist.new({
title: title,
id: plid,
author: author_fallback || "",
ucid: author_id_fallback || "",
video_count: video_count,
videos: [] of SearchPlaylistVideo,
thumbnail: playlist_thumbnail
)
videos: [] of SearchPlaylistVideo,
thumbnail: playlist_thumbnail,
})
elsif i = item["playlistRenderer"]?
title = i["title"]["simpleText"]?.try &.as_s || ""
plid = i["playlistId"]?.try &.as_s || ""
@@ -432,24 +279,24 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
v_title = v["title"]["simpleText"]?.try &.as_s || ""
v_id = v["videoId"]?.try &.as_s || ""
v_length_seconds = v["lengthText"]?.try &.["simpleText"]?.try { |t| decode_length_seconds(t.as_s) } || 0
SearchPlaylistVideo.new(
title: v_title,
id: v_id,
length_seconds: v_length_seconds
)
SearchPlaylistVideo.new({
title: v_title,
id: v_id,
length_seconds: v_length_seconds,
})
end || [] of SearchPlaylistVideo
# TODO: i["publishedTimeText"]?
items << SearchPlaylist.new(
title: title,
id: plid,
author: author,
ucid: author_id,
items << SearchPlaylist.new({
title: title,
id: plid,
author: author,
ucid: author_id,
video_count: video_count,
videos: videos,
thumbnail: playlist_thumbnail
)
videos: videos,
thumbnail: playlist_thumbnail,
})
elsif i = item["radioRenderer"]? # Mix
# TODO
elsif i = item["showRenderer"]? # Show
@@ -465,6 +312,7 @@ end
def check_enum(db, logger, enum_name, struct_type = nil)
return # TODO
if !db.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool)
logger.puts("CREATE TYPE #{enum_name}")
@@ -488,7 +336,7 @@ def check_table(db, logger, table_name, struct_type = nil)
return if !struct_type
struct_array = struct_type.to_type_tuple
struct_array = struct_type.type_array
column_array = get_column_array(db, table_name)
column_types = File.read("config/sql/#{table_name}.sql").match(/CREATE TABLE public\.#{table_name}\n\((?<types>[\d\D]*?)\);/)
.try &.["types"].split(",").map { |line| line.strip }.reject &.starts_with?("CONSTRAINT")

2
src/invidious/helpers/jobs.cr
View File

@@ -67,7 +67,7 @@ def refresh_feeds(db, logger, config)
begin
# Drop outdated views
column_array = get_column_array(db, view_name)
ChannelVideo.to_type_tuple.each_with_index do |name, i|
ChannelVideo.type_array.each_with_index do |name, i|
if name != column_array[i]?
logger.puts("DROP MATERIALIZED VIEW #{view_name}")
db.exec("DROP MATERIALIZED VIEW #{view_name}")

76
src/invidious/helpers/macros.cr
View File

@@ -1,43 +1,51 @@
macro db_mapping(mapping)
def initialize({{*mapping.keys.map { |id| "@#{id}".id }}})
end
module DB::Serializable
macro included
{% verbatim do %}
macro finished
def self.type_array
\{{ @type.instance_vars
.reject { |var| var.annotation(::DB::Field) && var.annotation(::DB::Field)[:ignore] }
.map { |name| name.stringify }
}}
end
def to_a
return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ]
end
def initialize(tuple)
\{% for var in @type.instance_vars %}
\{% ann = var.annotation(::DB::Field) %}
\{% if ann && ann[:ignore] %}
\{% else %}
@\{{var.name}} = tuple[:\{{var.name.id}}]
\{% end %}
\{% end %}
end
def self.to_type_tuple
return { {{*mapping.keys.map { |id| "#{id}" }}} }
def to_a
\{{ @type.instance_vars
.reject { |var| var.annotation(::DB::Field) && var.annotation(::DB::Field)[:ignore] }
.map { |name| name }
}}
end
end
{% end %}
end
DB.mapping( {{mapping}} )
end
macro json_mapping(mapping)
def initialize({{*mapping.keys.map { |id| "@#{id}".id }}})
module JSON::Serializable
macro included
{% verbatim do %}
macro finished
def initialize(tuple)
\{% for var in @type.instance_vars %}
\{% ann = var.annotation(::JSON::Field) %}
\{% if ann && ann[:ignore] %}
\{% else %}
@\{{var.name}} = tuple[:\{{var.name.id}}]
\{% end %}
\{% end %}
end
end
{% end %}
end
def to_a
return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ]
end
patched_json_mapping( {{mapping}} )
YAML.mapping( {{mapping}} )
end
macro yaml_mapping(mapping)
def initialize({{*mapping.keys.map { |id| "@#{id}".id }}})
end
def to_a
return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ]
end
def to_tuple
return { {{*mapping.keys.map { |id| "@#{id}".id }}} }
end
YAML.mapping({{mapping}})
end
macro templated(filename, template = "template")

166
src/invidious/helpers/patch_mapping.cr
View File

@@ -1,166 +0,0 @@
# Overloads https://github.com/crystal-lang/crystal/blob/0.28.0/src/json/from_json.cr#L24
def Object.from_json(string_or_io, default) : self
parser = JSON::PullParser.new(string_or_io)
new parser, default
end
# Adds configurable 'default'
macro patched_json_mapping(_properties_, strict = false)
{% for key, value in _properties_ %}
{% _properties_[key] = {type: value} unless value.is_a?(HashLiteral) || value.is_a?(NamedTupleLiteral) %}
{% end %}
{% for key, value in _properties_ %}
{% _properties_[key][:key_id] = key.id.gsub(/\?$/, "") %}
{% end %}
{% for key, value in _properties_ %}
@{{value[:key_id]}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }}
{% if value[:setter] == nil ? true : value[:setter] %}
def {{value[:key_id]}}=(_{{value[:key_id]}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }})
@{{value[:key_id]}} = _{{value[:key_id]}}
end
{% end %}
{% if value[:getter] == nil ? true : value[:getter] %}
def {{key.id}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }}
@{{value[:key_id]}}
end
{% end %}
{% if value[:presence] %}
@{{value[:key_id]}}_present : Bool = false
def {{value[:key_id]}}_present?
@{{value[:key_id]}}_present
end
{% end %}
{% end %}
def initialize(%pull : ::JSON::PullParser, default = nil)
{% for key, value in _properties_ %}
%var{key.id} = nil
%found{key.id} = false
{% end %}
%location = %pull.location
begin
%pull.read_begin_object
rescue exc : ::JSON::ParseException
raise ::JSON::MappingError.new(exc.message, self.class.to_s, nil, *%location, exc)
end
until %pull.kind.end_object?
%key_location = %pull.location
key = %pull.read_object_key
case key
{% for key, value in _properties_ %}
when {{value[:key] || value[:key_id].stringify}}
%found{key.id} = true
begin
%var{key.id} =
{% if value[:nilable] || value[:default] != nil %} %pull.read_null_or { {% end %}
{% if value[:root] %}
%pull.on_key!({{value[:root]}}) do
{% end %}
{% if value[:converter] %}
{{value[:converter]}}.from_json(%pull)
{% elsif value[:type].is_a?(Path) || value[:type].is_a?(Generic) %}
{{value[:type]}}.new(%pull)
{% else %}
::Union({{value[:type]}}).new(%pull)
{% end %}
{% if value[:root] %}
end
{% end %}
{% if value[:nilable] || value[:default] != nil %} } {% end %}
rescue exc : ::JSON::ParseException
raise ::JSON::MappingError.new(exc.message, self.class.to_s, {{value[:key] || value[:key_id].stringify}}, *%key_location, exc)
end
{% end %}
else
{% if strict %}
raise ::JSON::MappingError.new("Unknown JSON attribute: #{key}", self.class.to_s, nil, *%key_location, nil)
{% else %}
%pull.skip
{% end %}
end
end
%pull.read_next
{% for key, value in _properties_ %}
{% unless value[:nilable] || value[:default] != nil %}
if %var{key.id}.nil? && !%found{key.id} && !::Union({{value[:type]}}).nilable?
raise ::JSON::MappingError.new("Missing JSON attribute: {{(value[:key] || value[:key_id]).id}}", self.class.to_s, nil, *%location, nil)
end
{% end %}
{% if value[:nilable] %}
{% if value[:default] != nil %}
@{{value[:key_id]}} = %found{key.id} ? %var{key.id} : (default.responds_to?(:{{value[:key_id]}}) ? default.{{value[:key_id]}} : {{value[:default]}})
{% else %}
@{{value[:key_id]}} = %var{key.id}
{% end %}
{% elsif value[:default] != nil %}
@{{value[:key_id]}} = %var{key.id}.nil? ? (default.responds_to?(:{{value[:key_id]}}) ? default.{{value[:key_id]}} : {{value[:default]}}) : %var{key.id}
{% else %}
@{{value[:key_id]}} = (%var{key.id}).as({{value[:type]}})
{% end %}
{% if value[:presence] %}
@{{value[:key_id]}}_present = %found{key.id}
{% end %}
{% end %}
end
def to_json(json : ::JSON::Builder)
json.object do
{% for key, value in _properties_ %}
_{{value[:key_id]}} = @{{value[:key_id]}}
{% unless value[:emit_null] %}
unless _{{value[:key_id]}}.nil?
{% end %}
json.field({{value[:key] || value[:key_id].stringify}}) do
{% if value[:root] %}
{% if value[:emit_null] %}
if _{{value[:key_id]}}.nil?
nil.to_json(json)
else
{% end %}
json.object do
json.field({{value[:root]}}) do
{% end %}
{% if value[:converter] %}
if _{{value[:key_id]}}
{{ value[:converter] }}.to_json(_{{value[:key_id]}}, json)
else
nil.to_json(json)
end
{% else %}
_{{value[:key_id]}}.to_json(json)
{% end %}
{% if value[:root] %}
{% if value[:emit_null] %}
end
{% end %}
end
end
{% end %}
end
{% unless value[:emit_null] %}
end
{% end %}
{% end %}
end
end
end

57
src/invidious/mixes.cr
View File

@@ -1,21 +1,21 @@
struct MixVideo
db_mapping({
title: String,
id: String,
author: String,
ucid: String,
length_seconds: Int32,
index: Int32,
rdid: String,
})
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property length_seconds : Int32
property index : Int32
property rdid : String
end
struct Mix
db_mapping({
title: String,
id: String,
videos: Array(MixVideo),
})
include DB::Serializable
property title : String
property id : String
property videos : Array(MixVideo)
end
def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
@@ -48,23 +48,22 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
id = item["videoId"].as_s
title = item["title"]?.try &.["simpleText"].as_s
if !title
next
end
next if !title
author = item["longBylineText"]["runs"][0]["text"].as_s
ucid = item["longBylineText"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"].as_s
length_seconds = decode_length_seconds(item["lengthText"]["simpleText"].as_s)
index = item["navigationEndpoint"]["watchEndpoint"]["index"].as_i
videos << MixVideo.new(
title,
id,
author,
ucid,
length_seconds,
index,
rdid
)
videos << MixVideo.new({
title: title,
id: id,
author: author,
ucid: ucid,
length_seconds: length_seconds,
index: index,
rdid: rdid,
})
end
if !cookies
@@ -74,7 +73,11 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
videos.uniq! { |video| video.id }
videos = videos.first(50)
return Mix.new(mix_title, rdid, videos)
return Mix.new({
title: mix_title,
id: rdid,
videos: videos,
})
end
def template_mix(mix)

171
src/invidious/playlists.cr
View File

@@ -1,4 +1,16 @@
struct PlaylistVideo
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property length_seconds : Int32
property published : Time
property plid : String
property index : Int64
property live_now : Bool
def to_xml(auto_generated, xml : XML::Builder)
xml.element("entry") do
xml.element("id") { xml.text "yt:video:#{self.id}" }
@@ -78,21 +90,22 @@ struct PlaylistVideo
end
end
end
db_mapping({
title: String,
id: String,
author: String,
ucid: String,
length_seconds: Int32,
published: Time,
plid: String,
index: Int64,
live_now: Bool,
})
end
struct Playlist
include DB::Serializable
property title : String
property id : String
property author : String
property author_thumbnail : String
property ucid : String
property description : String
property video_count : Int32
property views : Int64
property updated : Time
property thumbnail : String?
def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
json.object do
json.field "type", "playlist"
@@ -147,19 +160,6 @@ struct Playlist
end
end
db_mapping({
title: String,
id: String,
author: String,
author_thumbnail: String,
ucid: String,
description: String,
video_count: Int32,
views: Int64,
updated: Time,
thumbnail: String?,
})
def privacy
PlaylistPrivacy::Public
end
@@ -176,6 +176,29 @@ enum PlaylistPrivacy
end
struct InvidiousPlaylist
include DB::Serializable
property title : String
property id : String
property author : String
property description : String = ""
property video_count : Int32
property created : Time
property updated : Time
@[DB::Field(converter: InvidiousPlaylist::PlaylistPrivacyConverter)]
property privacy : PlaylistPrivacy = PlaylistPrivacy::Private
property index : Array(Int64)
@[DB::Field(ignore: true)]
property thumbnail_id : String?
module PlaylistPrivacyConverter
def self.from_rs(rs)
return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8))))
end
end
def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
json.object do
json.field "type", "invidiousPlaylist"
@@ -216,26 +239,6 @@ struct InvidiousPlaylist
end
end
property thumbnail_id
module PlaylistPrivacyConverter
def self.from_rs(rs)
return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8))))
end
end
db_mapping({
title: String,
id: String,
author: String,
description: {type: String, default: ""},
video_count: Int32,
created: Time,
updated: Time,
privacy: {type: PlaylistPrivacy, default: PlaylistPrivacy::Private, converter: PlaylistPrivacyConverter},
index: Array(Int64),
})
def thumbnail
@thumbnail_id ||= PG_DB.query_one?("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 1", self.id, self.index, as: String) || "-----------"
"/vi/#{@thumbnail_id}/mqdefault.jpg"
@@ -261,17 +264,17 @@ end
def create_playlist(db, title, privacy, user)
plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}"
playlist = InvidiousPlaylist.new(
title: title.byte_slice(0, 150),
id: plid,
author: user.email,
playlist = InvidiousPlaylist.new({
title: title.byte_slice(0, 150),
id: plid,
author: user.email,
description: "", # Max 5000 characters
video_count: 0,
created: Time.utc,
updated: Time.utc,
privacy: privacy,
index: [] of Int64,
)
created: Time.utc,
updated: Time.utc,
privacy: privacy,
index: [] of Int64,
})
playlist_array = playlist.to_a
args = arg_array(playlist_array)
@@ -282,17 +285,17 @@ def create_playlist(db, title, privacy, user)
end
def subscribe_playlist(db, user, playlist)
playlist = InvidiousPlaylist.new(
title: playlist.title.byte_slice(0, 150),
id: playlist.id,
author: user.email,
playlist = InvidiousPlaylist.new({
title: playlist.title.byte_slice(0, 150),
id: playlist.id,
author: user.email,
description: "", # Max 5000 characters
video_count: playlist.video_count,
created: Time.utc,
updated: playlist.updated,
privacy: PlaylistPrivacy::Private,
index: [] of Int64,
)
created: Time.utc,
updated: playlist.updated,
privacy: PlaylistPrivacy::Private,
index: [] of Int64,
})
playlist_array = playlist.to_a
args = arg_array(playlist_array)
@@ -393,18 +396,18 @@ def fetch_playlist(plid, locale)
author = author_info["title"]["runs"][0]["text"]?.try &.as_s || ""
ucid = author_info["title"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"]?.try &.as_s || ""
return Playlist.new(
title: title,
id: plid,
author: author,
return Playlist.new({
title: title,
id: plid,
author: author,
author_thumbnail: author_thumbnail,
ucid: ucid,
description: description,
video_count: video_count,
views: views,
updated: updated,
thumbnail: thumbnail
)
ucid: ucid,
description: description,
video_count: video_count,
views: views,
updated: updated,
thumbnail: thumbnail,
})
end
def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil)
@@ -471,17 +474,17 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
length_seconds = 0
end
videos << PlaylistVideo.new(
title: title,
id: video_id,
author: author,
ucid: ucid,
videos << PlaylistVideo.new({
title: title,
id: video_id,
author: author,
ucid: ucid,
length_seconds: length_seconds,
published: Time.utc,
plid: plid,
live_now: live,
index: index - 1
)
published: Time.utc,
plid: plid,
live_now: live,
index: index - 1,
})
end
end

80
src/invidious/search.cr
View File

@@ -1,4 +1,19 @@
struct SearchVideo
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property published : Time
property views : Int64
property description_html : String
property length_seconds : Int32
property live_now : Bool
property paid : Bool
property premium : Bool
property premiere_timestamp : Time?
def to_xml(auto_generated, query_params, xml : XML::Builder)
query_params["v"] = self.id
@@ -99,32 +114,27 @@ struct SearchVideo
def is_upcoming
premiere_timestamp ? true : false
end
db_mapping({
title: String,
id: String,
author: String,
ucid: String,
published: Time,
views: Int64,
description_html: String,
length_seconds: Int32,
live_now: Bool,
paid: Bool,
premium: Bool,
premiere_timestamp: Time?,
})
end
struct SearchPlaylistVideo
db_mapping({
title: String,
id: String,
length_seconds: Int32,
})
include DB::Serializable
property title : String
property id : String
property length_seconds : Int32
end
struct SearchPlaylist
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property video_count : Int32
property videos : Array(SearchPlaylistVideo)
property thumbnail : String?
def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "playlist"
@@ -164,19 +174,19 @@ struct SearchPlaylist
end
end
end
db_mapping({
title: String,
id: String,
author: String,
ucid: String,
video_count: Int32,
videos: Array(SearchPlaylistVideo),
thumbnail: String?,
})
end
struct SearchChannel
include DB::Serializable
property author : String
property ucid : String
property author_thumbnail : String
property subscriber_count : Int32
property video_count : Int32
property description_html : String
property auto_generated : Bool
def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "channel"
@@ -216,16 +226,6 @@ struct SearchChannel
end
end
end
db_mapping({
author: String,
ucid: String,
author_thumbnail: String,
subscriber_count: Int32,
video_count: Int32,
description_html: String,
auto_generated: Bool,
})
end
alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist

333
src/invidious/users.cr
View File

@@ -4,6 +4,20 @@ require "crypto/bcrypt/password"
MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" }
struct User
include DB::Serializable
property updated : Time
property notifications : Array(String)
property subscriptions : Array(String)
property email : String
@[DB::Field(converter: User::PreferencesConverter)]
property preferences : Preferences
property password : String?
property token : String
property watched : Array(String)
property feed_needs_update : Bool?
module PreferencesConverter
def self.from_rs(rs)
begin
@@ -13,31 +27,78 @@ struct User
end
end
end
db_mapping({
updated: Time,
notifications: Array(String),
subscriptions: Array(String),
email: String,
preferences: {
type: Preferences,
converter: PreferencesConverter,
},
password: String?,
token: String,
watched: Array(String),
feed_needs_update: Bool?,
})
end
struct Preferences
module ProcessString
include JSON::Serializable
include YAML::Serializable
property annotations : Bool = CONFIG.default_user_preferences.annotations
property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed
property autoplay : Bool = CONFIG.default_user_preferences.autoplay
@[JSON::Field(converter: Preferences::StringToArray)]
@[YAML::Field(converter: Preferences::StringToArray)]
property captions : Array(String) = CONFIG.default_user_preferences.captions
@[JSON::Field(converter: Preferences::StringToArray)]
@[YAML::Field(converter: Preferences::StringToArray)]
property comments : Array(String) = CONFIG.default_user_preferences.comments
property continue : Bool = CONFIG.default_user_preferences.continue
property continue_autoplay : Bool = CONFIG.default_user_preferences.continue_autoplay
@[JSON::Field(converter: Preferences::BoolToString)]
@[YAML::Field(converter: Preferences::BoolToString)]
property dark_mode : String = CONFIG.default_user_preferences.dark_mode
property latest_only : Bool = CONFIG.default_user_preferences.latest_only
property listen : Bool = CONFIG.default_user_preferences.listen
property local : Bool = CONFIG.default_user_preferences.local
@[JSON::Field(converter: Preferences::ProcessString)]
property locale : String = CONFIG.default_user_preferences.locale
@[JSON::Field(converter: Preferences::ClampInt)]
property max_results : Int32 = CONFIG.default_user_preferences.max_results
property notifications_only : Bool = CONFIG.default_user_preferences.notifications_only
@[JSON::Field(converter: Preferences::ProcessString)]
property player_style : String = CONFIG.default_user_preferences.player_style
@[JSON::Field(converter: Preferences::ProcessString)]
property quality : String = CONFIG.default_user_preferences.quality
property default_home : String = CONFIG.default_user_preferences.default_home
property feed_menu : Array(String) = CONFIG.default_user_preferences.feed_menu
property related_videos : Bool = CONFIG.default_user_preferences.related_videos
@[JSON::Field(converter: Preferences::ProcessString)]
property sort : String = CONFIG.default_user_preferences.sort
property speed : Float32 = CONFIG.default_user_preferences.speed
property thin_mode : Bool = CONFIG.default_user_preferences.thin_mode
property unseen_only : Bool = CONFIG.default_user_preferences.unseen_only
property video_loop : Bool = CONFIG.default_user_preferences.video_loop
property volume : Int32 = CONFIG.default_user_preferences.volume
module BoolToString
def self.to_json(value : String, json : JSON::Builder)
json.string value
end
def self.from_json(value : JSON::PullParser) : String
HTML.escape(value.read_string[0, 100])
begin
result = value.read_string
if result.empty?
CONFIG.default_user_preferences.dark_mode
else
result
end
rescue ex
if value.read_bool
"dark"
else
"light"
end
end
end
def self.to_yaml(value : String, yaml : YAML::Nodes::Builder)
@@ -45,7 +106,20 @@ struct Preferences
end
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String
HTML.escape(node.value[0, 100])
unless node.is_a?(YAML::Nodes::Scalar)
node.raise "Expected scalar, not #{node.class}"
end
case node.value
when "true"
"dark"
when "false"
"light"
when ""
CONFIG.default_user_preferences.dark_mode
else
node.value
end
end
end
@@ -67,33 +141,130 @@ struct Preferences
end
end
json_mapping({
annotations: {type: Bool, default: CONFIG.default_user_preferences.annotations},
annotations_subscribed: {type: Bool, default: CONFIG.default_user_preferences.annotations_subscribed},
autoplay: {type: Bool, default: CONFIG.default_user_preferences.autoplay},
captions: {type: Array(String), default: CONFIG.default_user_preferences.captions, converter: ConfigPreferences::StringToArray},
comments: {type: Array(String), default: CONFIG.default_user_preferences.comments, converter: ConfigPreferences::StringToArray},
continue: {type: Bool, default: CONFIG.default_user_preferences.continue},
continue_autoplay: {type: Bool, default: CONFIG.default_user_preferences.continue_autoplay},
dark_mode: {type: String, default: CONFIG.default_user_preferences.dark_mode, converter: ConfigPreferences::BoolToString},
latest_only: {type: Bool, default: CONFIG.default_user_preferences.latest_only},
listen: {type: Bool, default: CONFIG.default_user_preferences.listen},
local: {type: Bool, default: CONFIG.default_user_preferences.local},
locale: {type: String, default: CONFIG.default_user_preferences.locale, converter: ProcessString},
max_results: {type: Int32, default: CONFIG.default_user_preferences.max_results, converter: ClampInt},
notifications_only: {type: Bool, default: CONFIG.default_user_preferences.notifications_only},
player_style: {type: String, default: CONFIG.default_user_preferences.player_style, converter: ProcessString},
quality: {type: String, default: CONFIG.default_user_preferences.quality, converter: ProcessString},
default_home: {type: String, default: CONFIG.default_user_preferences.default_home},
feed_menu: {type: Array(String), default: CONFIG.default_user_preferences.feed_menu},
related_videos: {type: Bool, default: CONFIG.default_user_preferences.related_videos},
sort: {type: String, default: CONFIG.default_user_preferences.sort, converter: ProcessString},
speed: {type: Float32, default: CONFIG.default_user_preferences.speed},
thin_mode: {type: Bool, default: CONFIG.default_user_preferences.thin_mode},
unseen_only: {type: Bool, default: CONFIG.default_user_preferences.unseen_only},
video_loop: {type: Bool, default: CONFIG.default_user_preferences.video_loop},
volume: {type: Int32, default: CONFIG.default_user_preferences.volume},
})
module FamilyConverter
def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder)
case value
when Socket::Family::UNSPEC
yaml.scalar nil
when Socket::Family::INET
yaml.scalar "ipv4"
when Socket::Family::INET6
yaml.scalar "ipv6"
when Socket::Family::UNIX
raise "Invalid socket family #{value}"
end
end
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family
if node.is_a?(YAML::Nodes::Scalar)
case node.value.downcase
when "ipv4"
Socket::Family::INET
when "ipv6"
Socket::Family::INET6
else
Socket::Family::UNSPEC
end
else
node.raise "Expected scalar, not #{node.class}"
end
end
end
module ProcessString
def self.to_json(value : String, json : JSON::Builder)
json.string value
end
def self.from_json(value : JSON::PullParser) : String
HTML.escape(value.read_string[0, 100])
end
def self.to_yaml(value : String, yaml : YAML::Nodes::Builder)
yaml.scalar value
end
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String
HTML.escape(node.value[0, 100])
end
end
module StringToArray
def self.to_json(value : Array(String), json : JSON::Builder)
json.array do
value.each do |element|
json.string element
end
end
end
def self.from_json(value : JSON::PullParser) : Array(String)
begin
result = [] of String
value.read_array do
result << HTML.escape(value.read_string[0, 100])
end
rescue ex
result = [HTML.escape(value.read_string[0, 100]), ""]
end
result
end
def self.to_yaml(value : Array(String), yaml : YAML::Nodes::Builder)
yaml.sequence do
value.each do |element|
yaml.scalar element
end
end
end
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(String)
begin
unless node.is_a?(YAML::Nodes::Sequence)
node.raise "Expected sequence, not #{node.class}"
end
result = [] of String
node.nodes.each do |item|
unless item.is_a?(YAML::Nodes::Scalar)
node.raise "Expected scalar, not #{item.class}"
end
result << HTML.escape(item.value[0, 100])
end
rescue ex
if node.is_a?(YAML::Nodes::Scalar)
result = [HTML.escape(node.value[0, 100]), ""]
else
result = ["", ""]
end
end
result
end
end
module StringToCookies
def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder)
(value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml)
end
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies
unless node.is_a?(YAML::Nodes::Scalar)
node.raise "Expected scalar, not #{node.class}"
end
cookies = HTTP::Cookies.new
node.value.split(";").each do |cookie|
next if cookie.strip.empty?
name, value = cookie.split("=", 2)
cookies << HTTP::Cookie.new(name.strip, value.strip)
end
cookies
end
end
end
def get_user(sid, headers, db, refresh = true)
@@ -103,8 +274,7 @@ def get_user(sid, headers, db, refresh = true)
if refresh && Time.utc - user.updated > 1.minute
user, sid = fetch_user(sid, headers, db)
user_array = user.to_a
user_array[4] = user_array[4].to_json
user_array[4] = user_array[4].to_json # User preferences
args = arg_array(user_array)
db.exec("INSERT INTO users VALUES (#{args}) \
@@ -122,8 +292,7 @@ def get_user(sid, headers, db, refresh = true)
else
user, sid = fetch_user(sid, headers, db)
user_array = user.to_a
user_array[4] = user_array[4].to_json
user_array[4] = user_array[4].to_json # User preferences
args = arg_array(user.to_a)
db.exec("INSERT INTO users VALUES (#{args}) \
@@ -166,7 +335,17 @@ def fetch_user(sid, headers, db)
token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
user = User.new(Time.utc, [] of String, channels, email, CONFIG.default_user_preferences, nil, token, [] of String, true)
user = User.new({
updated: Time.utc,
notifications: [] of String,
subscriptions: channels,
email: email,
preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple),
password: nil,
token: token,
watched: [] of String,
feed_needs_update: true,
})
return user, sid
end
@@ -174,7 +353,17 @@ def create_user(sid, email, password)
password = Crypto::Bcrypt::Password.create(password, cost: 10)
token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
user = User.new(Time.utc, [] of String, [] of String, email, CONFIG.default_user_preferences, password.to_s, token, [] of String, true)
user = User.new({
updated: Time.utc,
notifications: [] of String,
subscriptions: [] of String,
email: email,
preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple),
password: password.to_s,
token: token,
watched: [] of String,
feed_needs_update: true,
})
return user, sid
end
@@ -281,48 +470,6 @@ def subscribe_ajax(channel_id, action, env_headers)
end
end
# TODO: Playlist stub, sync with YouTube for Google accounts
# def playlist_ajax(video_ids, source_playlist_id, name, privacy, action, env_headers)
# headers = HTTP::Headers.new
# headers["Cookie"] = env_headers["Cookie"]
#
# html = YT_POOL.client &.get("/view_all_playlists", headers)
#
# cookies = HTTP::Cookies.from_headers(headers)
# html.cookies.each do |cookie|
# if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name
# if cookies[cookie.name]?
# cookies[cookie.name] = cookie
# else
# cookies << cookie
# end
# end
# end
# headers = cookies.add_request_headers(headers)
#
# if match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[^"]+)"/)
# session_token = match["session_token"]
#
# headers["content-type"] = "application/x-www-form-urlencoded"
#
# post_req = {
# video_ids: [] of String,
# source_playlist_id: "",
# n: name,
# p: privacy,
# session_token: session_token,
# }
# post_url = "/playlist_ajax?#{action}=1"
#
# response = client.post(post_url, headers, form: post_req)
# if response.status_code == 200
# return JSON.parse(response.body)["result"]["playlistId"].as_s
# else
# return nil
# end
# end
# end
def get_subscription_feed(db, user, max_results = 40, page = 1)
limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE)
offset = (page - 1) * limit

142
src/invidious/videos.cr
View File

@@ -222,30 +222,50 @@ VIDEO_FORMATS = {
}
struct VideoPreferences
json_mapping({
annotations: Bool,
autoplay: Bool,
comments: Array(String),
continue: Bool,
continue_autoplay: Bool,
controls: Bool,
listen: Bool,
local: Bool,
preferred_captions: Array(String),
player_style: String,
quality: String,
raw: Bool,
region: String?,
related_videos: Bool,
speed: (Float32 | Float64),
video_end: (Float64 | Int32),
video_loop: Bool,
video_start: (Float64 | Int32),
volume: Int32,
})
include JSON::Serializable
property annotations : Bool
property autoplay : Bool
property comments : Array(String)
property continue : Bool
property continue_autoplay : Bool
property controls : Bool
property listen : Bool
property local : Bool
property preferred_captions : Array(String)
property player_style : String
property quality : String
property raw : Bool
property region : String?
property related_videos : Bool
property speed : Float32 | Float64
property video_end : Float64 | Int32
property video_loop : Bool
property video_start : Float64 | Int32
property volume : Int32
end
struct Video
include DB::Serializable
property id : String
@[DB::Field(converter: Video::JSONConverter)]
property info : Hash(String, JSON::Any)
property updated : Time
@[DB::Field(ignore: true)]
property captions : Array(Caption)?
@[DB::Field(ignore: true)]
property adaptive_fmts : Array(Hash(String, JSON::Any))?
@[DB::Field(ignore: true)]
property fmt_stream : Array(Hash(String, JSON::Any))?
@[DB::Field(ignore: true)]
property description : String?
module JSONConverter
def self.from_rs(rs)
JSON.parse(rs.read(String)).as_h
@@ -552,6 +572,7 @@ struct Video
def fmt_stream
return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream
fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
fmt_stream.each do |fmt|
if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
@@ -751,30 +772,20 @@ struct Video
def session_token : String?
info["sessionToken"]?.try &.as_s?
end
db_mapping({
id: String,
info: {type: Hash(String, JSON::Any), converter: Video::JSONConverter},
updated: Time,
})
@captions : Array(Caption)?
@adaptive_fmts : Array(Hash(String, JSON::Any))?
@fmt_stream : Array(Hash(String, JSON::Any))?
end
struct Caption
json_mapping({
name: CaptionName,
baseUrl: String,
languageCode: String,
})
end
struct CaptionName
json_mapping({
simpleText: String,
})
include JSON::Serializable
property simpleText : String
end
struct Caption
include JSON::Serializable
property name : CaptionName
property baseUrl : String
property languageCode : String
end
class VideoRedirect < Exception
@@ -990,7 +1001,12 @@ def fetch_video(id, region)
raise info["reason"]?.try &.as_s || "" if !info["videoDetails"]?
video = Video.new(id, info, Time.utc)
video = Video.new({
id: id,
info: info,
updated: Time.utc,
})
return video
end
@@ -1097,27 +1113,27 @@ def process_video_params(query, preferences)
controls ||= 1
controls = controls >= 1
params = VideoPreferences.new(
annotations: annotations,
autoplay: autoplay,
comments: comments,
continue: continue,
continue_autoplay: continue_autoplay,
controls: controls,
listen: listen,
local: local,
player_style: player_style,
params = VideoPreferences.new({
annotations: annotations,
autoplay: autoplay,
comments: comments,
continue: continue,
continue_autoplay: continue_autoplay,
controls: controls,
listen: listen,
local: local,
player_style: player_style,
preferred_captions: preferred_captions,
quality: quality,
raw: raw,
region: region,
related_videos: related_videos,
speed: speed,
video_end: video_end,
video_loop: video_loop,
video_start: video_start,
volume: volume,
)
quality: quality,
raw: raw,
region: region,
related_videos: related_videos,
speed: speed,
video_end: video_end,
video_loop: video_loop,
video_start: video_start,
volume: volume,
})
return params
end