Merge branch 'iv-org:master' into youtube-playlist-import

This commit is contained in:
Gavin
2023-04-03 17:09:34 -07:00
committed by GitHub
96 changed files with 1668 additions and 581 deletions

View File

@@ -34,6 +34,7 @@ require "protodec/utils"
require "./invidious/database/*"
require "./invidious/database/migrations/*"
require "./invidious/http_server/*"
require "./invidious/helpers/*"
require "./invidious/yt_backend/*"
require "./invidious/frontend/*"

View File

@@ -1,3 +1,5 @@
private IMAGE_QUALITIES = {320, 560, 640, 1280, 2000}
# TODO: Add "sort_by"
def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
response = YT_POOL.client &.get("/channel/#{ucid}/community?gl=US&hl=en")
@@ -69,7 +71,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
next if !post
content_html = post["contentText"]?.try { |t| parse_content(t) } || ""
author = post["authorText"]?.try &.["simpleText"]? || ""
author = post["authorText"]["runs"]?.try &.[0]?.try &.["text"]? || ""
json.object do
json.field "author", author
@@ -108,6 +110,8 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
like_count = post["actionButtons"]["commentActionButtonsRenderer"]["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"]
.try &.as_s.gsub(/\D/, "").to_i? || 0
reply_count = short_text_to_number(post.dig?("actionButtons", "commentActionButtonsRenderer", "replyButton", "buttonRenderer", "text", "simpleText").try &.as_s || "0")
json.field "content", html_to_content(content_html)
json.field "contentHtml", content_html
@@ -115,6 +119,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale))
json.field "likeCount", like_count
json.field "replyCount", reply_count
json.field "commentId", post["postId"]? || post["commentId"]? || ""
json.field "authorIsChannelOwner", post["authorEndpoint"]["browseEndpoint"]["browseId"] == ucid
@@ -174,9 +179,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
aspect_ratio = (width.to_f / height.to_f)
url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640")
qualities = {320, 560, 640, 1280, 2000}
qualities.each do |quality|
IMAGE_QUALITIES.each do |quality|
json.object do
json.field "url", url.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
@@ -185,10 +188,63 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
end
end
end
# TODO
# when .has_key?("pollRenderer")
# attachment = attachment["pollRenderer"]
# json.field "type", "poll"
when .has_key?("pollRenderer")
attachment = attachment["pollRenderer"]
json.field "type", "poll"
json.field "totalVotes", short_text_to_number(attachment["totalVotes"]["simpleText"].as_s.split(" ")[0])
json.field "choices" do
json.array do
attachment["choices"].as_a.each do |choice|
json.object do
json.field "text", choice.dig("text", "runs", 0, "text").as_s
# A choice can have an image associated with it.
# Ex post: https://www.youtube.com/post/UgkxD4XavXUD4NQiddJXXdohbwOwcVqrH9Re
if choice["image"]?
thumbnail = choice["image"]["thumbnails"][0].as_h
width = thumbnail["width"].as_i
height = thumbnail["height"].as_i
aspect_ratio = (width.to_f / height.to_f)
url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640")
json.field "image" do
json.array do
IMAGE_QUALITIES.each do |quality|
json.object do
json.field "url", url.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", (quality / aspect_ratio).ceil.to_i
end
end
end
end
end
end
end
end
end
when .has_key?("postMultiImageRenderer")
attachment = attachment["postMultiImageRenderer"]
json.field "type", "multiImage"
json.field "images" do
json.array do
attachment["images"].as_a.each do |image|
json.array do
thumbnail = image["backstageImageRenderer"]["image"]["thumbnails"][0].as_h
width = thumbnail["width"].as_i
height = thumbnail["height"].as_i
aspect_ratio = (width.to_f / height.to_f)
url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640")
IMAGE_QUALITIES.each do |quality|
json.object do
json.field "url", url.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", (quality / aspect_ratio).ceil.to_i
end
end
end
end
end
end
else
json.field "type", "unknown"
json.field "error", "Unrecognized attachment type."

View File

@@ -30,7 +30,9 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
"15:embedded" => {
"1:embedded" => {
"1:string" => object_inner_2_encoded,
"2:string" => "00000000-0000-0000-0000-000000000000",
},
"2:embedded" => {
"1:string" => "00000000-0000-0000-0000-000000000000",
},
"3:varint" => sort_by_numerical,
},

View File

@@ -181,6 +181,12 @@ def fetch_youtube_comments(id, cursor, format, locale, thin_mode, region, sort_b
json.field "content", html_to_content(content_html)
json.field "contentHtml", content_html
json.field "isPinned", (node_comment["pinnedCommentBadge"]? != nil)
json.field "isSponsor", (node_comment["sponsorCommentBadge"]? != nil)
if node_comment["sponsorCommentBadge"]?
# Sponsor icon thumbnails always have one object and there's only ever the url property in it
json.field "sponsorIconUrl", node_comment.dig("sponsorCommentBadge", "sponsorCommentBadgeRenderer", "customBadge", "thumbnails", 0, "url").to_s
end
json.field "published", published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale))
@@ -322,11 +328,21 @@ def template_youtube_comments(comments, locale, thin_mode, is_replies = false)
end
author_name = HTML.escape(child["author"].as_s)
sponsor_icon = ""
if child["verified"]?.try &.as_bool && child["authorIsChannelOwner"]?.try &.as_bool
author_name += "&nbsp;<i class=\"icon ion ion-md-checkmark-circle\"></i>"
elsif child["verified"]?.try &.as_bool
author_name += "&nbsp;<i class=\"icon ion ion-md-checkmark\"></i>"
end
if child["isSponsor"]?.try &.as_bool
sponsor_icon = String.build do |str|
str << %(<img alt="" )
str << %(src="/ggpht) << URI.parse(child["sponsorIconUrl"].as_s).request_target << "\" "
str << %(title=") << translate(locale, "Channel Sponsor") << "\" "
str << %(width="16" height="16" />)
end
end
html << <<-END_HTML
<div class="pure-g" style="width:100%">
<div class="channel-profile pure-u-4-24 pure-u-md-2-24">
@@ -337,6 +353,7 @@ def template_youtube_comments(comments, locale, thin_mode, is_replies = false)
<b>
<a class="#{child["authorIsChannelOwner"] == true ? "channel-owner" : ""}" href="#{child["authorUrl"]}">#{author_name}</a>
</b>
#{sponsor_icon}
<p style="white-space:pre-wrap">#{child["contentHtml"]}</p>
END_HTML
@@ -670,8 +687,30 @@ def content_to_comment_html(content, video_id : String? = "")
end
text = "<b>#{text}</b>" if run["bold"]?
text = "<s>#{text}</s>" if run["strikethrough"]?
text = "<i>#{text}</i>" if run["italics"]?
# check for custom emojis
if run["emoji"]?
if run["emoji"]["isCustomEmoji"]?.try &.as_bool
if emojiImage = run.dig?("emoji", "image")
emojiAlt = emojiImage.dig?("accessibility", "accessibilityData", "label").try &.as_s || text
emojiThumb = emojiImage["thumbnails"][0]
text = String.build do |str|
str << %(<img alt=") << emojiAlt << "\" "
str << %(src="/ggpht) << URI.parse(emojiThumb["url"].as_s).request_target << "\" "
str << %(title=") << emojiAlt << "\" "
str << %(width=") << emojiThumb["width"] << "\" "
str << %(height=") << emojiThumb["height"] << "\" "
str << %(class="channel-emoji"/>)
end
else
# Hide deleted channel emoji
text = ""
end
end
end
text
end

View File

@@ -20,7 +20,7 @@ module Invidious::Frontend::WatchPage
def download_widget(locale : String, video : Video, video_assets : VideoAssets) : String
if CONFIG.disabled?("downloads")
return "<p id=\"download\">#{translate(locale, "Download is disabled.")}</p>"
return "<p id=\"download\">#{translate(locale, "Download is disabled")}</p>"
end
return String.build(4000) do |str|

View File

@@ -20,7 +20,7 @@ module JSONFilter
/^\(|\(\(|\/\(/
end
def self.parse_fields(fields_text : String) : Nil
def self.parse_fields(fields_text : String, &) : Nil
if fields_text.empty?
raise FieldsParser::ParseError.new "Fields is empty"
end
@@ -42,7 +42,7 @@ module JSONFilter
parse_nest_groups(fields_text) { |nest_list| yield nest_list }
end
def self.parse_single_nests(fields_text : String) : Nil
def self.parse_single_nests(fields_text : String, &) : Nil
single_nests = remove_nest_groups(fields_text)
if !single_nests.empty?
@@ -60,7 +60,7 @@ module JSONFilter
end
end
def self.parse_nest_groups(fields_text : String) : Nil
def self.parse_nest_groups(fields_text : String, &) : Nil
nest_stack = [] of NamedTuple(group_name: String, closing_bracket_index: Int64)
bracket_pairs = get_bracket_pairs(fields_text, true)

View File

@@ -74,6 +74,7 @@ struct SearchVideo
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "authorVerified", self.author_verified
json.field "videoThumbnails" do
Invidious::JSONify::APIv1.thumbnails(json, self.id)

View File

@@ -162,7 +162,7 @@ def number_with_separator(number)
end
def short_text_to_number(short_text : String) : Int64
matches = /(?<number>\d+(\.\d+)?)\s?(?<suffix>[mMkKbB])?/.match(short_text)
matches = /(?<number>\d+(\.\d+)?)\s?(?<suffix>[mMkKbB]?)/.match(short_text)
number = matches.try &.["number"].to_f || 0.0
case matches.try &.["suffix"].downcase
@@ -259,7 +259,7 @@ def get_referer(env, fallback = "/", unroll = true)
end
referer = referer.request_target
referer = "/" + referer.gsub(/[^\/?@&%=\-_.0-9a-zA-Z]/, "").lstrip("/\\")
referer = "/" + referer.gsub(/[^\/?@&%=\-_.:,0-9a-zA-Z]/, "").lstrip("/\\")
if referer == env.request.path
referer = fallback

View File

@@ -0,0 +1,20 @@
module Invidious::HttpServer
module Utils
extend self
def proxy_video_url(raw_url : String, *, region : String? = nil, absolute : Bool = false)
url = URI.parse(raw_url)
# Add some URL parameters
params = url.query_params
params["host"] = url.host.not_nil! # Should never be nil, in theory
params["region"] = region if !region.nil?
if absolute
return "#{HOST_URL}#{url.request_target}?#{params}"
else
return "#{url.request_target}?#{params}"
end
end
end
end

View File

@@ -3,7 +3,7 @@ require "json"
module Invidious::JSONify::APIv1
extend self
def video(video : Video, json : JSON::Builder, *, locale : String?)
def video(video : Video, json : JSON::Builder, *, locale : String?, proxy : Bool = false)
json.object do
json.field "type", video.video_type
@@ -89,7 +89,14 @@ module Invidious::JSONify::APIv1
# Not available on MPEG-4 Timed Text (`text/mp4`) streams (livestreams only)
json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]?
json.field "url", fmt["url"]
if proxy
json.field "url", Invidious::HttpServer::Utils.proxy_video_url(
fmt["url"].to_s, absolute: true
)
else
json.field "url", fmt["url"]
end
json.field "itag", fmt["itag"].as_i.to_s
json.field "type", fmt["mimeType"]
json.field "clen", fmt["contentLength"]? || "-1"
@@ -190,6 +197,21 @@ module Invidious::JSONify::APIv1
end
end
if !video.music.empty?
json.field "musicTracks" do
json.array do
video.music.each do |music|
json.object do
json.field "song", music.song
json.field "artist", music.artist
json.field "album", music.album
json.field "license", music.license
end
end
end
end
end
json.field "recommendedVideos" do
json.array do
video.related_videos.each do |rv|

View File

@@ -203,7 +203,7 @@ module Invidious::Routes::Account
referer = get_referer(env)
if !user
return env.redirect referer
return env.redirect "/login?referer=#{URI.encode_path_segment(env.request.resource)}"
end
user = user.as(User)
@@ -262,6 +262,7 @@ module Invidious::Routes::Account
end
query["token"] = access_token
query["username"] = URI.encode_path_segment(user.email)
url.query = query.to_s
env.redirect url.to_s

View File

@@ -29,7 +29,7 @@ module Invidious::Routes::API::Manifest
if local
uri = URI.parse(url)
url = "#{uri.request_target}host/#{uri.host}/"
url = "#{HOST_URL}#{uri.request_target}host/#{uri.host}/"
end
"<BaseURL>#{url}</BaseURL>"
@@ -42,7 +42,7 @@ module Invidious::Routes::API::Manifest
if local
adaptive_fmts.each do |fmt|
fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target)
fmt["url"] = JSON::Any.new("#{HOST_URL}#{URI.parse(fmt["url"].as_s).request_target}")
end
end

View File

@@ -31,6 +31,88 @@ module Invidious::Routes::API::V1::Authenticated
env.response.status_code = 204
end
def self.export_invidious(env)
env.response.content_type = "application/json"
user = env.get("user").as(User)
return Invidious::User::Export.to_invidious(user)
end
def self.import_invidious(env)
user = env.get("user").as(User)
begin
if body = env.request.body
body = env.request.body.not_nil!.gets_to_end
else
body = "{}"
end
Invidious::User::Import.from_invidious(user, body)
rescue
end
env.response.status_code = 204
end
def self.get_history(env)
env.response.content_type = "application/json"
user = env.get("user").as(User)
page = env.params.query["page"]?.try &.to_i?.try &.clamp(0, Int32::MAX)
page ||= 1
max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE)
max_results ||= user.preferences.max_results
max_results ||= CONFIG.default_user_preferences.max_results
start_index = (page - 1) * max_results
if user.watched[start_index]?
watched = user.watched.reverse[start_index, max_results]
end
watched ||= [] of String
return watched.to_json
end
def self.mark_watched(env)
user = env.get("user").as(User)
if !user.preferences.watch_history
return error_json(409, "Watch history is disabled in preferences.")
end
id = env.params.url["id"]
if !id.match(/^[a-zA-Z0-9_-]{11}$/)
return error_json(400, "Invalid video id.")
end
Invidious::Database::Users.mark_watched(user, id)
env.response.status_code = 204
end
def self.mark_unwatched(env)
user = env.get("user").as(User)
if !user.preferences.watch_history
return error_json(409, "Watch history is disabled in preferences.")
end
id = env.params.url["id"]
if !id.match(/^[a-zA-Z0-9_-]{11}$/)
return error_json(400, "Invalid video id.")
end
Invidious::Database::Users.mark_unwatched(user, id)
env.response.status_code = 204
end
def self.clear_history(env)
user = env.get("user").as(User)
Invidious::Database::Users.clear_watch_history(user)
env.response.status_code = 204
end
def self.feed(env)
env.response.content_type = "application/json"

View File

@@ -89,6 +89,8 @@ module Invidious::Routes::API::V1::Channels
json.field "descriptionHtml", channel.description_html
json.field "allowedRegions", channel.allowed_regions
json.field "tabs", channel.tabs
json.field "authorVerified", channel.verified
json.field "latestVideos" do
json.array do

View File

@@ -150,4 +150,31 @@ module Invidious::Routes::API::V1::Misc
response
end
# resolve channel and clip urls, return the UCID
def self.resolve_url(env)
env.response.content_type = "application/json"
url = env.params.query["url"]?
return error_json(400, "Missing URL to resolve") if !url
begin
resolved_url = YoutubeAPI.resolve_url(url.as(String))
endpoint = resolved_url["endpoint"]
pageType = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || ""
if resolved_ucid = endpoint.dig?("watchEndpoint", "videoId")
elsif resolved_ucid = endpoint.dig?("browseEndpoint", "browseId")
elsif pageType == "WEB_PAGE_TYPE_UNKNOWN"
return error_json(400, "Unknown url")
end
rescue ex
return error_json(500, ex)
end
JSON.build do |json|
json.object do
json.field "ucid", resolved_ucid.try &.as_s || ""
json.field "pageType", pageType
end
end
end
end

View File

@@ -6,6 +6,7 @@ module Invidious::Routes::API::V1::Videos
id = env.params.url["id"]
region = env.params.query["region"]?
proxy = {"1", "true"}.any? &.== env.params.query["local"]?
begin
video = get_video(id, region: region)
@@ -15,7 +16,9 @@ module Invidious::Routes::API::V1::Videos
return error_json(500, ex)
end
video.to_json(locale, nil)
return JSON.build do |json|
Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy)
end
end
def self.captions(env)
@@ -90,45 +93,50 @@ module Invidious::Routes::API::V1::Videos
# as well as some other markup that makes it cumbersome, so we try to fix that here
if caption.name.includes? "auto-generated"
caption_xml = YT_POOL.client &.get(url).body
caption_xml = XML.parse(caption_xml)
webvtt = String.build do |str|
str << <<-END_VTT
WEBVTT
Kind: captions
Language: #{tlang || caption.language_code}
if caption_xml.starts_with?("<?xml")
webvtt = caption.timedtext_to_vtt(caption_xml, tlang)
else
caption_xml = XML.parse(caption_xml)
webvtt = String.build do |str|
str << <<-END_VTT
WEBVTT
Kind: captions
Language: #{tlang || caption.language_code}
END_VTT
END_VTT
caption_nodes = caption_xml.xpath_nodes("//transcript/text")
caption_nodes.each_with_index do |node, i|
start_time = node["start"].to_f.seconds
duration = node["dur"]?.try &.to_f.seconds
duration ||= start_time
caption_nodes = caption_xml.xpath_nodes("//transcript/text")
caption_nodes.each_with_index do |node, i|
start_time = node["start"].to_f.seconds
duration = node["dur"]?.try &.to_f.seconds
duration ||= start_time
if caption_nodes.size > i + 1
end_time = caption_nodes[i + 1]["start"].to_f.seconds
else
end_time = start_time + duration
if caption_nodes.size > i + 1
end_time = caption_nodes[i + 1]["start"].to_f.seconds
else
end_time = start_time + duration
end
start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}"
end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}"
text = HTML.unescape(node.content)
text = text.gsub(/<font color="#[a-fA-F0-9]{6}">/, "")
text = text.gsub(/<\/font>/, "")
if md = text.match(/(?<name>.*) : (?<text>.*)/)
text = "<v #{md["name"]}>#{md["text"]}</v>"
end
str << <<-END_CUE
#{start_time} --> #{end_time}
#{text}
END_CUE
end
start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}"
end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}"
text = HTML.unescape(node.content)
text = text.gsub(/<font color="#[a-fA-F0-9]{6}">/, "")
text = text.gsub(/<\/font>/, "")
if md = text.match(/(?<name>.*) : (?<text>.*)/)
text = "<v #{md["name"]}>#{md["text"]}</v>"
end
str << <<-END_CUE
#{start_time} --> #{end_time}
#{text}
END_CUE
end
end
else
@@ -138,7 +146,12 @@ module Invidious::Routes::API::V1::Videos
#
# See: https://github.com/iv-org/invidious/issues/2391
webvtt = YT_POOL.client &.get("#{url}&format=vtt").body
.gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1")
if webvtt.starts_with?("<?xml")
webvtt = caption.timedtext_to_vtt(webvtt)
else
webvtt = YT_POOL.client &.get("#{url}&format=vtt").body
.gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1")
end
end
if title = env.params.query["title"]?

View File

@@ -6,14 +6,14 @@ module Invidious::Routes::Login
user = env.get? "user"
return env.redirect "/feed/subscriptions" if user
referer = get_referer(env, "/feed/subscriptions")
return env.redirect referer if user
if !CONFIG.login_enabled
return error_template(400, "Login has been disabled by administrator.")
end
referer = get_referer(env, "/feed/subscriptions")
email = nil
password = nil
captcha = nil

View File

@@ -104,33 +104,8 @@ module Invidious::Routes::Subscriptions
if format == "json"
env.response.content_type = "application/json"
env.response.headers["content-disposition"] = "attachment"
playlists = Invidious::Database::Playlists.select_like_iv(user.email)
return JSON.build do |json|
json.object do
json.field "subscriptions", user.subscriptions
json.field "watch_history", user.watched
json.field "preferences", user.preferences
json.field "playlists" do
json.array do
playlists.each do |playlist|
json.object do
json.field "title", playlist.title
json.field "description", html_to_content(playlist.description_html)
json.field "privacy", playlist.privacy.to_s
json.field "videos" do
json.array do
Invidious::Database::PlaylistVideos.select_ids(playlist.id, playlist.index, limit: 500).each do |video_id|
json.string video_id
end
end
end
end
end
end
end
end
end
return Invidious::User::Export.to_invidious(user)
else
env.response.content_type = "application/xml"
env.response.headers["content-disposition"] = "attachment"

View File

@@ -35,6 +35,13 @@ module Invidious::Routes::VideoPlayback
end
end
# See: https://github.com/iv-org/invidious/issues/3302
range_header = env.request.headers["Range"]?
if range_header.nil?
range_for_head = query_params["range"]? || "0-640"
headers["Range"] = "bytes=#{range_for_head}"
end
client = make_client(URI.parse(host), region)
response = HTTP::Client::Response.new(500)
error = ""
@@ -70,6 +77,9 @@ module Invidious::Routes::VideoPlayback
end
end
# Remove the Range header added previously.
headers.delete("Range") if range_header.nil?
if response.status_code >= 400
env.response.content_type = "text/plain"
haltf env, response.status_code
@@ -91,14 +101,8 @@ module Invidious::Routes::VideoPlayback
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = resp.headers["Location"]?
location = URI.parse(location)
location = "#{location.request_target}&host=#{location.host}"
if region
location += "&region=#{region}"
end
return env.redirect location
url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region)
return env.redirect url
end
IO.copy(resp.body_io, env.response)
@@ -252,7 +256,7 @@ module Invidious::Routes::VideoPlayback
return error_template(400, "Invalid video ID")
end
if itag.nil? || itag <= 0 || itag >= 1000
if !itag.nil? && (itag <= 0 || itag >= 1000)
return error_template(400, "Invalid itag")
end
@@ -273,7 +277,11 @@ module Invidious::Routes::VideoPlayback
return error_template(500, ex)
end
fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag }
if itag.nil?
fmt = video.fmt_stream[-1]?
else
fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag }
end
url = fmt.try &.["url"]?.try &.as_s
if !url

View File

@@ -132,6 +132,8 @@ module Invidious::Routing
get "/c/:user#{path}", Routes::Channels, :brand_redirect
# /user/linustechtips | Not always the same as /c/
get "/user/:user#{path}", Routes::Channels, :brand_redirect
# /@LinusTechTips | Handle
get "/@:user#{path}", Routes::Channels, :brand_redirect
# /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow
get "/attribution_link#{path}", Routes::Channels, :brand_redirect
# /profile?user=linustechtips
@@ -252,6 +254,14 @@ module Invidious::Routing
get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences
post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences
get "/api/v1/auth/export/invidious", {{namespace}}::Authenticated, :export_invidious
post "/api/v1/auth/import/invidious", {{namespace}}::Authenticated, :import_invidious
get "/api/v1/auth/history", {{namespace}}::Authenticated, :get_history
post "/api/v1/auth/history/:id", {{namespace}}::Authenticated, :mark_watched
delete "/api/v1/auth/history/:id", {{namespace}}::Authenticated, :mark_unwatched
delete "/api/v1/auth/history", {{namespace}}::Authenticated, :clear_history
get "/api/v1/auth/feed", {{namespace}}::Authenticated, :feed
get "/api/v1/auth/subscriptions", {{namespace}}::Authenticated, :get_subscriptions
@@ -279,6 +289,7 @@ module Invidious::Routing
get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist
get "/api/v1/auth/playlists/:plid", {{namespace}}::Misc, :get_playlist
get "/api/v1/mixes/:rdid", {{namespace}}::Misc, :mixes
get "/api/v1/resolveurl", {{namespace}}::Misc, :resolve_url
{% end %}
end
end

View File

@@ -4,11 +4,12 @@ def fetch_trending(trending_type, region, locale)
plid = nil
if trending_type == "Music"
case trending_type.try &.downcase
when "music"
params = "4gINGgt5dG1hX2NoYXJ0cw%3D%3D"
elsif trending_type == "Gaming"
when "gaming"
params = "4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D"
elsif trending_type == "Movies"
when "movies"
params = "4gIKGgh0cmFpbGVycw%3D%3D"
else # Default
params = ""

View File

@@ -0,0 +1,35 @@
struct Invidious::User
module Export
extend self
def to_invidious(user : User)
playlists = Invidious::Database::Playlists.select_like_iv(user.email)
return JSON.build do |json|
json.object do
json.field "subscriptions", user.subscriptions
json.field "watch_history", user.watched
json.field "preferences", user.preferences
json.field "playlists" do
json.array do
playlists.each do |playlist|
json.object do
json.field "title", playlist.title
json.field "description", html_to_content(playlist.description_html)
json.field "privacy", playlist.privacy.to_s
json.field "videos" do
json.array do
Invidious::Database::PlaylistVideos.select_ids(playlist.id, playlist.index, limit: CONFIG.playlist_length_limit).each do |video_id|
json.string video_id
end
end
end
end
end
end
end
end
end
end
end # module
end

View File

@@ -247,6 +247,17 @@ struct Video
info["reason"]?.try &.as_s
end
def music : Array(VideoMusic)
info["music"].as_a.map { |music_json|
VideoMusic.new(
music_json["song"].as_s,
music_json["album"].as_s,
music_json["artist"].as_s,
music_json["license"].as_s
)
}
end
# Macros defining getters/setters for various types of data
private macro getset_string(name)

View File

@@ -31,6 +31,72 @@ module Invidious::Videos
return captions_list
end
def timedtext_to_vtt(timedtext : String, tlang = nil) : String
# In the future, we could just directly work with the url. This is more of a POC
cues = [] of XML::Node
tree = XML.parse(timedtext)
tree = tree.children.first
tree.children.each do |item|
if item.name == "body"
item.children.each do |cue|
if cue.name == "p" && !(cue.children.size == 1 && cue.children[0].content == "\n")
cues << cue
end
end
break
end
end
result = String.build do |result|
result << <<-END_VTT
WEBVTT
Kind: captions
Language: #{tlang || @language_code}
END_VTT
result << "\n\n"
cues.each_with_index do |node, i|
start_time = node["t"].to_f.milliseconds
duration = node["d"]?.try &.to_f.milliseconds
duration ||= start_time
if cues.size > i + 1
end_time = cues[i + 1]["t"].to_f.milliseconds
else
end_time = start_time + duration
end
# start_time
result << start_time.hours.to_s.rjust(2, '0')
result << ':' << start_time.minutes.to_s.rjust(2, '0')
result << ':' << start_time.seconds.to_s.rjust(2, '0')
result << '.' << start_time.milliseconds.to_s.rjust(3, '0')
result << " --> "
# end_time
result << end_time.hours.to_s.rjust(2, '0')
result << ':' << end_time.minutes.to_s.rjust(2, '0')
result << ':' << end_time.seconds.to_s.rjust(2, '0')
result << '.' << end_time.milliseconds.to_s.rjust(3, '0')
result << "\n"
node.children.each do |s|
result << s.content
end
result << "\n"
result << "\n"
end
end
return result
end
# List of all caption languages available on Youtube.
LANGUAGES = {
"",

View File

@@ -0,0 +1,13 @@
require "json"
struct VideoMusic
include JSON::Serializable
property song : String
property album : String
property artist : String
property license : String
def initialize(@song : String, @album : String, @artist : String, @license : String)
end
end

View File

@@ -66,8 +66,10 @@ def extract_video_info(video_id : String, proxy_region : String? = nil)
reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("")
reason ||= player_response.dig("playabilityStatus", "reason").as_s
# Stop here if video is not a scheduled livestream
if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status)
# Stop here if video is not a scheduled livestream or
# for LOGIN_REQUIRED when videoDetails element is not found because retrying won't help
if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) ||
playability_status == "LOGIN_REQUIRED" && !player_response.dig?("videoDetails")
return {
"version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
"reason" => JSON::Any.new(reason),
@@ -183,10 +185,12 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
# We have to try to extract viewCount from videoPrimaryInfoRenderer first,
# then from videoDetails, as the latter is "0" for livestreams (we want
# to get the amount of viewers watching).
views_txt = video_primary_renderer
.try &.dig?("viewCount", "videoViewCountRenderer", "viewCount", "runs", 0, "text")
views_txt ||= video_details["viewCount"]?
views = views_txt.try &.as_s.gsub(/\D/, "").to_i64?
views_txt = extract_text(
video_primary_renderer
.try &.dig?("viewCount", "videoViewCountRenderer", "viewCount")
)
views_txt ||= video_details["viewCount"]?.try &.as_s || ""
views = views_txt.gsub(/\D/, "").to_i64?
length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"])
.try &.as_s.to_i64
@@ -309,6 +313,44 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
end
end
# Music section
music_list = [] of VideoMusic
music_desclist = player_response.dig?(
"engagementPanels", 1, "engagementPanelSectionListRenderer",
"content", "structuredDescriptionContentRenderer", "items", 2,
"videoDescriptionMusicSectionRenderer", "carouselLockups"
)
music_desclist.try &.as_a.each do |music_desc|
artist = nil
album = nil
music_license = nil
# Used when the video has multiple songs
if song_title = music_desc.dig?("carouselLockupRenderer", "videoLockup", "compactVideoRenderer", "title")
# "simpleText" for plain text / "runs" when song has a link
song = song_title["simpleText"]? || song_title.dig?("runs", 0, "text")
# some videos can have empty tracks. See: https://www.youtube.com/watch?v=eBGIQ7ZuuiU
next if !song
end
music_desc.dig?("carouselLockupRenderer", "infoRows").try &.as_a.each do |desc|
desc_title = extract_text(desc.dig?("infoRowRenderer", "title"))
if desc_title == "ARTIST"
artist = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata"))
elsif desc_title == "SONG"
song = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata"))
elsif desc_title == "ALBUM"
album = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata"))
elsif desc_title == "LICENSES"
music_license = extract_text(desc.dig?("infoRowRenderer", "expandedMetadata"))
end
end
music_list << VideoMusic.new(song.to_s, album.to_s, artist.to_s, music_license.to_s)
end
# Author infos
author = video_details["author"]?.try &.as_s
@@ -359,6 +401,8 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
"genre" => JSON::Any.new(genre.try &.as_s || ""),
"genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""),
"license" => JSON::Any.new(license.try &.as_s || ""),
# Music section
"music" => JSON.parse(music_list.to_json),
# Author infos
"author" => JSON::Any.new(author || ""),
"ucid" => JSON::Any.new(ucid || ""),

View File

@@ -39,6 +39,8 @@
<% end %>
</div>
<script src="/js/watched_indicator.js"></script>
<% if query %>
<%- query_encoded = URI.encode_www_form(query.text, space_to_plus: true) -%>
<div class="pure-g h-box">

View File

@@ -49,6 +49,8 @@
<% end %>
</div>
<script src="/js/watched_indicator.js"></script>
<div class="pure-g h-box">
<div class="pure-u-1 pure-u-md-4-5"></div>
<div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">

View File

@@ -1,3 +1,5 @@
<% item_watched = !item.is_a?(SearchChannel | SearchPlaylist | InvidiousPlaylist | Category) && env.get?("user").try &.as(User).watched.index(item.id) != nil %>
<div class="pure-u-1 pure-u-md-1-4">
<div class="h-box">
<% case item when %>
@@ -40,6 +42,11 @@
<% if item.length_seconds != 0 %>
<p class="length"><%= recode_length_seconds(item.length_seconds) %></p>
<% end %>
<% if item_watched %>
<div class="watched-overlay"></div>
<div class="watched-indicator" data-length="<%= item.length_seconds %>" data-id="<%= item.id %>"></div>
<% end %>
</div>
<% end %>
<p dir="auto"><%= HTML.escape(item.title) %></p>
@@ -67,6 +74,11 @@
<% elsif item.length_seconds != 0 %>
<p class="length"><%= recode_length_seconds(item.length_seconds) %></p>
<% end %>
<% if item_watched %>
<div class="watched-overlay"></div>
<div class="watched-indicator" data-length="<%= item.length_seconds %>" data-id="<%= item.id %>"></div>
<% end %>
</div>
<% end %>
<p dir="auto"><%= HTML.escape(item.title) %></p>
@@ -124,6 +136,11 @@
<% elsif item.length_seconds != 0 %>
<p class="length"><%= recode_length_seconds(item.length_seconds) %></p>
<% end %>
<% if item_watched %>
<div class="watched-overlay"></div>
<div class="watched-indicator" data-length="<%= item.length_seconds %>" data-id="<%= item.id %>"></div>
<% end %>
</div>
<% end %>
<p dir="auto"><%= HTML.escape(item.title) %></p>

View File

@@ -62,6 +62,8 @@
<% end %>
</div>
<script src="/js/watched_indicator.js"></script>
<div class="pure-g h-box">
<div class="pure-u-1 pure-u-lg-1-5">
<% if page > 1 %>

View File

@@ -39,3 +39,5 @@
<%= rendered "components/item" %>
<% end %>
</div>
<script src="/js/watched_indicator.js"></script>

View File

@@ -16,3 +16,5 @@
<%= rendered "components/item" %>
<% end %>
</div>
<script src="/js/watched_indicator.js"></script>

View File

@@ -62,6 +62,8 @@
<% end %>
</div>
<script src="/js/watched_indicator.js"></script>
<div class="pure-g h-box">
<div class="pure-u-1 pure-u-lg-1-5">
<% if page > 1 %>

View File

@@ -45,3 +45,5 @@
<%= rendered "components/item" %>
<% end %>
</div>
<script src="/js/watched_indicator.js"></script>

View File

@@ -24,6 +24,8 @@
<%- end -%>
</div>
<script src="/js/watched_indicator.js"></script>
<div class="pure-g h-box">
<div class="pure-u-1 pure-u-lg-1-5">
<%- if page > 1 -%>

View File

@@ -106,6 +106,8 @@
<% end %>
</div>
<script src="/js/watched_indicator.js"></script>
<div class="pure-g h-box">
<div class="pure-u-1 pure-u-lg-1-5">
<% if page > 1 %>

View File

@@ -37,6 +37,8 @@
</div>
<%- end -%>
<script src="/js/watched_indicator.js"></script>
<div class="pure-g h-box">
<div class="pure-u-1 pure-u-lg-1-5">
<%- if query.page > 1 -%>

View File

@@ -181,7 +181,11 @@ we're going to need to do it here in order to allow for translations.
<% end %>
</p>
<% if video.license %>
<p id="license"><%= translate(locale, "License: ") %><%= video.license %></p>
<% if video.license.empty? %>
<p id="license"><%= translate(locale, "License: ") %><%= translate(locale, "Standard YouTube license") %></p>
<% else %>
<p id="license"><%= translate(locale, "License: ") %><%= video.license %></p>
<% end %>
<% end %>
<p id="family_friendly"><%= translate(locale, "Family friendly? ") %><%= translate_bool(locale, video.is_family_friendly) %></p>
<p id="wilson" style="display: none; visibility: hidden;"></p>
@@ -235,6 +239,28 @@ we're going to need to do it here in order to allow for translations.
<hr>
<% if !video.music.empty? %>
<input id="music-desc-expansion" type="checkbox"/>
<label for="music-desc-expansion">
<h3 id="music-description-title">
<%= translate(locale, "Music in this video") %>
<span class="icon ion-ios-arrow-up"></span>
<span class="icon ion-ios-arrow-down"></span>
</h3>
</label>
<div id="music-description-box">
<% video.music.each do |music| %>
<div class="music-item">
<p class="music-song"><%= translate(locale, "Song: ") %><%= music.song %></p>
<p class="music-artist"><%= translate(locale, "Artist: ") %><%= music.artist %></p>
<p class="music-album"><%= translate(locale, "Album: ") %><%= music.album %></p>
</div>
<% end %>
</div>
<hr>
<% end %>
<div id="comments">
<% if nojs %>
<%= comment_html %>

View File

@@ -18,6 +18,7 @@ private ITEM_PARSERS = {
Parsers::CategoryRendererParser,
Parsers::RichItemRendererParser,
Parsers::ReelItemRendererParser,
Parsers::ItemSectionRendererParser,
Parsers::ContinuationItemRendererParser,
}
@@ -172,7 +173,17 @@ private module Parsers
# When public subscriber count is disabled, the subscriberCountText isn't sent by InnerTube.
# Always simpleText
# TODO change default value to nil
subscriber_count = item_contents.dig?("subscriberCountText", "simpleText")
# Since youtube added channel handles, `VideoCountText` holds the number of
# subscribers and `subscriberCountText` holds the handle, except when the
# channel doesn't have a handle (e.g: some topic music channels).
# See https://github.com/iv-org/invidious/issues/3394#issuecomment-1321261688
if !subscriber_count || !subscriber_count.as_s.includes? " subscriber"
subscriber_count = item_contents.dig?("videoCountText", "simpleText")
end
subscriber_count = subscriber_count
.try { |s| short_text_to_number(s.as_s.split(" ")[0]).to_i32 } || 0
# Auto-generated channels doesn't have videoCountText
@@ -367,6 +378,30 @@ private module Parsers
end
end
# Parses an InnerTube itemSectionRenderer into a SearchVideo.
# Returns nil when the given object isn't a ItemSectionRenderer
#
# A itemSectionRenderer seems to be a simple wrapper for a videoRenderer, used
# by the result page for channel searches. It is located inside a continuationItems
# container.It is very similar to RichItemRendererParser
#
module ItemSectionRendererParser
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = item.dig?("itemSectionRenderer", "contents", 0)
return self.parse(item_contents, author_fallback)
end
end
private def self.parse(item_contents, author_fallback)
child = VideoRendererParser.process(item_contents, author_fallback)
return child
end
def self.parser_name
return {{@type.name}}
end
end
# Parses an InnerTube richItemRenderer into a SearchVideo.
# Returns nil when the given object isn't a RichItemRenderer
#
@@ -682,7 +717,11 @@ module HelperExtractors
# Returns a 0 when it's unable to do so
def self.get_video_count(container : JSON::Any) : Int32
if box = container["videoCountText"]?
return extract_text(box).try &.gsub(/\D/, "").to_i || 0
if (extracted_text = extract_text(box)) && !extracted_text.includes? " subscriber"
return extracted_text.gsub(/\D/, "").to_i
else
return 0
end
elsif box = container["videoCount"]?
return box.as_s.to_i
else
@@ -759,6 +798,7 @@ end
def extract_items(initial_data : InitialData, &block)
if unpackaged_data = initial_data["contents"]?.try &.as_h
elsif unpackaged_data = initial_data["response"]?.try &.as_h
elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 1).try &.as_h
elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 0).try &.as_h
else
unpackaged_data = initial_data