Mirrored_Repos/invidious
1
0
Fork 0
mirror of https://github.com/iv-org/invidious.git synced 2024-11-22 05:27:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add support for the new channel layout - part 2 (#3419)

Browse Source
This commit is contained in:
Samantaz Fox 2023-01-10 21:16:12 +01:00
commit 05258d56bd
No known key found for this signature in database
GPG Key ID: F42821059186176E
28 changed files with 765 additions and 684 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
locales/en-US.json
View File

@ -404,9 +404,7 @@
"`x` marked it with a ❤": "`x` marked it with a ❤",
"Audio mode": "Audio mode",
"Video mode": "Video mode",
"Videos": "Videos",
"Playlists": "Playlists",
"Community": "Community",
"search_filters_title": "Filters",
"search_filters_date_label": "Upload date",
"search_filters_date_option_none": "Any date",
@ -472,5 +470,11 @@
"crash_page_read_the_faq": "read the <a href=\"`x`\">Frequently Asked Questions (FAQ)</a>",
"crash_page_search_issue": "searched for <a href=\"`x`\">existing issues on GitHub</a>",
"crash_page_report_issue": "If none of the above helped, please <a href=\"`x`\">open a new issue on GitHub</a> (preferably in English) and include the following text in your message (do NOT translate that text):",
"error_video_not_in_playlist": "The requested video doesn't exist in this playlist. <a href=\"`x`\">Click here for the playlist home page.</a>"
"error_video_not_in_playlist": "The requested video doesn't exist in this playlist. <a href=\"`x`\">Click here for the playlist home page.</a>",
"channel_tab_videos_label": "Videos",
"channel_tab_shorts_label": "Shorts",
"channel_tab_streams_label": "Livestreams",
"channel_tab_playlists_label": "Playlists",
"channel_tab_community_label": "Community",
"channel_tab_channels_label": "Channels"
}

2
shard.lock
View File

@ -34,7 +34,7 @@ shards:
protodec:
git: https://github.com/iv-org/protodec.git
version: 0.1.4
version: 0.1.5
radix:
git: https://github.com/luislavena/radix.git

2
shard.yml
View File

@ -24,7 +24,7 @@ dependencies:
version: ~> 0.6.1
protodec:
github: iv-org/protodec
version: ~> 0.1.4
version: ~> 0.1.5
lsquic:
github: iv-org/lsquic.cr
version: ~> 2.18.1-2

4
spec/invidious/hashtag_spec.cr
View File

@ -4,7 +4,7 @@ Spectator.describe Invidious::Hashtag do
it "parses richItemRenderer containers (test 1)" do
# Enable mock
test_content = load_mock("hashtag/martingarrix_page1")
videos = extract_items(test_content)
videos, _ = extract_items(test_content)
expect(typeof(videos)).to eq(Array(SearchItem))
expect(videos.size).to eq(60)
@ -57,7 +57,7 @@ Spectator.describe Invidious::Hashtag do
it "parses richItemRenderer containers (test 2)" do
# Enable mock
test_content = load_mock("hashtag/martingarrix_page2")
videos = extract_items(test_content)
videos, _ = extract_items(test_content)
expect(typeof(videos)).to eq(Array(SearchItem))
expect(videos.size).to eq(60)

6
spec/invidious/helpers_spec.cr
View File

@ -23,12 +23,6 @@ Spectator.describe "Helper" do
end
end
describe "#produce_channel_playlists_url" do
it "correctly produces a /browse_ajax URL with the given UCID and cursor" do
expect(produce_channel_playlists_url("UCCj956IF62FbT7Gouszaj9w", "AIOkY9EQpi_gyn1_QrFuZ1reN81_MMmI1YmlBblw8j7JHItEFG5h7qcJTNd4W9x5Quk_CVZ028gW")).to eq("/browse_ajax?continuation=4qmFsgLNARIYVUNDajk1NklGNjJGYlQ3R291c3phajl3GrABRWdsd2JHRjViR2x6ZEhNd0FqZ0JZQUZxQUxnQkFIcG1VVlZzVUdFeGF6VlNWa1ozWVZZNWJtVlhOSGhZTVVaNVVtNVdZVTFZU214VWFtZDRXREF4VG1KVmEzaFhWekZ6VVcxS2MyUjZhSEZPTUhCSlUxaFNSbEpyWXpGaFJHUjRXVEJ3VlZSdFVUQldlbXcwVGxaR01XRXhPVVJXYkc5M1RXcG9ibFozSUFFWUF3PT0%3D&gl=US&hl=en")
end
end
describe "#produce_comment_continuation" do
it "correctly produces a continuation token for comments" do
expect(produce_comment_continuation("_cE8xSu6swE", "ADSJ_i2qvJeFtL0htmS5_K5Ctj3eGFVBMWL9Wd42o3kmUL6_mAzdLp85-liQZL0mYr_16BhaggUqX652Sv9JqV6VXinShSP-ZT6rL4NolPBaPXVtJsO5_rA_qE3GubAuLFw9uzIIXU2-HnpXbdgPLWTFavfX206hqWmmpHwUOrmxQV_OX6tYkM3ux3rPAKCDrT8eWL7MU3bLiNcnbgkW8o0h8KYLL_8BPa8LcHbTv8pAoNkjerlX1x7K4pqxaXPoyz89qNlnh6rRx6AXgAzzoHH1dmcyQ8CIBeOHg-m4i8ZxdX4dP88XWrIFg-jJGhpGP8JUMDgZgavxVx225hUEYZMyrLGler5em4FgbG62YWC51moLDLeYEA")).to eq("EkMSC19jRTh4U3U2c3dFyAEA4AEBogINKP___________wFAAMICHQgEGhdodHRwczovL3d3dy55b3V0dWJlLmNvbSIAGAYyjAMK9gJBRFNKX2kycXZKZUZ0TDBodG1TNV9LNUN0ajNlR0ZWQk1XTDlXZDQybzNrbVVMNl9tQXpkTHA4NS1saVFaTDBtWXJfMTZCaGFnZ1VxWDY1MlN2OUpxVjZWWGluU2hTUC1aVDZyTDROb2xQQmFQWFZ0SnNPNV9yQV9xRTNHdWJBdUxGdzl1eklJWFUyLUhucFhiZGdQTFdURmF2ZlgyMDZocVdtbXBId1VPcm14UVZfT1g2dFlrTTN1eDNyUEFLQ0RyVDhlV0w3TVUzYkxpTmNuYmdrVzhvMGg4S1lMTF84QlBhOExjSGJUdjhwQW9Oa2plcmxYMXg3SzRwcXhhWFBveXo4OXFObG5oNnJSeDZBWGdBenpvSEgxZG1jeVE4Q0lCZU9IZy1tNGk4WnhkWDRkUDg4WFdySUZnLWpKR2hwR1A4SlVNRGdaZ2F2eFZ4MjI1aFVFWVpNeXJMR2xlcjVlbTRGZ2JHNjJZV0M1MW1vTERMZVlFQSIPIgtfY0U4eFN1NnN3RTAAKBQ%3D")

9
src/invidious.cr
View File

@ -48,6 +48,13 @@ require "./invidious/search/*"
require "./invidious/routes/**"
require "./invidious/jobs/**"
# Declare the base namespace for invidious
module Invidious
end
# Simple alias to make code easier to read
alias IV = Invidious
CONFIG = Config.load
HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32)
@ -172,7 +179,7 @@ if CONFIG.popular_enabled
Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB)
end
CONNECTION_CHANNEL = Channel({Bool, Channel(PQ::Notification)}).new(32)
CONNECTION_CHANNEL = ::Channel({Bool, ::Channel(PQ::Notification)}).new(32)
Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL, CONFIG.database_url)
Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new

98
src/invidious/channels/about.cr
View File

@ -16,12 +16,6 @@ record AboutChannel,
tabs : Array(String),
verified : Bool
record AboutRelatedChannel,
ucid : String,
author : String,
author_url : String,
author_thumbnail : String
def get_about_info(ucid, locale) : AboutChannel
begin
# "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"}
@ -100,35 +94,47 @@ def get_about_info(ucid, locale) : AboutChannel
total_views = 0_i64
joined = Time.unix(0)
tabs = [] of String
tab_names = [] of String
tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?.try &.as_a?
if !tabs_json.nil?
# Retrieve information from the tabs array. The index we are looking for varies between channels.
tabs_json.each do |node|
# Try to find the about section which is located in only one of the tabs.
channel_about_meta = node["tabRenderer"]?.try &.["content"]?.try &.["sectionListRenderer"]?
.try &.["contents"]?.try &.[0]?.try &.["itemSectionRenderer"]?.try &.["contents"]?
.try &.[0]?.try &.["channelAboutFullMetadataRenderer"]?
if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?
# Get the name of the tabs available on this channel
tab_names = tabs_json.as_a.compact_map do |entry|
name = entry.dig?("tabRenderer", "title").try &.as_s.downcase
# This is a small fix to not add extra code on the HTML side
# I.e, the URL for the "live" tab is .../streams, so use "streams"
# everywhere for the sake of simplicity
(name == "live") ? "streams" : name
end
# Get the currently active tab ("About")
about_tab = extract_selected_tab(tabs_json)
# Try to find the about metadata section
channel_about_meta = about_tab.dig?(
"content",
"sectionListRenderer", "contents", 0,
"itemSectionRenderer", "contents", 0,
"channelAboutFullMetadataRenderer"
)
if !channel_about_meta.nil?
total_views = channel_about_meta["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D/, "").to_i64? || 0_i64
total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64
# The joined text is split to several sub strings. The reduce joins those strings before parsing the date.
joined = channel_about_meta["joinedDateText"]?.try &.["runs"]?.try &.as_a.reduce("") { |acc, nd| acc + nd["text"].as_s }
joined = extract_text(channel_about_meta["joinedDateText"]?)
.try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
# Normal Auto-generated channels
# https://support.google.com/youtube/answer/2579942
# For auto-generated channels, channel_about_meta only has ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"]
if (channel_about_meta["primaryLinks"]?.try &.size || 0) == 1 && (channel_about_meta["primaryLinks"][0]?) &&
(channel_about_meta["primaryLinks"][0]["title"]?.try &.["simpleText"]?.try &.as_s? || "") == "Auto-generated by YouTube"
auto_generated = true
# For auto-generated channels, channel_about_meta only has
# ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"]
auto_generated = (
(channel_about_meta["primaryLinks"]?.try &.size) == 1 && \
extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube"
)
end
end
end
tabs = tabs_json.reject { |node| node["tabRenderer"]?.nil? }.map(&.["tabRenderer"]["title"].as_s.downcase)
end
sub_count = initdata
.dig?("header", "c4TabbedHeaderRenderer", "subscriberCountText", "simpleText").try &.as_s?
@ -148,46 +154,20 @@ def get_about_info(ucid, locale) : AboutChannel
joined: joined,
is_family_friendly: is_family_friendly,
allowed_regions: allowed_regions,
tabs: tabs,
tabs: tab_names,
verified: author_verified || false,
)
end
def fetch_related_channels(about_channel : AboutChannel) : Array(AboutRelatedChannel)
def fetch_related_channels(about_channel : AboutChannel, continuation : String? = nil) : {Array(SearchChannel), String?}
if continuation.nil?
# params is {"2:string":"channels"} encoded
channels = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D")
tabs = channels.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs").try(&.as_a?) || [] of JSON::Any
tab = tabs.find(&.dig?("tabRenderer", "title").try(&.as_s?).try(&.== "Channels"))
return [] of AboutRelatedChannel if tab.nil?
items = tab.dig?(
"tabRenderer", "content",
"sectionListRenderer", "contents", 0,
"itemSectionRenderer", "contents", 0,
"gridRenderer", "items"
).try &.as_a?
related = [] of AboutRelatedChannel
return related if (items.nil? || items.empty?)
items.each do |item|
renderer = item["gridChannelRenderer"]?
next if !renderer
related_id = renderer.dig("channelId").as_s
related_title = renderer.dig("title", "simpleText").as_s
related_author_url = renderer.dig("navigationEndpoint", "browseEndpoint", "canonicalBaseUrl").as_s
related_author_thumbnail = HelperExtractors.get_thumbnails(renderer)
related << AboutRelatedChannel.new(
ucid: related_id,
author: related_title,
author_url: related_author_url,
author_thumbnail: related_author_thumbnail,
)
initial_data = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D")
else
initial_data = YoutubeAPI.browse(continuation)
end
return related
items, continuation = extract_items(initial_data)
return items.select(SearchChannel), continuation
end

45
src/invidious/channels/channels.cr
View File

@ -180,11 +180,16 @@ def fetch_channel(ucid, pull_all_videos : Bool)
LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}")
page = 1
channel = InvidiousChannel.new({
id: ucid,
author: author,
updated: Time.utc,
deleted: false,
subscribed: nil,
})
LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page")
initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
videos = extract_videos(initial_data, author, ucid)
videos, continuation = IV::Channel::Tabs.get_videos(channel)
LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
rss.xpath_nodes("//feed/entry").each do |entry|
@ -197,7 +202,9 @@ def fetch_channel(ucid, pull_all_videos : Bool)
views = entry.xpath_node("group/community/statistics").try &.["views"]?.try &.to_i64?
views ||= 0_i64
channel_video = videos.select { |video| video.id == video_id }[0]?
channel_video = videos
.select(SearchVideo)
.select(&.id.== video_id)[0]?
length_seconds = channel_video.try &.length_seconds
length_seconds ||= 0
@ -239,16 +246,14 @@ def fetch_channel(ucid, pull_all_videos : Bool)
end
if pull_all_videos
page += 1
ids = [] of String
loop do
initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
videos = extract_videos(initial_data, author, ucid)
# Keep fetching videos using the continuation token retrieved earlier
videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation)
count = videos.size
videos = videos.map { |video| ChannelVideo.new({
count = 0
videos.select(SearchVideo).each do |video|
count += 1
video = ChannelVideo.new({
id: video.id,
title: video.title,
published: video.published,
@ -259,10 +264,7 @@ def fetch_channel(ucid, pull_all_videos : Bool)
live_now: video.live_now,
premiere_timestamp: video.premiere_timestamp,
views: video.views,
}) }
videos.each do |video|
ids << video.id
})
# We are notified of Red videos elsewhere (PubSub), which includes a correct published date,
# so since they don't provide a published date here we can safely ignore them.
@ -279,17 +281,10 @@ def fetch_channel(ucid, pull_all_videos : Bool)
end
break if count < 25
page += 1
sleep 500.milliseconds
end
end
channel = InvidiousChannel.new({
id: ucid,
author: author,
updated: Time.utc,
deleted: false,
subscribed: nil,
})
channel.updated = Time.utc
return channel
end

97
src/invidious/channels/playlists.cr
View File

@ -1,93 +1,28 @@
def fetch_channel_playlists(ucid, author, continuation, sort_by)
if continuation
response_json = YoutubeAPI.browse(continuation)
continuation_items = response_json["onResponseReceivedActions"]?
.try &.[0]["appendContinuationItemsAction"]["continuationItems"]
return [] of SearchItem, nil if !continuation_items
items = [] of SearchItem
continuation_items.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item|
extract_item(item, author, ucid).try { |t| items << t }
}
continuation = continuation_items.as_a.last["continuationItemRenderer"]?
.try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s
initial_data = YoutubeAPI.browse(continuation)
else
url = "/channel/#{ucid}/playlists?flow=list&view=1"
params =
case sort_by
when "last", "last_added"
#
# Equivalent to "&sort=lad"
# {"2:string": "playlists", "3:varint": 4, "4:varint": 1, "6:varint": 1}
"EglwbGF5bGlzdHMYBCABMAE%3D"
when "oldest", "oldest_created"
url += "&sort=da"
# formerly "&sort=da"
# Not available anymore :c or maybe ??
# {"2:string": "playlists", "3:varint": 2, "4:varint": 1, "6:varint": 1}
"EglwbGF5bGlzdHMYAiABMAE%3D"
# {"2:string": "playlists", "3:varint": 1, "4:varint": 1, "6:varint": 1}
# "EglwbGF5bGlzdHMYASABMAE%3D"
when "newest", "newest_created"
url += "&sort=dd"
else nil # Ignore
# Formerly "&sort=dd"
# {"2:string": "playlists", "3:varint": 3, "4:varint": 1, "6:varint": 1}
"EglwbGF5bGlzdHMYAyABMAE%3D"
end
response = YT_POOL.client &.get(url)
initial_data = extract_initial_data(response.body)
return [] of SearchItem, nil if !initial_data
items = extract_items(initial_data, author, ucid)
continuation = response.body.match(/"token":"(?<continuation>[^"]+)"/).try &.["continuation"]?
initial_data = YoutubeAPI.browse(ucid, params: params || "")
end
return items, continuation
end
# ## NOTE: DEPRECATED
# Reason -> Unstable
# The Protobuf object must be provided with an id of the last playlist from the current "page"
# in order to fetch the next one accurately
# (if the id isn't included, entries shift around erratically between pages,
# leading to repetitions and skip overs)
#
# Since it's impossible to produce the appropriate Protobuf without an id being provided by the user,
# it's better to stick to continuation tokens provided by the first request and onward
def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated = false)
object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:base64" => {
"2:string" => "playlists",
"6:varint" => 2_i64,
"7:varint" => 1_i64,
"12:varint" => 1_i64,
"13:string" => "",
"23:varint" => 0_i64,
},
},
}
if cursor
cursor = Base64.urlsafe_encode(cursor, false) if !auto_generated
object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = cursor
end
if auto_generated
object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x32_i64
else
object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 1_i64
case sort
when "oldest", "oldest_created"
object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 2_i64
when "newest", "newest_created"
object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 3_i64
when "last", "last_added"
object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 4_i64
else nil # Ignore
end
end
object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"])))
object["80226972:embedded"].delete("3:base64")
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
return extract_items(initial_data, author, ucid)
end

166
src/invidious/channels/videos.cr
View File

@ -16,6 +16,14 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
sort_by_numerical =
case sort_by
when "newest" then 1_i64
when "popular" then 2_i64
when "oldest" then 3_i64 # Broken as of 10/2022 :c
else 1_i64 # Fallback to "newest"
end
object_inner_1 = {
"110:embedded" => {
"3:embedded" => {
@ -24,7 +32,7 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
"1:string" => object_inner_2_encoded,
"2:string" => "00000000-0000-0000-0000-000000000000",
},
"3:varint" => 1_i64,
"3:varint" => sort_by_numerical,
},
},
},
@ -52,34 +60,138 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
return continuation
end
def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest")
continuation = produce_channel_videos_continuation(ucid, page,
auto_generated: auto_generated, sort_by: sort_by, v2: true)
return YoutubeAPI.browse(continuation)
end
def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")
videos = [] of SearchVideo
# 2.times do |i|
# initial_data = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)
initial_data = get_channel_videos_response(ucid, 1, auto_generated: auto_generated, sort_by: sort_by)
videos = extract_videos(initial_data, author, ucid)
# end
return videos.size, videos
end
def get_latest_videos(ucid)
initial_data = get_channel_videos_response(ucid)
author = initial_data["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s
return extract_videos(initial_data, author, ucid)
end
# Used in bypass_captcha_job.cr
def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false)
continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2)
return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
end
module Invidious::Channel::Tabs
extend self
# -------------------
# Regular videos
# -------------------
def make_initial_video_ctoken(ucid, sort_by) : String
return produce_channel_videos_continuation(ucid, sort_by: sort_by)
end
# Wrapper for AboutChannel, as we still need to call get_videos with
# an author name and ucid directly (e.g in RSS feeds).
# TODO: figure out how to get rid of that
def get_videos(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
return get_videos(
channel.author, channel.ucid,
continuation: continuation, sort_by: sort_by
)
end
# Wrapper for InvidiousChannel, as we still need to call get_videos with
# an author name and ucid directly (e.g in RSS feeds).
# TODO: figure out how to get rid of that
def get_videos(channel : InvidiousChannel, *, continuation : String? = nil, sort_by = "newest")
return get_videos(
channel.author, channel.id,
continuation: continuation, sort_by: sort_by
)
end
def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest")
continuation ||= make_initial_video_ctoken(ucid, sort_by)
initial_data = YoutubeAPI.browse(continuation: continuation)
return extract_items(initial_data, author, ucid)
end
def get_60_videos(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
if continuation.nil?
# Fetch the first "page" of video
items, next_continuation = get_videos(channel, sort_by: sort_by)
else
# Fetch a "page" of videos using the given continuation token
items, next_continuation = get_videos(channel, continuation: continuation)
end
# If there is more to load, then load a second "page"
# and replace the previous continuation token
if !next_continuation.nil?
items_2, next_continuation = get_videos(channel, continuation: next_continuation)
items.concat items_2
end
return items, next_continuation
end
# -------------------
# Shorts
# -------------------
private def fetch_shorts_data(ucid : String, continuation : String? = nil)
if continuation.nil?
# EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts"
# TODO: try to extract the continuation tokens that allows other sorting options
return YoutubeAPI.browse(ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D")
else
return YoutubeAPI.browse(continuation: continuation)
end
end
def get_shorts(channel : AboutChannel, continuation : String? = nil)
initial_data = self.fetch_shorts_data(channel.ucid, continuation)
begin
# Try to parse the initial data fetched above
return extract_items(initial_data, channel.author, channel.ucid)
rescue ex : RetryOnceException
# Sometimes, for a completely unknown reason, the "reelItemRenderer"
# object is missing some critical information (it happens once in about
# 20 subsequent requests). Refreshing the page is required to properly
# show the "shorts" tab.
#
# In order to make the experience smoother for the user, we simulate
# said page refresh by fetching again the JSON. If that still doesn't
# work, we raise a BrokenTubeException, as something is really broken.
begin
initial_data = self.fetch_shorts_data(channel.ucid, continuation)
return extract_items(initial_data, channel.author, channel.ucid)
rescue ex : RetryOnceException
raise BrokenTubeException.new "reelPlayerHeaderSupportedRenderers"
end
end
end
# -------------------
# Livestreams
# -------------------
def get_livestreams(channel : AboutChannel, continuation : String? = nil)
if continuation.nil?
# EgdzdHJlYW1z8gYECgJ6AA%3D%3D is the protobuf object to load "streams"
initial_data = YoutubeAPI.browse(channel.ucid, params: "EgdzdHJlYW1z8gYECgJ6AA%3D%3D")
else
initial_data = YoutubeAPI.browse(continuation: continuation)
end
return extract_items(initial_data, channel.author, channel.ucid)
end
def get_60_livestreams(channel : AboutChannel, continuation : String? = nil)
if continuation.nil?
# Fetch the first "page" of streams
items, next_continuation = get_livestreams(channel)
else
# Fetch a "page" of streams using the given continuation token
items, next_continuation = get_livestreams(channel, continuation: continuation)
end
# If there is more to load, then load a second "page"
# and replace the previous continuation token
if !next_continuation.nil?
items_2, next_continuation = get_livestreams(channel, continuation: next_continuation)
items.concat items_2
end
return items, next_continuation
end
end

5
src/invidious/exceptions.cr
View File

@ -33,3 +33,8 @@ end
class VideoNotAvailableException < Exception
end
# Exception used to indicate that the JSON response from YT is missing
# some important informations, and that the query should be sent again.
class RetryOnceException < Exception
end

44
src/invidious/frontend/channel_page.cr Normal file
View File

@ -0,0 +1,44 @@
module Invidious::Frontend::ChannelPage
extend self
enum TabsAvailable
Videos
Shorts
Streams
Playlists
Community
Channels
end
def generate_tabs_links(locale : String, channel : AboutChannel, selected_tab : TabsAvailable)
return String.build(1500) do |str|
base_url = "/channel/#{channel.ucid}"
TabsAvailable.each do |tab|
# Ignore playlists, as it is not supported for auto-generated channels yet
next if (tab.playlists? && channel.auto_generated)
tab_name = tab.to_s.downcase
if channel.tabs.includes? tab_name
str << %(<div class="pure-u-1 pure-md-1-3">\n)
if tab == selected_tab
str << "\t<b>"
str << translate(locale, "channel_tab_#{tab_name}_label")
str << "</b>\n"
else
# Video tab doesn't have the last path component
url = tab.videos? ? base_url : "#{base_url}/#{tab_name}"
str << %(\t<a href=") << url << %(">)
str << translate(locale, "channel_tab_#{tab_name}_label")
str << "</a>\n"
end
str << "</div>"
end
end
end
end
end

3
src/invidious/hashtag.cr
View File

@ -8,7 +8,8 @@ module Invidious::Hashtag
client_config = YoutubeAPI::ClientConfig.new(region: region)
response = YoutubeAPI.browse(continuation: ctoken, client_config: client_config)
return extract_items(response)
items, _ = extract_items(response)
return items
end
def generate_continuation(hashtag : String, cursor : Int)

7
src/invidious/helpers/serialized_yt_data.cr
View File

@ -265,4 +265,11 @@ class Category
end
end
struct Continuation
getter token
def initialize(@token : String)
end
end
alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category

4
src/invidious/jobs/notification_job.cr
View File

@ -1,12 +1,12 @@
class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob
private getter connection_channel : Channel({Bool, Channel(PQ::Notification)})
private getter connection_channel : ::Channel({Bool, ::Channel(PQ::Notification)})
private getter pg_url : URI
def initialize(@connection_channel, @pg_url)
end
def begin
connections = [] of Channel(PQ::Notification)
connections = [] of ::Channel(PQ::Notification)
PG.connect_listen(pg_url, "notifications") { |event| connections.each(&.send(event)) }

2
src/invidious/jobs/refresh_channels_job.cr
View File

@ -8,7 +8,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
max_fibers = CONFIG.channel_threads
lim_fibers = max_fibers
active_fibers = 0
active_channel = Channel(Bool).new
active_channel = ::Channel(Bool).new
backoff = 2.minutes
loop do

2
src/invidious/jobs/refresh_feeds_job.cr
View File

@ -7,7 +7,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
def begin
max_fibers = CONFIG.feed_threads
active_fibers = 0
active_channel = Channel(Bool).new
active_channel = ::Channel(Bool).new
loop do
db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs|

2
src/invidious/jobs/subscribe_to_feeds_job.cr
View File

@ -12,7 +12,7 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob
end
active_fibers = 0
active_channel = Channel(Bool).new
active_channel = ::Channel(Bool).new
loop do
db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > interval '4 days' OR subscribed IS NULL") do |rs|

231
src/invidious/routes/api/v1/channels.cr
View File

@ -1,13 +1,7 @@
module Invidious::Routes::API::V1::Channels
def self.home(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
ucid = env.params.url["ucid"]
sort_by = env.params.query["sort_by"]?.try &.downcase
sort_by ||= "newest"
# Macro to avoid duplicating some code below
# This sets the `channel` variable, or handles Exceptions.
private macro get_channel
begin
channel = get_about_info(ucid, locale)
rescue ex : ChannelRedirect
@ -18,18 +12,26 @@ module Invidious::Routes::API::V1::Channels
rescue ex
return error_json(500, ex)
end
end
def self.home(env)
locale = env.get("preferences").as(Preferences).locale
ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
# Use the private macro defined above.
channel = nil # Make the compiler happy
get_channel()
# Retrieve "sort by" setting from URL parameters
sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
page = 1
if channel.auto_generated
videos = [] of SearchVideo
count = 0
else
begin
count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by)
rescue ex
return error_json(500, ex)
end
end
JSON.build do |json|
# TODO: Refactor into `to_json` for InvidiousChannel
@ -100,31 +102,13 @@ module Invidious::Routes::API::V1::Channels
json.array do
# Fetch related channels
begin
related_channels = fetch_related_channels(channel)
related_channels, _ = fetch_related_channels(channel)
rescue ex
related_channels = [] of AboutRelatedChannel
related_channels = [] of SearchChannel
end
related_channels.each do |related_channel|
json.object do
json.field "author", related_channel.author
json.field "authorId", related_channel.ucid
json.field "authorUrl", related_channel.author_url
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
end
related_channel.to_json(locale, json)
end
end
end # relatedChannels
@ -134,62 +118,113 @@ module Invidious::Routes::API::V1::Channels
end
def self.latest(env)
locale = env.get("preferences").as(Preferences).locale
# Remove parameters that could affect this endpoint's behavior
env.params.query.delete("sort_by") if env.params.query.has_key?("sort_by")
env.params.query.delete("continuation") if env.params.query.has_key?("continuation")
env.response.content_type = "application/json"
ucid = env.params.url["ucid"]
begin
videos = get_latest_videos(ucid)
rescue ex
return error_json(500, ex)
end
JSON.build do |json|
json.array do
videos.each do |video|
video.to_json(locale, json)
end
end
end
return self.videos(env)
end
def self.videos(env)
locale = env.get("preferences").as(Preferences).locale
ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
ucid = env.params.url["ucid"]
page = env.params.query["page"]?.try &.to_i?
page ||= 1
sort_by = env.params.query["sort"]?.try &.downcase
sort_by ||= env.params.query["sort_by"]?.try &.downcase
sort_by ||= "newest"
# Use the private macro defined above.
channel = nil # Make the compiler happy
get_channel()
# Retrieve some URL parameters
sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
continuation = env.params.query["continuation"]?
begin
channel = get_about_info(ucid, locale)
rescue ex : ChannelRedirect
env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
rescue ex : NotFoundException
return error_json(404, ex)
videos, next_continuation = Channel::Tabs.get_60_videos(
channel, continuation: continuation, sort_by: sort_by
)
rescue ex
return error_json(500, ex)
end
begin
count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
rescue ex
return error_json(500, ex)
end
JSON.build do |json|
return JSON.build do |json|
json.object do
json.field "videos" do
json.array do
videos.each do |video|
video.to_json(locale, json)
videos.each &.to_json(locale, json)
end
end
json.field "continuation", next_continuation if next_continuation
end
end
end
def self.shorts(env)
locale = env.get("preferences").as(Preferences).locale
ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
# Use the private macro defined above.
channel = nil # Make the compiler happy
get_channel()
# Retrieve continuation from URL parameters
continuation = env.params.query["continuation"]?
begin
videos, next_continuation = Channel::Tabs.get_shorts(
channel, continuation: continuation
)
rescue ex
return error_json(500, ex)
end
return JSON.build do |json|
json.object do
json.field "videos" do
json.array do
videos.each &.to_json(locale, json)
end
end
json.field "continuation", next_continuation if next_continuation
end
end
end
def self.streams(env)
locale = env.get("preferences").as(Preferences).locale
ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
# Use the private macro defined above.
channel = nil # Make the compiler happy
get_channel()
# Retrieve continuation from URL parameters
continuation = env.params.query["continuation"]?
begin
videos, next_continuation = Channel::Tabs.get_60_livestreams(
channel, continuation: continuation
)
rescue ex
return error_json(500, ex)
end
return JSON.build do |json|
json.object do
json.field "videos" do
json.array do
videos.each &.to_json(locale, json)
end
end
json.field "continuation", next_continuation if next_continuation
end
end
end
@ -204,16 +239,9 @@ module Invidious::Routes::API::V1::Channels
env.params.query["sort_by"]?.try &.downcase ||
"last"
begin
channel = get_about_info(ucid, locale)
rescue ex : ChannelRedirect
env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
rescue ex : NotFoundException
return error_json(404, ex)
rescue ex
return error_json(500, ex)
end
# Use the macro defined above
channel = nil # Make the compiler happy
get_channel()
items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
@ -255,6 +283,37 @@ module Invidious::Routes::API::V1::Channels
end
end
def self.channels(env)
locale = env.get("preferences").as(Preferences).locale
ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
# Use the macro defined above
channel = nil # Make the compiler happy
get_channel()
continuation = env.params.query["continuation"]?
begin
items, next_continuation = fetch_related_channels(channel, continuation)
rescue ex
return error_json(500, ex)
end
JSON.build do |json|
json.object do
json.field "relatedChannels" do
json.array do
items.each &.to_json(locale, json)
end
end
json.field "continuation", next_continuation if next_continuation
end
end
end
def self.search(env)
locale = env.get("preferences").as(Preferences).locale
region = env.params.query["region"]?

108
src/invidious/routes/channels.cr
View File

@ -7,21 +7,19 @@ module Invidious::Routes::Channels
def self.videos(env)
data = self.fetch_basic_information(env)
if !data.is_a?(Tuple)
return data
end
locale, user, subscriptions, continuation, ucid, channel = data
return data if !data.is_a?(Tuple)
page = env.params.query["page"]?.try &.to_i?
page ||= 1
locale, user, subscriptions, continuation, ucid, channel = data
sort_by = env.params.query["sort_by"]?.try &.downcase
if channel.auto_generated
sort_options = {"last", "oldest", "newest"}
sort_by ||= "last"
items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
items, next_continuation = fetch_channel_playlists(
channel.ucid, channel.author, continuation, (sort_by || "last")
)
items.uniq! do |item|
if item.responds_to?(:title)
item.title
@ -33,34 +31,85 @@ module Invidious::Routes::Channels
items.each(&.author = "")
else
sort_options = {"newest", "oldest", "popular"}
sort_by ||= "newest"
count, items = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
# Fetch items and continuation token
items, next_continuation = Channel::Tabs.get_videos(
channel, continuation: continuation, sort_by: (sort_by || "newest")
)
end
selected_tab = Frontend::ChannelPage::TabsAvailable::Videos
templated "channel"
end
def self.shorts(env)
data = self.fetch_basic_information(env)
return data if !data.is_a?(Tuple)
locale, user, subscriptions, continuation, ucid, channel = data
if !channel.tabs.includes? "shorts"
return env.redirect "/channel/#{channel.ucid}"
end
# TODO: support sort option for shorts
sort_by = ""
sort_options = [] of String
# Fetch items and continuation token
items, next_continuation = Channel::Tabs.get_shorts(
channel, continuation: continuation
)
selected_tab = Frontend::ChannelPage::TabsAvailable::Shorts
templated "channel"
end
def self.streams(env)
data = self.fetch_basic_information(env)
return data if !data.is_a?(Tuple)
locale, user, subscriptions, continuation, ucid, channel = data
if !channel.tabs.includes? "streams"
return env.redirect "/channel/#{channel.ucid}"
end
# TODO: support sort option for livestreams
sort_by = ""
sort_options = [] of String
# Fetch items and continuation token
items, next_continuation = Channel::Tabs.get_60_livestreams(
channel, continuation: continuation
)
selected_tab = Frontend::ChannelPage::TabsAvailable::Streams
templated "channel"
end
def self.playlists(env)
data = self.fetch_basic_information(env)
if !data.is_a?(Tuple)
return data
end
return data if !data.is_a?(Tuple)
locale, user, subscriptions, continuation, ucid, channel = data
sort_options = {"last", "oldest", "newest"}
sort_by = env.params.query["sort_by"]?.try &.downcase
sort_by ||= "last"
if channel.auto_generated
return env.redirect "/channel/#{channel.ucid}"
end
items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
items, next_continuation = fetch_channel_playlists(
channel.ucid, channel.author, continuation, (sort_by || "last")
)
items = items.select(SearchPlaylist).map(&.as(SearchPlaylist))
items.each(&.author = "")
templated "playlists"
selected_tab = Frontend::ChannelPage::TabsAvailable::Playlists
templated "channel"
end
def self.community(env)
@ -74,12 +123,15 @@ module Invidious::Routes::Channels
thin_mode = thin_mode == "true"
continuation = env.params.query["continuation"]?
# sort_by = env.params.query["sort_by"]?.try &.downcase
if !channel.tabs.includes? "community"
return env.redirect "/channel/#{channel.ucid}"
end
# TODO: support sort options for community posts
sort_by = ""
sort_options = [] of String
begin
items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode))
rescue ex : InfoException
@ -95,6 +147,26 @@ module Invidious::Routes::Channels
templated "community"
end
def self.channels(env)
data = self.fetch_basic_information(env)
return data if !data.is_a?(Tuple)
locale, user, subscriptions, continuation, ucid, channel = data
if channel.auto_generated
return env.redirect "/channel/#{channel.ucid}"
end
items, next_continuation = fetch_related_channels(channel, continuation)
# Featured/related channels can't be sorted
sort_options = [] of String
sort_by = nil
selected_tab = Frontend::ChannelPage::TabsAvailable::Channels
templated "channel"
end
def self.about(env)
data = self.fetch_basic_information(env)
if !data.is_a?(Tuple)
@ -125,7 +197,7 @@ module Invidious::Routes::Channels
end
selected_tab = env.request.path.split("/")[-1]
if ["home", "videos", "playlists", "community", "channels", "about"].includes? selected_tab
if {"home", "videos", "shorts", "streams", "playlists", "community", "channels", "about"}.includes? selected_tab
url = "/channel/#{ucid}/#{selected_tab}"
else
url = "/channel/#{ucid}"

9
src/invidious/routing.cr
View File

@ -117,14 +117,17 @@ module Invidious::Routing
get "/channel/:ucid", Routes::Channels, :home
get "/channel/:ucid/home", Routes::Channels, :home
get "/channel/:ucid/videos", Routes::Channels, :videos
get "/channel/:ucid/shorts", Routes::Channels, :shorts
get "/channel/:ucid/streams", Routes::Channels, :streams
get "/channel/:ucid/playlists", Routes::Channels, :playlists
get "/channel/:ucid/community", Routes::Channels, :community
get "/channel/:ucid/channels", Routes::Channels, :channels
get "/channel/:ucid/about", Routes::Channels, :about
get "/channel/:ucid/live", Routes::Channels, :live
get "/user/:user/live", Routes::Channels, :live
get "/c/:user/live", Routes::Channels, :live
["", "/videos", "/playlists", "/community", "/about"].each do |path|
{"", "/videos", "/shorts", "/streams", "/playlists", "/community", "/about"}.each do |path|
# /c/LinusTechTips
get "/c/:user#{path}", Routes::Channels, :brand_redirect
# /user/linustechtips | Not always the same as /c/
@ -222,6 +225,10 @@ module Invidious::Routing
# Channels
get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home
get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts
get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams
get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels
{% for route in {"videos", "latest", "playlists", "community", "search"} %}
get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}}
get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}}

14
src/invidious/search/processors.cr
View File

@ -9,7 +9,8 @@ module Invidious::Search
client_config = YoutubeAPI::ClientConfig.new(region: query.region)
initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config)
return extract_items(initial_data)
items, _ = extract_items(initial_data)
return items
end
# Search a youtube channel
@ -30,16 +31,7 @@ module Invidious::Search
continuation = produce_channel_search_continuation(ucid, query.text, query.page)
response_json = YoutubeAPI.browse(continuation)
continuation_items = response_json["onResponseReceivedActions"]?
.try &.[0]["appendContinuationItemsAction"]["continuationItems"]
return [] of SearchItem if !continuation_items
items = [] of SearchItem
continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item|
extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t }
end
items, _ = extract_items(response_json, "", ucid)
return items
end

122
src/invidious/views/channel.ecr
View File

@ -1,8 +1,24 @@
<% ucid = channel.ucid %>
<% author = HTML.escape(channel.author) %>
<% channel_profile_pic = URI.parse(channel.author_thumbnail).request_target %>
<%-
ucid = channel.ucid
author = HTML.escape(channel.author)
channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
relative_url =
case selected_tab
when .shorts? then "/channel/#{ucid}/shorts"
when .streams? then "/channel/#{ucid}/streams"
when .playlists? then "/channel/#{ucid}/playlists"
when .channels? then "/channel/#{ucid}/channels"
else
"/channel/#{ucid}"
end
youtube_url = "https://www.youtube.com#{relative_url}"
redirect_url = Invidious::Frontend::Misc.redirect_url(env)
-%>
<% content_for "header" do %>
<%- if selected_tab.videos? -%>
<meta name="description" content="<%= channel.description %>">
<meta property="og:site_name" content="Invidious">
<meta property="og:url" content="<%= HOST_URL %>/channel/<%= ucid %>">
@ -14,91 +30,14 @@
<meta name="twitter:title" content="<%= author %>">
<meta name="twitter:description" content="<%= channel.description %>">
<meta name="twitter:image" content="/ggpht<%= channel_profile_pic %>">
<link rel="alternate" href="https://www.youtube.com/channel/<%= ucid %>">
<title><%= author %> - Invidious</title>
<link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/channel/<%= ucid %>" />
<%- end -%>
<link rel="alternate" href="<%= youtube_url %>">
<title><%= author %> - Invidious</title>
<% end %>
<% if channel.banner %>
<div class="h-box">
<img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>">
</div>
<div class="h-box">
<hr>
</div>
<% end %>
<div class="pure-g h-box">
<div class="pure-u-2-3">
<div class="channel-profile">
<img src="/ggpht<%= channel_profile_pic %>">
<span><%= author %></span><% if !channel.verified.nil? && channel.verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %>
</div>
</div>
<div class="pure-u-1-3">
<h3 style="text-align:right">
<a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a>
</h3>
</div>
</div>
<div class="h-box">
<div id="descriptionWrapper">
<p><span style="white-space:pre-wrap"><%= channel.description_html %></span></p>
</div>
</div>
<div class="h-box">
<% sub_count_text = number_to_short_text(channel.sub_count) %>
<%= rendered "components/subscribe_widget" %>
</div>
<div class="pure-g h-box">
<div class="pure-u-1-3">
<a href="https://www.youtube.com/channel/<%= ucid %>"><%= translate(locale, "View channel on YouTube") %></a>
<div class="pure-u-1 pure-md-1-3">
<% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
<a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a>
<% else %>
<a href="https://redirect.invidious.io<%= env.request.path %>"><%= translate(locale, "Switch Invidious Instance") %></a>
<% end %>
</div>
<% if !channel.auto_generated %>
<div class="pure-u-1 pure-md-1-3">
<b><%= translate(locale, "Videos") %></b>
</div>
<% end %>
<div class="pure-u-1 pure-md-1-3">
<% if channel.auto_generated %>
<b><%= translate(locale, "Playlists") %></b>
<% else %>
<a href="/channel/<%= ucid %>/playlists"><%= translate(locale, "Playlists") %></a>
<% end %>
</div>
<div class="pure-u-1 pure-md-1-3">
<% if channel.tabs.includes? "community" %>
<a href="/channel/<%= ucid %>/community"><%= translate(locale, "Community") %></a>
<% end %>
</div>
</div>
<div class="pure-u-1-3"></div>
<div class="pure-u-1-3">
<div class="pure-g" style="text-align:right">
<% sort_options.each do |sort| %>
<div class="pure-u-1 pure-md-1-3">
<% if sort_by == sort %>
<b><%= translate(locale, sort) %></b>
<% else %>
<a href="/channel/<%= ucid %>?page=<%= page %>&sort_by=<%= sort %>">
<%= translate(locale, sort) %>
</a>
<% end %>
</div>
<% end %>
</div>
</div>
</div>
<%= rendered "components/channel_info" %>
<div class="h-box">
<hr>
@ -111,17 +50,10 @@
</div>
<div class="pure-g h-box">
<div class="pure-u-1 pure-u-lg-1-5">
<% if page > 1 %>
<a href="/channel/<%= ucid %>?page=<%= page - 1 %><% if sort_by != "newest" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
<%= translate(locale, "Previous page") %>
</a>
<% end %>
</div>
<div class="pure-u-1 pure-u-lg-3-5"></div>
<div class="pure-u-1 pure-u-md-4-5"></div>
<div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
<% if count == 60 %>
<a href="/channel/<%= ucid %>?page=<%= page + 1 %><% if sort_by != "newest" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
<% if next_continuation %>
<a href="<%= relative_url %>?continuation=<%= next_continuation %><% if sort_options.any? sort_by %>&sort_by=<%= sort_by %><% end %>">
<%= translate(locale, "Next page") %>
</a>
<% end %>

76
src/invidious/views/community.ecr
View File

@ -1,71 +1,21 @@
<% ucid = channel.ucid %>
<% author = HTML.escape(channel.author) %>
<%-
ucid = channel.ucid
author = HTML.escape(channel.author)
channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
relative_url = "/channel/#{ucid}/community"
youtube_url = "https://www.youtube.com#{relative_url}"
redirect_url = Invidious::Frontend::Misc.redirect_url(env)
selected_tab = Invidious::Frontend::ChannelPage::TabsAvailable::Community
-%>
<% content_for "header" do %>
<link rel="alternate" href="<%= youtube_url %>">
<title><%= author %> - Invidious</title>
<% end %>
<% if channel.banner %>
<div class="h-box">
<img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>">
</div>
<div class="h-box">
<hr>
</div>
<% end %>
<div class="pure-g h-box">
<div class="pure-u-2-3">
<div class="channel-profile">
<img src="/ggpht<%= URI.parse(channel.author_thumbnail).request_target %>">
<span><%= author %></span><% if !channel.verified.nil? && channel.verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %>
</div>
</div>
<div class="pure-u-1-3" style="text-align:right">
<h3 style="text-align:right">
<a href="/feed/channel/<%= channel.ucid %>"><i class="icon ion-logo-rss"></i></a>
</h3>
</div>
</div>
<div class="h-box">
<div id="descriptionWrapper">
<p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %></span></p>
</div>
</div>
<div class="h-box">
<% sub_count_text = number_to_short_text(channel.sub_count) %>
<%= rendered "components/subscribe_widget" %>
</div>
<div class="pure-g h-box">
<div class="pure-u-1-3">
<a href="https://www.youtube.com/channel/<%= channel.ucid %>/community"><%= translate(locale, "View channel on YouTube") %></a>
<div class="pure-u-1 pure-md-1-3">
<% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
<a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a>
<% else %>
<a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a>
<% end %>
</div>
<% if !channel.auto_generated %>
<div class="pure-u-1 pure-md-1-3">
<a href="/channel/<%= channel.ucid %>"><%= translate(locale, "Videos") %></a>
</div>
<% end %>
<div class="pure-u-1 pure-md-1-3">
<a href="/channel/<%= channel.ucid %>/playlists"><%= translate(locale, "Playlists") %></a>
</div>
<div class="pure-u-1 pure-md-1-3">
<% if channel.tabs.includes? "community" %>
<b><%= translate(locale, "Community") %></b>
<% end %>
</div>
</div>
<div class="pure-u-2-3"></div>
</div>
<%= rendered "components/channel_info" %>
<div class="h-box">
<hr>

60
src/invidious/views/components/channel_info.ecr Normal file
View File

@ -0,0 +1,60 @@
<% if channel.banner %>
<div class="h-box">
<img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>">
</div>
<div class="h-box">
<hr>
</div>
<% end %>
<div class="pure-g h-box">
<div class="pure-u-2-3">
<div class="channel-profile">
<img src="/ggpht<%= channel_profile_pic %>">
<span><%= author %></span><% if !channel.verified.nil? && channel.verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %>
</div>
</div>
<div class="pure-u-1-3">
<h3 style="text-align:right">
<a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a>
</h3>
</div>
</div>
<div class="h-box">
<div id="descriptionWrapper">
<p><span style="white-space:pre-wrap"><%= channel.description_html %></span></p>
</div>
</div>
<div class="h-box">
<% sub_count_text = number_to_short_text(channel.sub_count) %>
<%= rendered "components/subscribe_widget" %>
</div>
<div class="pure-g h-box">
<div class="pure-u-1-2">
<div class="pure-u-1 pure-md-1-3">
<a href="<%= youtube_url %>"><%= translate(locale, "View channel on YouTube") %></a>
</div>
<div class="pure-u-1 pure-md-1-3">
<a href="<%= redirect_url %>"><%= translate(locale, "Switch Invidious Instance") %></a>
</div>
<%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, selected_tab) %>
</div>
<div class="pure-u-1-2">
<div class="pure-g" style="text-align:end">
<% sort_options.each do |sort| %>
<div class="pure-u-1 pure-md-1-3">
<% if sort_by == sort %>
<b><%= translate(locale, sort) %></b>
<% else %>
<a href="<%= relative_url %>?sort_by=<%= sort %>"><%= translate(locale, sort) %></a>
<% end %>
</div>
<% end %>
</div>
</div>
</div>

108
src/invidious/views/playlists.ecr
View File

@ -1,108 +0,0 @@
<% ucid = channel.ucid %>
<% author = HTML.escape(channel.author) %>
<% content_for "header" do %>
<title><%= author %> - Invidious</title>
<% end %>
<% if channel.banner %>
<div class="h-box">
<img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>">
</div>
<div class="h-box">
<hr>
</div>
<% end %>
<div class="pure-g h-box">
<div class="pure-u-2-3">
<div class="channel-profile">
<img src="/ggpht<%= URI.parse(channel.author_thumbnail).request_target %>">
<span><%= author %></span><% if !channel.verified.nil? && channel.verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %>
</div>
</div>
<div class="pure-u-1-3" style="text-align:right">
<h3 style="text-align:right">
<a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a>
</h3>
</div>
</div>
<div class="h-box">
<div id="descriptionWrapper">
<p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content if !channel.description_html.empty? %></span></p>
</div>
</div>
<div class="h-box">
<% sub_count_text = number_to_short_text(channel.sub_count) %>
<%= rendered "components/subscribe_widget" %>
</div>
<div class="pure-g h-box">
<div class="pure-g pure-u-1-3">
<div class="pure-u-1 pure-md-1-3">
<a href="https://www.youtube.com/channel/<%= ucid %>/playlists"><%= translate(locale, "View channel on YouTube") %></a>
</div>
<div class="pure-u-1 pure-md-1-3">
<% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
<a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a>
<% else %>
<a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a>
<% end %>
</div>
<div class="pure-u-1 pure-md-1-3">
<a href="/channel/<%= ucid %>"><%= translate(locale, "Videos") %></a>
</div>
<div class="pure-u-1 pure-md-1-3">
<% if !channel.auto_generated %>
<b><%= translate(locale, "Playlists") %></b>
<% end %>
</div>
<div class="pure-u-1 pure-md-1-3">
<% if channel.tabs.includes? "community" %>
<a href="/channel/<%= ucid %>/community"><%= translate(locale, "Community") %></a>
<% end %>
</div>
</div>
<div class="pure-u-1-3"></div>
<div class="pure-u-1-3">
<div class="pure-g" style="text-align:right">
<% {"last", "oldest", "newest"}.each do |sort| %>
<div class="pure-u-1 pure-md-1-3">
<% if sort_by == sort %>
<b><%= translate(locale, sort) %></b>
<% else %>
<a href="/channel/<%= ucid %>/playlists?sort_by=<%= sort %>">
<%= translate(locale, sort) %>
</a>
<% end %>
</div>
<% end %>
</div>
</div>
</div>
<div class="h-box">
<hr>
</div>
<div class="pure-g">
<% items.each do |item| %>
<%= rendered "components/item" %>
<% end %>
</div>
<div class="pure-g h-box">
<div class="pure-u-1 pure-u-md-4-5"></div>
<div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
<% if continuation %>
<a href="/channel/<%= ucid %>/playlists?continuation=<%= continuation %><% if sort_by != "last" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
<%= translate(locale, "Next page") %>
</a>
<% end %>
</div>
</div>

132
src/invidious/yt_backend/extractors.cr
View File

@ -7,7 +7,7 @@ require "../helpers/serialized_yt_data"
private ITEM_CONTAINER_EXTRACTOR = {
Extractors::YouTubeTabs,
Extractors::SearchResults,
Extractors::Continuation,
Extractors::ContinuationContent,
}
private ITEM_PARSERS = {
@ -18,8 +18,11 @@ private ITEM_PARSERS = {
Parsers::CategoryRendererParser,
Parsers::RichItemRendererParser,
Parsers::ReelItemRendererParser,
Parsers::ContinuationItemRendererParser,
}
private alias InitialData = Hash(String, JSON::Any)
record AuthorFallback, name : String, id : String
# Namespace for logic relating to parsing InnerTube data into various datastructs.
@ -345,14 +348,9 @@ private module Parsers
content_container = item_contents["contents"]
end
raw_contents = content_container["items"]?.try &.as_a
if !raw_contents.nil?
raw_contents.each do |item|
result = extract_item(item)
if !result.nil?
contents << result
end
end
content_container["items"]?.try &.as_a.each do |item|
result = parse_item(item, author_fallback.name, author_fallback.id)
contents << result if result.is_a?(SearchItem)
end
Category.new({
@ -384,7 +382,9 @@ private module Parsers
end
private def self.parse(item_contents, author_fallback)
return VideoRendererParser.process(item_contents, author_fallback)
child = VideoRendererParser.process(item_contents, author_fallback)
child ||= ReelItemRendererParser.process(item_contents, author_fallback)
return child
end
def self.parser_name
@ -408,9 +408,19 @@ private module Parsers
private def self.parse(item_contents, author_fallback)
video_id = item_contents["videoId"].as_s
video_details_container = item_contents.dig(
reel_player_overlay = item_contents.dig(
"navigationEndpoint", "reelWatchEndpoint",
"overlay", "reelPlayerOverlayRenderer",
"overlay", "reelPlayerOverlayRenderer"
)
# Sometimes, the "reelPlayerOverlayRenderer" object is missing the
# important part of the response. We use this exception to tell
# the calling function to fetch the content again.
if !reel_player_overlay.as_h.has_key?("reelPlayerHeaderSupportedRenderers")
raise RetryOnceException.new
end
video_details_container = reel_player_overlay.dig(
"reelPlayerHeaderSupportedRenderers",
"reelPlayerHeaderRenderer"
)
@ -436,9 +446,9 @@ private module Parsers
# View count
view_count_text = video_details_container.dig?("viewCountText", "simpleText")
view_count_text ||= video_details_container
.dig?("viewCountText", "accessibility", "accessibilityData", "label")
# View count used to be in the reelWatchEndpoint, but that changed?
view_count_text = item_contents.dig?("viewCountText", "simpleText")
view_count_text ||= video_details_container.dig?("viewCountText", "simpleText")
view_count = view_count_text.try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64
@ -450,8 +460,8 @@ private module Parsers
regex_match = /- (?<min>\d+ minutes? )?(?<sec>\d+ seconds?)+ -/.match(a11y_data)
minutes = regex_match.try &.["min"].to_i(strict: false) || 0
seconds = regex_match.try &.["sec"].to_i(strict: false) || 0
minutes = regex_match.try &.["min"]?.try &.to_i(strict: false) || 0
seconds = regex_match.try &.["sec"]?.try &.to_i(strict: false) || 0
duration = (minutes*60 + seconds)
@ -475,6 +485,35 @@ private module Parsers
return {{@type.name}}
end
end
# Parses an InnerTube continuationItemRenderer into a Continuation.
# Returns nil when the given object isn't a continuationItemRenderer.
#
# continuationItemRenderer contains various metadata ued to load more
# content (i.e when the user scrolls down). The interesting bit is the
# protobuf object known as the "continutation token". Previously, those
# were generated from sratch, but recent (as of 11/2022) Youtube changes
# are forcing us to extract them from replies.
#
module ContinuationItemRendererParser
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = item["continuationItemRenderer"]?
return self.parse(item_contents)
end
end
private def self.parse(item_contents)
token = item_contents
.dig?("continuationEndpoint", "continuationCommand", "token")
.try &.as_s
return Continuation.new(token) if token
end
def self.parser_name
return {{@type.name}}
end
end
end
# The following are the extractors for extracting an array of items from
@ -510,7 +549,7 @@ private module Extractors
# }]
#
module YouTubeTabs
def self.process(initial_data : Hash(String, JSON::Any))
def self.process(initial_data : InitialData)
if target = initial_data["twoColumnBrowseResultsRenderer"]?
self.extract(target)
end
@ -575,7 +614,7 @@ private module Extractors
# }
#
module SearchResults
def self.process(initial_data : Hash(String, JSON::Any))
def self.process(initial_data : InitialData)
if target = initial_data["twoColumnSearchResultsRenderer"]?
self.extract(target)
end
@ -608,8 +647,8 @@ private module Extractors
# The way they are structured is too varied to be accurately written down here.
# However, they all eventually lead to an array of parsable items after traversing
# through the JSON structure.
module Continuation
def self.process(initial_data : Hash(String, JSON::Any))
module ContinuationContent
def self.process(initial_data : InitialData)
if target = initial_data["continuationContents"]?
self.extract(target)
elsif target = initial_data["appendContinuationItemsAction"]?
@ -691,8 +730,7 @@ end
# Parses an item from Youtube's JSON response into a more usable structure.
# The end result can either be a SearchVideo, SearchPlaylist or SearchChannel.
def extract_item(item : JSON::Any, author_fallback : String? = "",
author_id_fallback : String? = "")
def parse_item(item : JSON::Any, author_fallback : String? = "", author_id_fallback : String? = "")
# We "allow" nil values but secretly use empty strings instead. This is to save us the
# hassle of modifying every author_fallback and author_id_fallback arg usage
# which is more often than not nil.
@ -702,24 +740,23 @@ def extract_item(item : JSON::Any, author_fallback : String? = "",
# Each parser automatically validates the data given to see if the data is
# applicable to itself. If not nil is returned and the next parser is attempted.
ITEM_PARSERS.each do |parser|
LOGGER.trace("extract_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)")
LOGGER.trace("parse_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)")
if result = parser.process(item, author_fallback)
LOGGER.debug("extract_item: Successfully parsed via #{parser.parser_name}")
LOGGER.debug("parse_item: Successfully parsed via #{parser.parser_name}")
return result
else
LOGGER.trace("extract_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...")
LOGGER.trace("parse_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...")
end
end
end
# Parses multiple items from YouTube's initial JSON response into a more usable structure.
# The end result is an array of SearchItem.
def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil,
author_id_fallback : String? = nil) : Array(SearchItem)
items = [] of SearchItem
#
# This function yields the container so that items can be parsed separately.
#
def extract_items(initial_data : InitialData, &block)
if unpackaged_data = initial_data["contents"]?.try &.as_h
elsif unpackaged_data = initial_data["response"]?.try &.as_h
elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 0).try &.as_h
@ -727,24 +764,37 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
unpackaged_data = initial_data
end
# This is identical to the parser cycling of extract_item().
# This is identical to the parser cycling of parse_item().
ITEM_CONTAINER_EXTRACTOR.each do |extractor|
LOGGER.trace("extract_items: Attempting to extract item container using \"#{extractor.extractor_name}\" (cycling...)")
if container = extractor.process(unpackaged_data)
LOGGER.debug("extract_items: Successfully unpacked container with \"#{extractor.extractor_name}\"")
# Extract items in container
container.each do |item|
if parsed_result = extract_item(item, author_fallback, author_id_fallback)
items << parsed_result
end
end
break
container.each { |item| yield item }
else
LOGGER.trace("extract_items: Extractor \"#{extractor.extractor_name}\" does not apply. Cycling to the next one...")
end
end
return items
end
# Wrapper using the block function above
def extract_items(
initial_data : InitialData,
author_fallback : String? = nil,
author_id_fallback : String? = nil
) : {Array(SearchItem), String?}
items = [] of SearchItem
continuation = nil
extract_items(initial_data) do |item|
parsed = parse_item(item, author_fallback, author_id_fallback)
case parsed
when .is_a?(Continuation) then continuation = parsed.token
when .is_a?(SearchItem) then items << parsed
end
end
return items, continuation
end

27
src/invidious/yt_backend/extractors_utils.cr
View File

@ -68,10 +68,10 @@ rescue ex
return false
end
def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
extracted = extract_items(initial_data, author_fallback, author_id_fallback)
def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) : Array(SearchVideo)
extracted, _ = extract_items(initial_data, author_fallback, author_id_fallback)
target = [] of SearchItem
target = [] of (SearchItem | Continuation)
extracted.each do |i|
if i.is_a?(Category)
i.contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video }
@ -79,28 +79,11 @@ def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : Str
target << i
end
end
return target.select(SearchVideo).map(&.as(SearchVideo))
return target.select(SearchVideo)
end
def extract_selected_tab(tabs)
# Extract the selected tab from the array of tabs Youtube returns
return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"]?.try &.as_bool)[0]["tabRenderer"]
end
def fetch_continuation_token(items : Array(JSON::Any))
# Fetches the continuation token from an array of items
return items.last["continuationItemRenderer"]?
.try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s
end
def fetch_continuation_token(initial_data : Hash(String, JSON::Any))
# Fetches the continuation token from initial data
if initial_data["onResponseReceivedActions"]?
continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"]
else
tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"])
continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"]
end
return fetch_continuation_token(continuation_items.as_a)
end