Add support channel home pages + gen. improvements

This commit adds support for channel home pages and all of the
categories within it. However, the frontend code is a mess and thus
needs to be refactor soon. Though that would likely require a rework of
items.ecr

This commit also comes with some general cleanups and improvements.

Before this commit channel brand URls would only be supported on the
videos page (now home page). It has been improved to be able to handle
all channel URLs.

The category_type and auxiliary_data property has also been removed from
the Category struct. The former was never used and the latter allows for
random data to be added to the Struct presenting documentation issues.

Since the auxiliary_data variable was mainly used to store values from
the browse_endpoint in order to create URLs, its much simpler to instead
just get the URL from the webCommandMetadata.

As a result of this change the browse_endpoint_data attribute of
Category has also been removed.

assets/css/channel.css

@@ -66,7 +66,6 @@
 }
 
 .category-heading {
-  font-size: 1.2em;
   user-select: none;
   display: inline;
 }
@@ -117,3 +116,23 @@ only show up when the screen is wide enough */
     margin-top: 1em;
   }
 }
+
+.trailer-metadata {
+  margin-left: 15px;
+  font-size: 12px;
+  color: rgb(232, 230, 227);
+}
+
+.trailer-metadata .read-more {
+  line-height: 20px;
+  text-transform: uppercase;
+  color: gray;
+  font-size: 13px;
+}
+
+.trailer-description {
+  overflow: hidden;
+  max-height: 150px;
+  line-height: 20px;
+  margin-top: 20px;
+}

assets/css/default.css

@@ -602,7 +602,8 @@ hr {
 }
 
 .category {
-  margin: 3em 0px 4em 0px;
+  margin-bottom: 2em;
+  margin-top: 1em;
 }
 
 .category .heading > p {
@@ -616,4 +617,9 @@ hr {
   border-radius: 5px;
   font-size: 14px;
   margin-left: 10px;
+}
+
+ /* Temp */ 
+.category-description {
+  color: #A8A095;
 }

src/invidious.cr

@@ -315,6 +315,10 @@ Invidious::Routing.get "/channel/:ucid/community", Invidious::Routes::Channels,
 Invidious::Routing.get "/channel/:ucid/channels", Invidious::Routes::Channels, :channels
 Invidious::Routing.get "/channel/:ucid/about", Invidious::Routes::Channels, :about
 
+["", "/home", "/videos", "/playlists", "/community", "/channels", "/about"].each do |path|
+  Invidious::Routing.get "/c/:user#{path}", Invidious::Routes::Channels, :brand_redirect
+end
+
 Invidious::Routing.get "/watch", Invidious::Routes::Watch, :handle
 Invidious::Routing.get "/watch/:id", Invidious::Routes::Watch, :redirect
 Invidious::Routing.get "/shorts/:id", Invidious::Routes::Watch, :redirect
@@ -1624,22 +1628,6 @@ end
   end
 end
 
-# YouTube appears to let users set a "brand" URL that
-# is different from their username, so we convert that here
-get "/c/:user" do |env|
-  locale = LOCALES[env.get("preferences").as(Preferences).locale]?
-
-  user = env.params.url["user"]
-
-  response = YT_POOL.client &.get("/c/#{user}")
-  html = XML.parse_html(response.body)
-
-  ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1]
-  next env.redirect "/" if !ucid
-
-  env.redirect "/channel/#{ucid}"
-end
-
 # Legacy endpoint for /user/:username
 get "/profile" do |env|
   user = env.params.query["user"]?

src/invidious/channels.cr

@@ -341,6 +341,30 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
   return channel
 end
 
+def fetch_channel_home(ucid, channel)
+  initial_data = request_youtube_api_browse(ucid, channel.tabs["home"][1])
+  items = extract_items(initial_data, channel.author, channel.ucid)
+
+  # Channel trailer needs some slight special handling
+  home_tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"])
+  trailer = home_tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["channelVideoPlayerRenderer"]? || nil
+
+  home_sections = [] of (Category | Video)
+  if trailer
+    trailer = get_video(trailer["videoId"].as_s, PG_DB)
+    home_sections << trailer
+  end
+
+  items.each do |category|
+    if category.is_a? Category
+      home_sections << category
+    end
+  end
+
+  return home_sections
+
+end
+
 def fetch_channel_playlists(ucid, author, continuation, sort_by)
   if continuation
     response_json = request_youtube_api_browse(continuation)
@@ -381,8 +405,6 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by)
 end
 
 def fetch_channel_featured_channels(ucid, tab_data, view = nil, shelf_id = nil, continuation = nil, query_title = nil) : {Array(Category), (String | Nil)}
-  auxiliary_data = {} of String => String
-
   if continuation.is_a?(String)
     initial_data = request_youtube_api_browse(continuation)
     items = extract_items(initial_data)
@@ -392,14 +414,13 @@ def fetch_channel_featured_channels(ucid, tab_data, view = nil, shelf_id = nil,
       title:                query_title.not_nil!, # If continuation contents is requested then the query_title has to be passed along.
       contents:             items,
       description_html:     "",
-      browse_endpoint_data: nil,
+      url:                  nil,
       badges:               nil,
-      auxiliary_data:       auxiliary_data,
     })], continuation_token
   else
+    url = nil
     if view && shelf_id
-      auxiliary_data["view"] = view
+      url = "/channel/#{ucid}/channels?view=#{view}&shelf_id=#{shelf_id}"
-      auxiliary_data["shelf_id"] = shelf_id
 
       params = produce_featured_channel_browse_param(view.to_i64, shelf_id.to_i64)
       initial_data = request_youtube_api_browse(ucid, params)
@@ -437,21 +458,20 @@ def fetch_channel_featured_channels(ucid, tab_data, view = nil, shelf_id = nil,
         title:                category.title.empty? ? fallback_title : category.title,
         contents:             category.contents,
         description_html:     category.description_html,
-        browse_endpoint_data: nil,
+        url:                  category.url,
         badges:               nil,
-        auxiliary_data:       category.auxiliary_data,
       })
     end
 
-    # If we don't have any categories we'll create one.
+    # If no categories has been parsed then it means that we're currently requesting a single one and not in
+    # the initial preview anymore. The frontend still needs a Category however, so we'll create one.
     if category_array.empty?
       category_array << Category.new({
         title:                fallback_title,
         contents:             items,
         description_html:     "",
-        browse_endpoint_data: nil,
+        url:                  url,
         badges:               nil,
-        auxiliary_data:       auxiliary_data,
       })
     end
 

src/invidious/helpers/extractors.cr

@@ -219,37 +219,7 @@ private class CategoryParser < ItemParser
       title = ""
     end
 
-    auxiliary_data = {} of String => String
+    url = item_contents["endpoint"]?.try &.["commandMetadata"]["webCommandMetadata"]["url"].as_s
-    browse_endpoint = item_contents["endpoint"]?.try &.["browseEndpoint"] || nil
-    browse_endpoint_data = ""
-    category_type = 0 # 0: Video, 1: Channels, 2: Playlist/feed, 3: trending
-
-    # There's no endpoint data for video and trending category
-    if !item_contents["endpoint"]?
-      if !item_contents["videoId"]?
-        category_type = 3
-      end
-    end
-
-    if !browse_endpoint.nil?
-      # Playlist/feed categories doesn't need the params value (nor is it even included in yt response)
-      # instead it uses the browseId parameter. So if there isn't a params value we can assume the
-      # category is a playlist/feed
-      if browse_endpoint["params"]?
-        # However, even though the channel category type returns the browse endpoint param
-        # we're not going to be using it in order to preserve compatablity with Youtube.
-        # and for an URL that looks cleaner
-        url = item_contents["endpoint"]["commandMetadata"]["webCommandMetadata"]["url"]
-        url = URI.parse(url.as_s)
-        auxiliary_data["view"] = url.query_params["view"]
-        auxiliary_data["shelf_id"] = url.query_params["shelf_id"]
-
-        category_type = 1
-      else
-        browse_endpoint_data = browse_endpoint["browseId"].as_s
-        category_type = 2
-      end
-    end
 
     # Sometimes a category can have badges.
     badges = [] of Tuple(String, String) # (Badge style, label)
@@ -284,9 +254,8 @@ private class CategoryParser < ItemParser
       title:                title,
       contents:             contents,
       description_html:     description_html,
-      browse_endpoint_data: browse_endpoint_data,
+      url:                  url,
       badges:               badges,
-      auxiliary_data:       auxiliary_data,
     })
   end
 end
@@ -325,6 +294,9 @@ private class YoutubeTabsExtractor < ItemsContainerExtractor
         raw_items << renderer_container_contents
         next
       elsif items_container = renderer_container_contents["gridRenderer"]?
+      elsif items_container = renderer_container_contents["channelVideoPlayerRenderer"]?
+        # Parsing for channel trailer is already taken elsewhere
+        next
       else
         items_container = renderer_container_contents
       end

src/invidious/helpers/invidiousitems.cr

@@ -232,14 +232,11 @@ class Category
   include DB::Serializable
 
   property title : String
-  property contents : Array(SearchItem)
+  property contents : Array(SearchItem) | Array(Video)
-  property browse_endpoint_data : String?
+  property url : String?
   property description_html : String
   property badges : Array(Tuple(String, String))?
 
-  # Data unique to only specific types of categories.
-  property auxiliary_data : Hash(String, String)
-
   def to_json(locale, json : JSON::Builder)
     json.object do
       json.field "title", self.title

src/invidious/routes/channels.cr

@@ -1,6 +1,18 @@
 class Invidious::Routes::Channels < Invidious::Routes::BaseRoute
   def home(env)
-    self.videos(env)
+    data = self.fetch_basic_information(env)
+    if !data.is_a?(Tuple)
+      return data
+    end
+    locale, user, subscriptions, continuation, ucid, channel = data
+    items = fetch_channel_home(ucid, channel)
+
+    has_trailer = false
+    if items[0].is_a? Video
+      has_trailer = true
+    end
+
+    templated "channel/home"
   end
 
   def videos(env)
@@ -149,6 +161,34 @@ class Invidious::Routes::Channels < Invidious::Routes::BaseRoute
     templated "channel/about", buffer_footer: true
   end
 
+  def brand_redirect(env)
+    locale = LOCALES[env.get("preferences").as(Preferences).locale]?
+
+    user = env.params.url["user"]
+
+    response = YT_POOL.client &.get("/c/#{user}")
+    html = XML.parse_html(response.body)
+
+    ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1]
+    if !ucid
+      env.response.status_code = 404 
+      return
+    end
+
+    url = "/channel/#{ucid}"
+
+    location = env.request.path.lchop?("/c/#{user}/")
+    if location
+      url += "/#{location}"
+    end
+
+    if env.params.query.size > 0
+      url += "?#{env.params.query}"
+    end
+
+    env.redirect url
+  end
+
   private def fetch_basic_information(env)
     locale = LOCALES[env.get("preferences").as(Preferences).locale]?
 

src/invidious/videos.cr

@@ -275,7 +275,7 @@ struct Video
     end
   end
 
-  def to_json(locale, json : JSON::Builder)
+  def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder)
     json.object do
       json.field "type", "video"
 

src/invidious/views/channel/channel.ecr

@@ -4,7 +4,7 @@
 <link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/channel/<%= channel.ucid %>" >
 <% end %>
 
-<% content_type =  0 %>
+<% content_type =  1 %>
 <%= rendered "components/channel-information" %>
 
 <div class="pure-g">

src/invidious/views/channel/community.ecr

@@ -3,7 +3,7 @@
 <link rel="stylesheet" href="/css/channel.css?v=<%= ASSET_COMMIT %>">
 <% end %>
 
-<% content_type =  2 %>
+<% content_type =  3 %>
 <% sort_options = Tuple.new %>
 <%= rendered "components/channel-information" %>
 

src/invidious/views/channel/featured_channels.ecr

@@ -13,10 +13,9 @@
             <div class="channel-section pure-u-1">
                 <details open="">
                     <summary style="display: revert;">
-                        <h3 class="category-heading">
+                        <span style="font-weight: bold;">
-                            <% if category.auxiliary_data.has_key?("view") %>
+                            <% if category.url %>
-                                <% category_url_param = "?view=#{category.auxiliary_data["view"]}&shelf_id=#{category.auxiliary_data["shelf_id"]}" %>
+                                <a href="<%=category.url%>">
-                                <a href="/channel/<%=channel.ucid%>/channels<%=HTML.escape(category_url_param)%>">
                                     <%= category.title %>
                                 </a>
                             <%else%>

src/invidious/views/channel/home.ecr

@@ -0,0 +1,60 @@
+<% content_for "header" do %>
+    <title><%= channel.author %> - Invidious</title>
+    <link rel="stylesheet" href="/css/channel.css?v=<%= ASSET_COMMIT %>">
+<% end %>
+
+<% content_type =  0 %>
+<% sort_options = Tuple.new %>
+<%= rendered "components/channel-information" %>
+
+ <div class="pure-g">
+    <% items.each do | section | %>
+        <% # Channel trailer %>
+        <% if section.is_a? Video %>
+            <div class="pure-u-1 h-box trailer-container category">
+                <% # Placeholder solution. A mini player should be placed here
+                %>
+                <div class="player-container pure-u-1 pure-u-md-1-3">
+                    <a style="width:100%" href="/watch?v=<%= section.id %>">
+                        <div class="thumbnail">
+                            <img class="thumbnail" src="/vi/<%=section.id%>/maxres.jpg"/>
+                            <% if section.length_seconds != 0 %>
+                                <p class="length"><%= recode_length_seconds(section.length_seconds) %></p>
+                            <% end %>
+                        </div>
+                    </a>
+                </div>
+
+                <div class="trailer-metadata pure-u-1 pure-u-md-1-3">
+                    <a style="color:rgb(209, 209, 209)"><%= HTML.escape(section.title) %></a>
+                    <p style="color: gray;">
+                        <%= translate(locale, "`x` views", number_to_short_text(section.views || 0)) %>
+                        <%= translate(locale, "Shared `x` ago", recode_date(section.published, locale)) %>
+                    </p>
+                    <div class="trailer-description">
+                        <%= section.description_html %>
+                    </div>
+                    <a class="read-more" href="/watch?v=<%= section.id %>">READ MORE</a>
+                </div>
+            </div>
+        <% else %>
+            <div class="category pure-u-1">
+                <details open = ""> 
+                    <summary style="display: revert;">
+                        <a class="category-heading" href="<%=section.url%>"> <%= section.title %> </a>
+                    </summary>
+
+                    <div class="category-description h-box">
+                        <p> <%= section.description_html %></p>
+                    </div>
+
+                    <div class="pure-g">
+                        <% section.contents.each do |item| %>
+                            <%= rendered "components/item" %>
+                        <% end %>
+                    </div>
+                </details>
+            </div>
+        <% end %>
+    <% end %>
+ </div>

src/invidious/views/channel/playlists.ecr

@@ -3,7 +3,7 @@
 <link rel="stylesheet" href="/css/channel.css?v=<%= ASSET_COMMIT %>">
 <% end %>
 
-<% content_type =  1 %>
+<% content_type =  2 %>
 <%= rendered "components/channel-information" %>
 
 <div class="pure-g">

src/invidious/views/components/channel-information.ecr

@@ -60,23 +60,37 @@
         <!-- TODO Refactor this! -->
         <div class="pure-menu pure-menu-horizontal">
           <ui class="pure-menu-list">
+              <% if content_type == 0 %>
+                  <li class="pure-menu-item pure-menu-selected">
+                      <a class="pure-menu-link" href="/channel/<%= channel.ucid %>">
+                        <b> <%= translate(locale, "Home") %> </b>
+                      </a>
+                  </li>
+              <% else %>
+                  <li class="pure-menu-item">
+                      <a class="pure-menu-link" href="/channel/<%= channel.ucid %>">
+                        <%= translate(locale, "Home") %>
+                      </a>
+                  </li>
+              <% end %>
+          
               <% if !channel.auto_generated %>
-                  <% if content_type == 0 %>
+                  <% if content_type == 1 %>
                       <li class="pure-menu-item pure-menu-selected">
-                        <a href="/channel/<%= channel.ucid %>" class="pure-menu-link">
+                        <a href="/channel/<%= channel.ucid %>/videos" class="pure-menu-link">
                           <b><%= translate(locale, "Videos") %></b>
                         </a>
                       </li>
                   <% else %>
                       <li class="pure-menu-item pure-menu">
-                        <a href="/channel/<%= channel.ucid %>" class="pure-menu-link">
+                        <a href="/channel/<%= channel.ucid %>/videos" class="pure-menu-link">
                           <%= translate(locale, "Videos") %>
                         </a>
                       </li>
                   <% end %>
               <% end %>
 
-              <% if content_type == 1 || channel.auto_generated %>
+              <% if content_type == 2 %>
                   <li class="pure-menu-item pure-menu-selected">
                       <a class="pure-menu-link" href="/channel/<%= channel.ucid %>/playlists">
                         <b> <%= translate(locale, "Playlists") %> </b>
@@ -91,7 +105,7 @@
               <% end %>
 
               <% if channel.tabs.has_key?("community") %>
-                  <% if content_type == 2 %>
+                  <% if content_type == 3 %>
                       <li class="pure-menu-item pure-menu-selected">
                           <a class="pure-menu-link" href="/channel/<%= channel.ucid %>/community">
                             <b> <%= translate(locale, "Community") %> </b>
@@ -152,7 +166,7 @@
     </div>
     <div class="pure-u-1-3"></div>
 
-    <% if content_type == 0 || content_type == 1 %>
+    <% if content_type == 1 || content_type == 2 %>
         <% route = content_type == 1 ? "/playlists" : "" %>
         <% url = "/channel/#{channel.ucid + route}" %>