Handle parse errors gracefully on timeline items (#5196)

assets/css/default.css

@@ -550,6 +550,10 @@ span > select {
   color: #565d64;
 }
 
+.light-theme .error-card {
+  border: 1px solid black;
+}
+
 @media (prefers-color-scheme: light) {
   .no-theme a:hover,
   .no-theme a:active,
@@ -596,6 +600,10 @@ span > select {
   .light-theme .pure-menu-heading {
     color: #565d64;
   }
+
+  .no-theme .error-card {
+    border: 1px solid black;
+  }
 }
 
 
@@ -658,6 +666,10 @@ body.dark-theme {
   color: inherit;
 }
 
+.dark-theme .error-card {
+  border: 1px solid #5e5e5e;
+}
+
 @media (prefers-color-scheme: dark) {
   .no-theme a:hover,
   .no-theme a:active,
@@ -719,6 +731,10 @@ body.dark-theme {
   .no-theme footer a {
     color: #adadad !important;
   }
+
+  .no-theme .error-card {
+    border: 1px solid #5e5e5e;
+  }
 }
 
 
@@ -816,3 +832,57 @@ h1, h2, h3, h4, h5, p,
 #download_widget {
     width: 100%;
 }
+
+.error-card {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  padding: 25px;
+  margin-bottom: 1em;
+  border-radius: 10px;
+  box-sizing: border-box;
+  height: 100%;
+}
+
+.error-card > .explanation {
+  display: grid;
+  grid-template-columns: max-content 1fr;
+  grid-template-rows: 1fr max-content;
+  align-items: center;
+  column-gap: 10px;
+  row-gap: 4px;
+}
+
+.error-card > .explanation > i {
+  color: #f44;
+  font-size: 24px;
+  grid-area: 1 / 1 / 2 / 2;
+}
+
+.error-card > .explanation > h4 {
+  grid-area: 1 / 2 / 2 / 3;
+  margin: 0;
+}
+
+.error-card > .explanation > p {
+  grid-area: 2 / 2 / 3 / 3;
+  margin: 0;
+}
+
+.error-card details {
+  margin-top: 10px;
+  width: 100%;
+}
+
+.error-card summary {
+  width: 100%;
+}
+
+.error-card pre {
+  height: 300px;
+}
+
+.error-issue-template {
+  padding: 20px;
+  background: rgba(0, 0, 0, 0.12345);
+}

assets/css/search.css

@@ -1,4 +1,4 @@
-summary {
+#filters-collapse summary {
 	/* This should hide the marker */
 	display: block;
 
@@ -8,10 +8,10 @@ summary {
 	cursor: pointer;
 }
 
-summary::-webkit-details-marker,
+#filters-collapse summary::-webkit-details-marker,
-summary::marker { display: none; }
+#filters-collapse summary::marker { display: none; }
 
-summary:before {
+#filters-collapse summary:before {
 	border-radius: 5px;
 	content: "[ + ]";
 	margin: -2px 10px 0 10px;
@@ -20,7 +20,7 @@ summary:before {
 	width: 40px;
 }
 
-details[open] > summary:before { content: "[ − ]"; }
+#filters-collapse details[open] > summary:before { content: "[ − ]"; }
 
 
 #filters-box {

locales/en-US.json

@@ -501,5 +501,8 @@
     "toggle_theme": "Toggle Theme",
     "carousel_slide": "Slide {{current}} of {{total}}",
     "carousel_skip": "Skip the Carousel",
-    "carousel_go_to": "Go to slide `x`"
+    "carousel_go_to": "Go to slide `x`",
+    "timeline_parse_error_placeholder_heading": "Unable to parse item",
+    "timeline_parse_error_placeholder_message": "Invidious encountered an error while trying to parse this item. For more information see below:",
+    "timeline_parse_error_show_technical_details": "Show technical details"
 }

src/invidious/helpers/errors.cr

@@ -18,16 +18,7 @@ def github_details(summary : String, content : String)
   return HTML.escape(details)
 end
 
-def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception)
+def get_issue_template(env : HTTP::Server::Context, exception : Exception) : Tuple(String, String)
-  if exception.is_a?(InfoException)
-    return error_template_helper(env, status_code, exception.message || "")
-  end
-
-  locale = env.get("preferences").as(Preferences).locale
-
-  env.response.content_type = "text/html"
-  env.response.status_code = status_code
-
   issue_title = "#{exception.message} (#{exception.class})"
 
   issue_template = <<-TEXT
@@ -40,6 +31,24 @@ def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exce
 
   issue_template += github_details("Backtrace", exception.inspect_with_backtrace)
 
+  return issue_title, issue_template
+end
+
+def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception)
+  if exception.is_a?(InfoException)
+    return error_template_helper(env, status_code, exception.message || "")
+  end
+
+  locale = env.get("preferences").as(Preferences).locale
+
+  env.response.content_type = "text/html"
+  env.response.status_code = status_code
+
+  # Unpacking into issue_title, issue_template directly causes a compiler error
+  # I have no idea why.
+  issue_template_components = get_issue_template(env, exception)
+  issue_title, issue_template = issue_template_components
+
   # URLs for the error message below
   url_faq = "https://github.com/iv-org/documentation/blob/master/docs/faq.md"
   url_search_issues = "https://github.com/iv-org/invidious/issues"
@@ -69,7 +78,7 @@ def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exce
       <p>#{translate(locale, "crash_page_report_issue", url_new_issue)}</p>
 
       <!-- TODO: Add a "copy to clipboard" button -->
-      <pre style="padding: 20px; background: rgba(0, 0, 0, 0.12345);">#{issue_template}</pre>
+      <pre class="error-issue-template">#{issue_template}</pre>
     </div>
   END_HTML
 

src/invidious/helpers/serialized_yt_data.cr

@@ -291,6 +291,55 @@ struct SearchHashtag
   end
 end
 
+# A `ProblematicTimelineItem` is a `SearchItem` created by Invidious that
+# represents an item that caused an exception during parsing.
+#
+# This is not a parsed object from YouTube but rather an Invidious-only type
+# created to gracefully communicate parse errors without throwing away
+# the rest of the (hopefully) successfully parsed item on a page.
+struct ProblematicTimelineItem
+  property parse_exception : Exception
+  property id : String
+
+  def initialize(@parse_exception)
+    @id = Random.new.hex(8)
+  end
+
+  def to_json(locale : String?, json : JSON::Builder)
+    json.object do
+      json.field "type", "parse-error"
+      json.field "errorMessage", @parse_exception.message
+      json.field "errorBacktrace", @parse_exception.inspect_with_backtrace
+    end
+  end
+
+  # Provides compatibility with PlaylistVideo
+  def to_json(json : JSON::Builder, *args, **kwargs)
+    return to_json("", json)
+  end
+
+  def to_xml(env, locale, xml : XML::Builder)
+    xml.element("entry") do
+      xml.element("id") { xml.text "iv-err-#{@id}" }
+      xml.element("title") { xml.text "Parse Error: This item has failed to parse" }
+      xml.element("updated") { xml.text Time.utc.to_rfc3339 }
+
+      xml.element("content", type: "xhtml") do
+        xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
+          xml.element("div") do
+            xml.element("h4") { translate(locale, "timeline_parse_error_placeholder_heading") }
+            xml.element("p") { translate(locale, "timeline_parse_error_placeholder_message") }
+          end
+
+          xml.element("pre") do
+            get_issue_template(env, @parse_exception)
+          end
+        end
+      end
+    end
+  end
+end
+
 class Category
   include DB::Serializable
 
@@ -333,4 +382,4 @@ struct Continuation
   end
 end
 
-alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | SearchHashtag | Category
+alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | SearchHashtag | Category | ProblematicTimelineItem

src/invidious/playlists.cr

@@ -432,7 +432,7 @@ def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32,
       offset = initial_data.dig?("contents", "twoColumnWatchNextResults", "playlist", "playlist", "currentIndex").try &.as_i || offset
     end
 
-    videos = [] of PlaylistVideo
+    videos = [] of PlaylistVideo | ProblematicTimelineItem
 
     until videos.size >= 200 || videos.size == playlist.video_count || offset >= playlist.video_count
       # 100 videos per request
@@ -448,7 +448,7 @@ def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32,
 end
 
 def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
-  videos = [] of PlaylistVideo
+  videos = [] of PlaylistVideo | ProblematicTimelineItem
 
   if initial_data["contents"]?
     tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]
@@ -500,6 +500,8 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
         index:          index,
       })
     end
+  rescue ex
+    videos << ProblematicTimelineItem.new(parse_exception: ex)
   end
 
   return videos

src/invidious/routes/embed.cr

@@ -12,13 +12,15 @@ module Invidious::Routes::Embed
           url = "/playlist?list=#{plid}"
           raise NotFoundException.new(translate(locale, "error_video_not_in_playlist", url))
         end
+
+        first_playlist_video = videos[0].as(PlaylistVideo)
       rescue ex : NotFoundException
         return error_template(404, ex)
       rescue ex
         return error_template(500, ex)
       end
 
-      url = "/embed/#{videos[0].id}?#{env.params.query}"
+      url = "/embed/#{first_playlist_video}?#{env.params.query}"
 
       if env.params.query.size > 0
         url += "?#{env.params.query}"
@@ -72,13 +74,15 @@ module Invidious::Routes::Embed
             url = "/playlist?list=#{plid}"
             raise NotFoundException.new(translate(locale, "error_video_not_in_playlist", url))
           end
+
+          first_playlist_video = videos[0].as(PlaylistVideo)
         rescue ex : NotFoundException
           return error_template(404, ex)
         rescue ex
           return error_template(500, ex)
         end
 
-        url = "/embed/#{videos[0].id}"
+        url = "/embed/#{first_playlist_video.id}"
       elsif video_series
         url = "/embed/#{video_series.shift}"
         env.params.query["playlist"] = video_series.join(",")

src/invidious/routes/feeds.cr

@@ -296,7 +296,13 @@ module Invidious::Routes::Feeds
               xml.element("name") { xml.text playlist.author }
             end
 
-            videos.each &.to_xml(xml)
+            videos.each do |video|
+              if video.is_a? PlaylistVideo
+                video.to_xml(xml)
+              else
+                video.to_xml(env, locale, xml)
+              end
+            end
           end
         end
       else

src/invidious/trending.cr

@@ -31,12 +31,12 @@ def fetch_trending(trending_type, region, locale)
       # See: https://github.com/iv-org/invidious/issues/2989
       next if (itm.contents.size < 24 && deduplicate)
 
-      extracted.concat extract_category(itm)
+      extracted.concat itm.contents.select(SearchItem)
     else
       extracted << itm
     end
   end
 
   # Deduplicate items before returning results
-  return extracted.select(SearchVideo).uniq!(&.id), plid
+  return extracted.select(SearchVideo | ProblematicTimelineItem).uniq!(&.id), plid
 end

src/invidious/views/components/item.ecr

@@ -1,6 +1,6 @@
 <%-
   thin_mode = env.get("preferences").as(Preferences).thin_mode
-  item_watched = !item.is_a?(SearchChannel | SearchHashtag | SearchPlaylist | InvidiousPlaylist | Category) && env.get?("user").try &.as(User).watched.index(item.id) != nil
+  item_watched = !item.is_a?(SearchChannel | SearchHashtag | SearchPlaylist | InvidiousPlaylist | Category | ProblematicTimelineItem) && env.get?("user").try &.as(User).watched.index(item.id) != nil
   author_verified = item.responds_to?(:author_verified) && item.author_verified
 -%>
 
@@ -97,6 +97,18 @@
                 </div>
             </div>
         <% when Category %>
+        <% when ProblematicTimelineItem %>
+            <div class="error-card">
+                <div class="explanation">
+                    <i class="icon ion-ios-alert"></i>
+                    <h4><%=translate(locale, "timeline_parse_error_placeholder_heading")%></h4>
+                    <p><%=translate(locale, "timeline_parse_error_placeholder_message")%></p>
+                </div>
+                <details>
+                    <summary class="pure-button pure-button-secondary"><%=translate(locale, "timeline_parse_error_show_technical_details")%></summary>
+                    <pre class="error-issue-template"><%=get_issue_template(env, item.parse_exception)[1]%></pre>
+                </details>
+            </div>
         <% else %>
             <%-
               # `endpoint_params` is used for the "video-context-buttons" component

src/invidious/yt_backend/extractors.cr

@@ -35,6 +35,20 @@ record AuthorFallback, name : String, id : String
 # data is passed to the private `#parse()` method which returns a datastruct of the given
 # type. Otherwise, nil is returned.
 private module Parsers
+  module BaseParser
+    def parse(*args)
+      begin
+        return parse_internal(*args)
+      rescue ex
+        LOGGER.debug("#{{{@type.name}}}: Failed to render item.")
+        LOGGER.debug("#{{{@type.name}}}: Got exception: #{ex.message}")
+        ProblematicTimelineItem.new(
+          parse_exception: ex
+        )
+      end
+    end
+  end
+
   # Parses a InnerTube videoRenderer into a SearchVideo. Returns nil when the given object isn't a videoRenderer
   #
   # A videoRenderer renders a video to click on within the YouTube and Invidious UI. It is **not**
@@ -45,13 +59,16 @@ private module Parsers
   # `videoRenderer`s can be found almost everywhere on YouTube. In categories, search results, channels, etc.
   #
   module VideoRendererParser
-    def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+    extend self
+    include BaseParser
+
+    def process(item : JSON::Any, author_fallback : AuthorFallback)
       if item_contents = (item["videoRenderer"]? || item["gridVideoRenderer"]?)
         return self.parse(item_contents, author_fallback)
       end
     end
 
-    private def self.parse(item_contents, author_fallback)
+    private def parse_internal(item_contents, author_fallback)
       video_id = item_contents["videoId"].as_s
       title = extract_text(item_contents["title"]?) || ""
 
@@ -170,13 +187,16 @@ private module Parsers
   # `channelRenderer`s can be found almost everywhere on YouTube. In categories, search results, channels, etc.
   #
   module ChannelRendererParser
-    def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+    extend self
+    include BaseParser
+
+    def process(item : JSON::Any, author_fallback : AuthorFallback)
       if item_contents = (item["channelRenderer"]? || item["gridChannelRenderer"]?)
         return self.parse(item_contents, author_fallback)
       end
     end
 
-    private def self.parse(item_contents, author_fallback)
+    private def parse_internal(item_contents, author_fallback)
       author = extract_text(item_contents["title"]) || author_fallback.name
       author_id = item_contents["channelId"]?.try &.as_s || author_fallback.id
       author_verified = has_verified_badge?(item_contents["ownerBadges"]?)
@@ -230,13 +250,16 @@ private module Parsers
   # A `hashtagTileRenderer` is a kind of search result.
   # It can be found when searching for any hashtag (e.g "#hi" or "#shorts")
   module HashtagRendererParser
-    def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+    extend self
+    include BaseParser
+
+    def process(item : JSON::Any, author_fallback : AuthorFallback)
       if item_contents = item["hashtagTileRenderer"]?
         return self.parse(item_contents)
       end
     end
 
-    private def self.parse(item_contents)
+    private def parse_internal(item_contents)
       title = extract_text(item_contents["hashtag"]).not_nil! # E.g "#hi"
 
       # E.g "/hashtag/hi"
@@ -263,10 +286,6 @@ private module Parsers
         video_count:   short_text_to_number(video_count_txt || ""),
         channel_count: short_text_to_number(channel_count_txt || ""),
       })
-    rescue ex
-      LOGGER.debug("HashtagRendererParser: Failed to extract renderer.")
-      LOGGER.debug("HashtagRendererParser: Got exception: #{ex.message}")
-      return nil
     end
 
     def self.parser_name
@@ -284,13 +303,16 @@ private module Parsers
   # `gridPlaylistRenderer`s can be found on the playlist-tabs of channels and expanded categories.
   #
   module GridPlaylistRendererParser
-    def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+    extend self
+    include BaseParser
+
+    def process(item : JSON::Any, author_fallback : AuthorFallback)
       if item_contents = item["gridPlaylistRenderer"]?
         return self.parse(item_contents, author_fallback)
       end
     end
 
-    private def self.parse(item_contents, author_fallback)
+    private def parse_internal(item_contents, author_fallback)
       title = extract_text(item_contents["title"]) || ""
       plid = item_contents["playlistId"]?.try &.as_s || ""
 
@@ -325,13 +347,16 @@ private module Parsers
   # `playlistRenderer`s can be found almost everywhere on YouTube. In categories, search results, recommended, etc.
   #
   module PlaylistRendererParser
-    def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+    extend self
+    include BaseParser
+
+    def process(item : JSON::Any, author_fallback : AuthorFallback)
       if item_contents = item["playlistRenderer"]?
         return self.parse(item_contents, author_fallback)
       end
     end
 
-    private def self.parse(item_contents, author_fallback)
+    private def parse_internal(item_contents, author_fallback)
       title = extract_text(item_contents["title"]) || ""
       plid = item_contents["playlistId"]?.try &.as_s || ""
 
@@ -385,13 +410,16 @@ private module Parsers
   # `shelfRenderer`s can be found almost everywhere on YouTube. In categories, search results, channels, etc.
   #
   module CategoryRendererParser
-    def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+    extend self
+    include BaseParser
+
+    def process(item : JSON::Any, author_fallback : AuthorFallback)
       if item_contents = item["shelfRenderer"]?
         return self.parse(item_contents, author_fallback)
       end
     end
 
-    private def self.parse(item_contents, author_fallback)
+    private def parse_internal(item_contents, author_fallback)
       title = extract_text(item_contents["title"]?) || ""
       url = item_contents.dig?("endpoint", "commandMetadata", "webCommandMetadata", "url")
         .try &.as_s
@@ -450,13 +478,16 @@ private module Parsers
   # container.It is very similar to RichItemRendererParser
   #
   module ItemSectionRendererParser
-    def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+    extend self
+    include BaseParser
+
+    def process(item : JSON::Any, author_fallback : AuthorFallback)
       if item_contents = item.dig?("itemSectionRenderer", "contents", 0)
         return self.parse(item_contents, author_fallback)
       end
     end
 
-    private def self.parse(item_contents, author_fallback)
+    private def parse_internal(item_contents, author_fallback)
       child = VideoRendererParser.process(item_contents, author_fallback)
       child ||= PlaylistRendererParser.process(item_contents, author_fallback)
 
@@ -476,13 +507,16 @@ private module Parsers
   # itself inside a richGridRenderer container.
   #
   module RichItemRendererParser
-    def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+    extend self
+    include BaseParser
+
+    def process(item : JSON::Any, author_fallback : AuthorFallback)
       if item_contents = item.dig?("richItemRenderer", "content")
         return self.parse(item_contents, author_fallback)
       end
     end
 
-    private def self.parse(item_contents, author_fallback)
+    private def parse_internal(item_contents, author_fallback)
       child = VideoRendererParser.process(item_contents, author_fallback)
       child ||= ReelItemRendererParser.process(item_contents, author_fallback)
       child ||= PlaylistRendererParser.process(item_contents, author_fallback)
@@ -506,13 +540,16 @@ private module Parsers
   # TODO: Confirm that hypothesis
   #
   module ReelItemRendererParser
-    def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+    extend self
+    include BaseParser
+
+    def process(item : JSON::Any, author_fallback : AuthorFallback)
       if item_contents = item["reelItemRenderer"]?
         return self.parse(item_contents, author_fallback)
       end
     end
 
-    private def self.parse(item_contents, author_fallback)
+    private def parse_internal(item_contents, author_fallback)
       video_id = item_contents["videoId"].as_s
 
       reel_player_overlay = item_contents.dig(
@@ -600,13 +637,16 @@ private module Parsers
   # a richItemRenderer or a richGridRenderer.
   #
   module LockupViewModelParser
-    def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+    extend self
+    include BaseParser
+
+    def process(item : JSON::Any, author_fallback : AuthorFallback)
       if item_contents = item["lockupViewModel"]?
         return self.parse(item_contents, author_fallback)
       end
     end
 
-    private def self.parse(item_contents, author_fallback)
+    private def parse_internal(item_contents, author_fallback)
       playlist_id = item_contents["contentId"].as_s
 
       thumbnail_view_model = item_contents.dig(
@@ -675,13 +715,16 @@ private module Parsers
   # usually (always?) encapsulated in a richItemRenderer.
   #
   module ShortsLockupViewModelParser
-    def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+    extend self
+    include BaseParser
+
+    def process(item : JSON::Any, author_fallback : AuthorFallback)
       if item_contents = item["shortsLockupViewModel"]?
         return self.parse(item_contents, author_fallback)
       end
     end
 
-    private def self.parse(item_contents, author_fallback)
+    private def parse_internal(item_contents, author_fallback)
       # TODO: Maybe add support for "oardefault.jpg" thumbnails?
       # thumbnail = item_contents.dig("thumbnail", "sources", 0, "url").as_s
       # Gives: https://i.ytimg.com/vi/{video_id}/oardefault.jpg?...