From c27bb90e4d8286665658e2805a26ad8881472618 Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Tue, 9 Apr 2024 16:26:16 -0400 Subject: [PATCH] Add support for new comment format --- src/invidious/comments/youtube.cr | 199 +++++++++++++++--------- src/invidious/routes/api/v1/channels.cr | 2 +- src/invidious/routes/channels.cr | 2 +- 3 files changed, 130 insertions(+), 73 deletions(-) diff --git a/src/invidious/comments/youtube.cr b/src/invidious/comments/youtube.cr index 185d8e436..375672d79 100644 --- a/src/invidious/comments/youtube.cr +++ b/src/invidious/comments/youtube.cr @@ -57,7 +57,7 @@ module Invidious::Comments return initial_data end - def parse_youtube(id, response, format, locale, thin_mode, sort_by = "top", isPost = false) + def parse_youtube(id, response, format, locale, thin_mode, sort_by = "top", is_post = false) contents = nil if on_response_received_endpoints = response["onResponseReceivedEndpoints"]? @@ -113,7 +113,7 @@ module Invidious::Comments json.field "commentCount", comment_count end - if isPost + if is_post json.field "postId", id else json.field "videoId", id @@ -131,89 +131,147 @@ module Invidious::Comments node_replies = node["replies"]["commentRepliesRenderer"] end - if node["comment"]? - node_comment = node["comment"]["commentRenderer"] + if node["commentViewModel"]? + cvm = node.dig("commentViewModel", "commentViewModel") + comment_key = cvm["commentKey"] + toolbar_key = cvm["toolbarStateKey"] + if mutations = response.dig?("frameworkUpdates", "entityBatchUpdate", "mutations") + comment_mutation = mutations.as_a.find { |i| i.dig?("payload", "commentEntityPayload", "key") == comment_key} + toolbar_mutation = mutations.as_a.find { |i| i.dig?("entityKey") == toolbar_key} + if !comment_mutation.nil? && !toolbar_mutation.nil? + html_content = comment_mutation.dig("payload", "commentEntityPayload", "properties", "content", "content").as_s + if comment_author = comment_mutation.dig?("payload", "commentEntityPayload", "author") + json.field "authorId", comment_author["channelId"].as_s + json.field "authorUrl", "/channel/#{comment_author["channelId"].as_s}" + json.field "author", comment_author["displayName"].as_s + json.field "verified", comment_author["isVerified"].as_bool + json.field "authorThumbnails" do + json.array do + comment_mutation.dig?("payload", "commentEntityPayload", "avatar", "image", "sources").try &.as_a.each do |thumbnail| + json.object do + json.field "url", thumbnail["url"] + json.field "width", thumbnail["width"] + json.field "height", thumbnail["height"] + end + end + end + end + json.field "authorIsChannelOwner", comment_author["isCreator"].as_bool + json.field "isSponsor", (comment_author["sponsorBadgeUrl"]?!= nil) + if comment_author["sponsorBadgeUrl"]? + # Sponsor icon thumbnails always have one object and there's only ever the url property in it + json.field "sponsorIconUrl", comment_author["sponsorBadgeUrl"].to_s + end + end + + if comment_toolbar = comment_mutation.dig?("payload", "commentEntityPayload", "toolbar") + json.field "likeCount", short_text_to_number(comment_toolbar["likeCountNotliked"].as_s) + json.field "replyCount", short_text_to_number(comment_toolbar["replyCount"]?.try &.as_s || "0") + if heart_state = toolbar_mutation.dig?("payload", "engagementToolbarStateEntityPayload", "heartState") + if heart_state.as_s == "TOOLBAR_HEART_STATE_HEARTED" + json.field "creatorHeart" do + json.object do + json.field "creatorThumbnail", comment_toolbar["creatorThumbnailUrl"].as_s + json.field "creatorName", comment_toolbar["heartActiveTooltip"].as_s.sub("❤ by ", "") + end + end + end + end + published_text = comment_mutation.dig?("payload", "commentEntityPayload", "properties", "publishedTime").try &.as_s + end + end + end + json.field "isPinned", (cvm.dig?("pinnedText") != nil) + json.field "isSponsored", false + json.field "commentId", cvm["commentId"] else - node_comment = node["commentRenderer"] - end + if node["comment"]? + node_comment = node["comment"]["commentRenderer"] + else + node_comment = node["commentRenderer"] + end + json.field "commentId", node_comment["commentId"] + html_content = node_comment["contentText"]?.try { |t| parse_content(t, id) } - content_html = node_comment["contentText"]?.try { |t| parse_content(t, id) } || "" - author = node_comment["authorText"]?.try &.["simpleText"]? || "" + json.field "verified", (node_comment["authorCommentBadge"]? != nil) - json.field "verified", (node_comment["authorCommentBadge"]? != nil) + json.field "author", node_comment["authorText"]?.try &.["simpleText"]? || "" + json.field "authorThumbnails" do + json.array do + node_comment["authorThumbnail"]["thumbnails"].as_a.each do |thumbnail| + json.object do + json.field "url", thumbnail["url"] + json.field "width", thumbnail["width"] + json.field "height", thumbnail["height"] + end + end + end + end - json.field "author", author - json.field "authorThumbnails" do - json.array do - node_comment["authorThumbnail"]["thumbnails"].as_a.each do |thumbnail| + if comment_action_buttons_renderer = node_comment.dig?("actionButtons", "commentActionButtonsRenderer") + json.field "likeCount", comment_action_buttons_renderer["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"].as_s.scan(/\d/).map(&.[0]).join.to_i + if comment_action_buttons_renderer["creatorHeart"]? + heart_data = comment_action_buttons_renderer["creatorHeart"]["creatorHeartRenderer"]["creatorThumbnail"] + json.field "creatorHeart" do + json.object do + json.field "creatorThumbnail", heart_data["thumbnails"][-1]["url"] + json.field "creatorName", heart_data["accessibility"]["accessibilityData"]["label"] + end + end + end + end + + if node_comment["authorEndpoint"]? + json.field "authorId", node_comment["authorEndpoint"]["browseEndpoint"]["browseId"] + json.field "authorUrl", node_comment["authorEndpoint"]["browseEndpoint"]["canonicalBaseUrl"] + else + json.field "authorId", "" + json.field "authorUrl", "" + end + + json.field "authorIsChannelOwner", node_comment["authorIsChannelOwner"] + json.field "isPinned", (node_comment["pinnedCommentBadge"]? != nil) + published_text = node_comment["publishedTimeText"]["runs"][0]["text"].as_s + + json.field "isSponsor", (node_comment["sponsorCommentBadge"]? != nil) + if node_comment["sponsorCommentBadge"]? + # Sponsor icon thumbnails always have one object and there's only ever the url property in it + json.field "sponsorIconUrl", node_comment.dig("sponsorCommentBadge", "sponsorCommentBadgeRenderer", "customBadge", "thumbnails", 0, "url").to_s + end + + if node_replies && !response["commentRepliesContinuation"]? + if node_replies["continuations"]? + continuation = node_replies["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s + elsif node_replies["contents"]? + continuation = node_replies["contents"]?.try &.as_a[0]["continuationItemRenderer"]["continuationEndpoint"]["continuationCommand"]["token"].as_s + end + continuation ||= "" + + json.field "replies" do json.object do - json.field "url", thumbnail["url"] - json.field "width", thumbnail["width"] - json.field "height", thumbnail["height"] + json.field "replyCount", node_comment["replyCount"]? || 1 + json.field "continuation", continuation end end end end - if node_comment["authorEndpoint"]? - json.field "authorId", node_comment["authorEndpoint"]["browseEndpoint"]["browseId"] - json.field "authorUrl", node_comment["authorEndpoint"]["browseEndpoint"]["canonicalBaseUrl"] - else - json.field "authorId", "" - json.field "authorUrl", "" - end - - published_text = node_comment["publishedTimeText"]["runs"][0]["text"].as_s - published = decode_date(published_text.rchop(" (edited)")) - - if published_text.includes?(" (edited)") - json.field "isEdited", true - else - json.field "isEdited", false - end - + content_html = html_content || "" json.field "content", html_to_content(content_html) json.field "contentHtml", content_html - json.field "isPinned", (node_comment["pinnedCommentBadge"]? != nil) - json.field "isSponsor", (node_comment["sponsorCommentBadge"]? != nil) - if node_comment["sponsorCommentBadge"]? - # Sponsor icon thumbnails always have one object and there's only ever the url property in it - json.field "sponsorIconUrl", node_comment.dig("sponsorCommentBadge", "sponsorCommentBadgeRenderer", "customBadge", "thumbnails", 0, "url").to_s - end - json.field "published", published.to_unix - json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale)) - - comment_action_buttons_renderer = node_comment["actionButtons"]["commentActionButtonsRenderer"] - - json.field "likeCount", comment_action_buttons_renderer["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"].as_s.scan(/\d/).map(&.[0]).join.to_i - json.field "commentId", node_comment["commentId"] - json.field "authorIsChannelOwner", node_comment["authorIsChannelOwner"] - - if comment_action_buttons_renderer["creatorHeart"]? - hearth_data = comment_action_buttons_renderer["creatorHeart"]["creatorHeartRenderer"]["creatorThumbnail"] - json.field "creatorHeart" do - json.object do - json.field "creatorThumbnail", hearth_data["thumbnails"][-1]["url"] - json.field "creatorName", hearth_data["accessibility"]["accessibilityData"]["label"] - end + if published_text != nil + published_text = published_text.to_s + if published_text.includes?(" (edited)") + json.field "isEdited", true + published = decode_date(published_text.rchop(" (edited)")) + else + json.field "isEdited", false + published = decode_date(published_text) end - end - if node_replies && !response["commentRepliesContinuation"]? - if node_replies["continuations"]? - continuation = node_replies["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s - elsif node_replies["contents"]? - continuation = node_replies["contents"]?.try &.as_a[0]["continuationItemRenderer"]["continuationEndpoint"]["continuationCommand"]["token"].as_s - end - continuation ||= "" - - json.field "replies" do - json.object do - json.field "replyCount", node_comment["replyCount"]? || 1 - json.field "continuation", continuation - end - end + json.field "published", published.to_unix + json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale)) end end end @@ -236,7 +294,6 @@ module Invidious::Comments if format == "html" response = JSON.parse(response) content_html = Frontend::Comments.template_youtube(response, locale, thin_mode) - response = JSON.build do |json| json.object do json.field "contentHtml", content_html diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 67018660b..c6be8b069 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -393,7 +393,7 @@ module Invidious::Routes::API::V1::Channels else comments = YoutubeAPI.browse(continuation: continuation) end - return Comments.parse_youtube(id, comments, format, locale, thin_mode, isPost: true) + return Comments.parse_youtube(id, comments, format, locale, thin_mode, is_post: true) end def self.channels(env) diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index d4d8b1c12..fea49bbe4 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -231,7 +231,7 @@ module Invidious::Routes::Channels if nojs comments = Comments.fetch_community_post_comments(ucid, id) - comment_html = JSON.parse(Comments.parse_youtube(id, comments, "html", locale, thin_mode, isPost: true))["contentHtml"] + comment_html = JSON.parse(Comments.parse_youtube(id, comments, "html", locale, thin_mode, is_post: true))["contentHtml"] end templated "post" end