mirror of
https://github.com/yattee/yattee.git
synced 2026-02-19 09:19:46 +00:00
Fix incomplete playlist loading by paginating through all pages
Playlists only loaded the first page of videos. Add full pagination for both Invidious and Piped playlist endpoints (public and authenticated). Deduplicate Invidious results by playlist index to handle its overlapping page windows. Also fix URL encoding in Invidious login to use strict form-encoding charset.
This commit is contained in:
@@ -175,9 +175,41 @@ actor InvidiousAPI: InstanceAPI {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func playlist(id: String, instance: Instance) async throws -> Playlist {
|
func playlist(id: String, instance: Instance) async throws -> Playlist {
|
||||||
let endpoint = GenericEndpoint.get("/api/v1/playlists/\(id)")
|
let firstEndpoint = GenericEndpoint.get("/api/v1/playlists/\(id)")
|
||||||
|
let firstResponse: InvidiousPlaylist = try await httpClient.fetch(firstEndpoint, baseURL: instance.url)
|
||||||
|
var allVideos = firstResponse.videos ?? []
|
||||||
|
let maxPages = 50
|
||||||
|
|
||||||
|
if firstResponse.videoCount > 0 {
|
||||||
|
var page = 2
|
||||||
|
while page <= maxPages {
|
||||||
|
let endpoint = GenericEndpoint.get("/api/v1/playlists/\(id)", query: ["page": String(page)])
|
||||||
let response: InvidiousPlaylist = try await httpClient.fetch(endpoint, baseURL: instance.url)
|
let response: InvidiousPlaylist = try await httpClient.fetch(endpoint, baseURL: instance.url)
|
||||||
return response.toPlaylist(baseURL: instance.url)
|
let pageVideos = response.videos ?? []
|
||||||
|
if pageVideos.isEmpty { break }
|
||||||
|
allVideos.append(contentsOf: pageVideos)
|
||||||
|
page += 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Invidious pagination uses overlapping pages — deduplicate by playlist index
|
||||||
|
var seenIndices = Set<Int>()
|
||||||
|
allVideos = allVideos.filter { item in
|
||||||
|
if case .video(let video) = item, let index = video.index {
|
||||||
|
return seenIndices.insert(index).inserted
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
return InvidiousPlaylist(
|
||||||
|
playlistId: firstResponse.playlistId,
|
||||||
|
title: firstResponse.title,
|
||||||
|
description: firstResponse.description,
|
||||||
|
author: firstResponse.author,
|
||||||
|
authorId: firstResponse.authorId,
|
||||||
|
videoCount: firstResponse.videoCount,
|
||||||
|
videos: allVideos
|
||||||
|
).toPlaylist(baseURL: instance.url)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fetches a user's playlist using authenticated endpoint.
|
/// Fetches a user's playlist using authenticated endpoint.
|
||||||
@@ -188,13 +220,50 @@ actor InvidiousAPI: InstanceAPI {
|
|||||||
/// - sid: The session ID from login
|
/// - sid: The session ID from login
|
||||||
/// - Returns: The playlist with videos
|
/// - Returns: The playlist with videos
|
||||||
func userPlaylist(id: String, instance: Instance, sid: String) async throws -> Playlist {
|
func userPlaylist(id: String, instance: Instance, sid: String) async throws -> Playlist {
|
||||||
let endpoint = GenericEndpoint.get("/api/v1/auth/playlists/\(id)")
|
let headers = ["Cookie": "SID=\(sid)"]
|
||||||
let response: InvidiousPlaylist = try await httpClient.fetch(
|
let firstEndpoint = GenericEndpoint(
|
||||||
endpoint,
|
path: "/api/v1/auth/playlists/\(id)",
|
||||||
baseURL: instance.url,
|
queryItems: nil,
|
||||||
customHeaders: ["Cookie": "SID=\(sid)"]
|
headers: headers
|
||||||
)
|
)
|
||||||
return response.toPlaylist(baseURL: instance.url)
|
let firstResponse: InvidiousPlaylist = try await httpClient.fetch(firstEndpoint, baseURL: instance.url)
|
||||||
|
var allVideos = firstResponse.videos ?? []
|
||||||
|
let maxPages = 50
|
||||||
|
|
||||||
|
if firstResponse.videoCount > 0 {
|
||||||
|
var page = 2
|
||||||
|
while page <= maxPages {
|
||||||
|
let endpoint = GenericEndpoint(
|
||||||
|
path: "/api/v1/auth/playlists/\(id)",
|
||||||
|
queryItems: [URLQueryItem(name: "page", value: String(page))],
|
||||||
|
headers: headers
|
||||||
|
)
|
||||||
|
let response: InvidiousPlaylist = try await httpClient.fetch(endpoint, baseURL: instance.url)
|
||||||
|
let pageVideos = response.videos ?? []
|
||||||
|
if pageVideos.isEmpty { break }
|
||||||
|
allVideos.append(contentsOf: pageVideos)
|
||||||
|
page += 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Invidious pagination uses overlapping pages — deduplicate by playlist index
|
||||||
|
var seenIndices = Set<Int>()
|
||||||
|
allVideos = allVideos.filter { item in
|
||||||
|
if case .video(let video) = item, let index = video.index {
|
||||||
|
return seenIndices.insert(index).inserted
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
return InvidiousPlaylist(
|
||||||
|
playlistId: firstResponse.playlistId,
|
||||||
|
title: firstResponse.title,
|
||||||
|
description: firstResponse.description,
|
||||||
|
author: firstResponse.author,
|
||||||
|
authorId: firstResponse.authorId,
|
||||||
|
videoCount: firstResponse.videoCount,
|
||||||
|
videos: allVideos
|
||||||
|
).toPlaylist(baseURL: instance.url)
|
||||||
}
|
}
|
||||||
|
|
||||||
func comments(videoID: String, instance: Instance, continuation: String?) async throws -> CommentsPage {
|
func comments(videoID: String, instance: Instance, continuation: String?) async throws -> CommentsPage {
|
||||||
@@ -286,8 +355,9 @@ actor InvidiousAPI: InstanceAPI {
|
|||||||
"password": password,
|
"password": password,
|
||||||
"action": "signin"
|
"action": "signin"
|
||||||
]
|
]
|
||||||
|
let formAllowed = CharacterSet.alphanumerics.union(CharacterSet(charactersIn: "-._~"))
|
||||||
let bodyString = bodyComponents
|
let bodyString = bodyComponents
|
||||||
.map { "\($0.key)=\($0.value.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) ?? $0.value)" }
|
.map { "\($0.key)=\($0.value.addingPercentEncoding(withAllowedCharacters: formAllowed) ?? $0.value)" }
|
||||||
.joined(separator: "&")
|
.joined(separator: "&")
|
||||||
|
|
||||||
guard let bodyData = bodyString.data(using: .utf8) else {
|
guard let bodyData = bodyString.data(using: .utf8) else {
|
||||||
@@ -627,6 +697,7 @@ private struct InvidiousAuthFeedResponse: Decodable, Sendable {
|
|||||||
|
|
||||||
private struct InvidiousVideo: Decodable, Sendable {
|
private struct InvidiousVideo: Decodable, Sendable {
|
||||||
let videoId: String
|
let videoId: String
|
||||||
|
let index: Int?
|
||||||
let title: String
|
let title: String
|
||||||
let description: String?
|
let description: String?
|
||||||
let author: String
|
let author: String
|
||||||
|
|||||||
@@ -121,8 +121,33 @@ actor PipedAPI: InstanceAPI {
|
|||||||
|
|
||||||
func playlist(id: String, instance: Instance) async throws -> Playlist {
|
func playlist(id: String, instance: Instance) async throws -> Playlist {
|
||||||
let endpoint = GenericEndpoint.get("/playlists/\(id)")
|
let endpoint = GenericEndpoint.get("/playlists/\(id)")
|
||||||
let response: PipedPlaylistResponse = try await httpClient.fetch(endpoint, baseURL: instance.url)
|
let firstResponse: PipedPlaylistResponse = try await httpClient.fetch(endpoint, baseURL: instance.url)
|
||||||
return response.toPlaylist(instanceURL: instance.url)
|
var allStreams = firstResponse.relatedStreams ?? []
|
||||||
|
let maxPages = 50
|
||||||
|
|
||||||
|
var nextpage = firstResponse.nextpage
|
||||||
|
var page = 1
|
||||||
|
while let token = nextpage, page < maxPages {
|
||||||
|
let nextEndpoint = GenericEndpoint.get("/nextpage/playlists/\(id)", query: ["nextpage": token])
|
||||||
|
let nextResponse: PipedPlaylistNextPageResponse = try await httpClient.fetch(nextEndpoint, baseURL: instance.url)
|
||||||
|
let pageStreams = nextResponse.relatedStreams ?? []
|
||||||
|
if pageStreams.isEmpty { break }
|
||||||
|
allStreams.append(contentsOf: pageStreams)
|
||||||
|
nextpage = nextResponse.nextpage
|
||||||
|
page += 1
|
||||||
|
}
|
||||||
|
|
||||||
|
return PipedPlaylistResponse(
|
||||||
|
name: firstResponse.name,
|
||||||
|
description: firstResponse.description,
|
||||||
|
uploader: firstResponse.uploader,
|
||||||
|
uploaderUrl: firstResponse.uploaderUrl,
|
||||||
|
uploaderAvatar: firstResponse.uploaderAvatar,
|
||||||
|
videos: firstResponse.videos,
|
||||||
|
relatedStreams: allStreams,
|
||||||
|
thumbnailUrl: firstResponse.thumbnailUrl,
|
||||||
|
nextpage: nil
|
||||||
|
).toPlaylist(instanceURL: instance.url)
|
||||||
}
|
}
|
||||||
|
|
||||||
func comments(videoID: String, instance: Instance, continuation: String?) async throws -> CommentsPage {
|
func comments(videoID: String, instance: Instance, continuation: String?) async throws -> CommentsPage {
|
||||||
@@ -341,13 +366,43 @@ actor PipedAPI: InstanceAPI {
|
|||||||
/// - authToken: The auth token from login
|
/// - authToken: The auth token from login
|
||||||
/// - Returns: Playlist with videos
|
/// - Returns: Playlist with videos
|
||||||
func userPlaylist(id: String, instance: Instance, authToken: String) async throws -> Playlist {
|
func userPlaylist(id: String, instance: Instance, authToken: String) async throws -> Playlist {
|
||||||
let endpoint = GenericEndpoint(
|
let headers = ["Authorization": authToken]
|
||||||
|
let firstEndpoint = GenericEndpoint(
|
||||||
path: "/playlists/\(id)",
|
path: "/playlists/\(id)",
|
||||||
method: .get,
|
method: .get,
|
||||||
headers: ["Authorization": authToken]
|
headers: headers
|
||||||
)
|
)
|
||||||
let response: PipedPlaylistResponse = try await httpClient.fetch(endpoint, baseURL: instance.url)
|
let firstResponse: PipedPlaylistResponse = try await httpClient.fetch(firstEndpoint, baseURL: instance.url)
|
||||||
return response.toPlaylist(instanceURL: instance.url, playlistID: id)
|
var allStreams = firstResponse.relatedStreams ?? []
|
||||||
|
let maxPages = 50
|
||||||
|
|
||||||
|
var nextpage = firstResponse.nextpage
|
||||||
|
var page = 1
|
||||||
|
while let token = nextpage, page < maxPages {
|
||||||
|
let nextEndpoint = GenericEndpoint(
|
||||||
|
path: "/nextpage/playlists/\(id)",
|
||||||
|
queryItems: [URLQueryItem(name: "nextpage", value: token)],
|
||||||
|
headers: headers
|
||||||
|
)
|
||||||
|
let nextResponse: PipedPlaylistNextPageResponse = try await httpClient.fetch(nextEndpoint, baseURL: instance.url)
|
||||||
|
let pageStreams = nextResponse.relatedStreams ?? []
|
||||||
|
if pageStreams.isEmpty { break }
|
||||||
|
allStreams.append(contentsOf: pageStreams)
|
||||||
|
nextpage = nextResponse.nextpage
|
||||||
|
page += 1
|
||||||
|
}
|
||||||
|
|
||||||
|
return PipedPlaylistResponse(
|
||||||
|
name: firstResponse.name,
|
||||||
|
description: firstResponse.description,
|
||||||
|
uploader: firstResponse.uploader,
|
||||||
|
uploaderUrl: firstResponse.uploaderUrl,
|
||||||
|
uploaderAvatar: firstResponse.uploaderAvatar,
|
||||||
|
videos: firstResponse.videos,
|
||||||
|
relatedStreams: allStreams,
|
||||||
|
thumbnailUrl: firstResponse.thumbnailUrl,
|
||||||
|
nextpage: nil
|
||||||
|
).toPlaylist(instanceURL: instance.url, playlistID: id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -885,6 +940,7 @@ private struct PipedPlaylistResponse: Decodable, Sendable {
|
|||||||
let videos: Int?
|
let videos: Int?
|
||||||
let relatedStreams: [PipedPlaylistItem]?
|
let relatedStreams: [PipedPlaylistItem]?
|
||||||
let thumbnailUrl: String?
|
let thumbnailUrl: String?
|
||||||
|
let nextpage: String?
|
||||||
|
|
||||||
nonisolated func toPlaylist(instanceURL: URL, playlistID: String? = nil) -> Playlist {
|
nonisolated func toPlaylist(instanceURL: URL, playlistID: String? = nil) -> Playlist {
|
||||||
// Extract only valid videos, skipping malformed items
|
// Extract only valid videos, skipping malformed items
|
||||||
@@ -913,6 +969,12 @@ private struct PipedPlaylistResponse: Decodable, Sendable {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Response from `/nextpage/playlists/{id}?nextpage=...` for paginated playlist videos.
|
||||||
|
private struct PipedPlaylistNextPageResponse: Decodable, Sendable {
|
||||||
|
let relatedStreams: [PipedPlaylistItem]?
|
||||||
|
let nextpage: String?
|
||||||
|
}
|
||||||
|
|
||||||
/// User playlist from Piped `/user/playlists` endpoint.
|
/// User playlist from Piped `/user/playlists` endpoint.
|
||||||
private struct PipedUserPlaylist: Decodable, Sendable {
|
private struct PipedUserPlaylist: Decodable, Sendable {
|
||||||
let id: String
|
let id: String
|
||||||
|
|||||||
Reference in New Issue
Block a user