Improve building AVPlayer composition

Apple TV/PlayerViewController.swift

@@ -18,7 +18,7 @@ struct PlayerViewController: UIViewControllerRepresentable {
     }
 
     fileprivate func loadStream(_ stream: Stream?, loadBest: Bool = false) {
-        if stream != state.streamToLoad {
+        if stream != state.nextStream {
             state.loadStream(stream)
             addTracksAndLoadAssets(stream!, loadBest: loadBest)
         }
@@ -34,31 +34,6 @@ struct PlayerViewController: UIViewControllerRepresentable {
         }
     }
 
-    fileprivate func addTrack(_ asset: AVURLAsset, stream: Stream, type: AVMediaType? = nil) {
-        let types: [AVMediaType] = stream.type == .adaptive ? [type!] : [.video, .audio]
-
-        types.forEach { type in
-            guard let assetTrack = asset.tracks(withMediaType: type).first else {
-                return
-            }
-
-            if let track = state.composition.tracks(withMediaType: type).first {
-                logger.info("removing \(type) track")
-                state.composition.removeTrack(track)
-            }
-
-            let track = state.composition.addMutableTrack(withMediaType: type, preferredTrackID: kCMPersistentTrackID_Invalid)!
-
-            try! track.insertTimeRange(
-                CMTimeRange(start: .zero, duration: CMTime(seconds: video.length, preferredTimescale: 1)),
-                of: assetTrack,
-                at: .zero
-            )
-
-            logger.info("inserted \(type) track")
-        }
-    }
-
     fileprivate func handleAssetLoad(_ stream: Stream, type: AVMediaType, loadBest: Bool = false) {
         logger.info("handling asset load: \(stream.type), \(stream.description)")
 
@@ -84,6 +59,12 @@ struct PlayerViewController: UIViewControllerRepresentable {
         }
     }
 
+    fileprivate func addTrack(_ asset: AVURLAsset, stream: Stream, type: AVMediaType? = nil) {
+        let types: [AVMediaType] = stream.type == .adaptive ? [type!] : [.video, .audio]
+
+        types.forEach { state.addTrackToNextComposition(asset, type: $0) }
+    }
+
     fileprivate func loadBestStream() {
         guard state.currentStream != video.bestStream else {
             return
@@ -108,7 +89,7 @@ struct PlayerViewController: UIViewControllerRepresentable {
     func updateUIViewController(_ controller: StreamAVPlayerViewController, context _: Context) {
         var items: [UIMenuElement] = []
 
-        if state.streamToLoad != nil {
+        if state.nextStream != nil {
             items.append(actionsMenu)
         }
 
@@ -116,15 +97,15 @@ struct PlayerViewController: UIViewControllerRepresentable {
 
         #if os(tvOS)
             controller.transportBarCustomMenuItems = items
-        #endif
 
-        if let skip = skipSegmentAction {
+            if let skip = skipSegmentAction {
-            if controller.contextualActions.isEmpty {
+                if controller.contextualActions.isEmpty {
-                controller.contextualActions = [skip]
+                    controller.contextualActions = [skip]
+                }
+            } else {
+                controller.contextualActions = []
             }
-        } else {
+        #endif
-            controller.contextualActions = []
-        }
     }
 
     fileprivate var streamingQualityMenu: UIMenu {
@@ -150,10 +131,10 @@ struct PlayerViewController: UIViewControllerRepresentable {
     }
 
     fileprivate var cancelLoadingAction: UIAction {
-        UIAction(title: "Cancel loading \(state.streamToLoad.description) stream") { _ in
+        UIAction(title: "Cancel loading \(state.nextStream.description) stream") { _ in
             DispatchQueue.main.async {
-                state.streamToLoad.cancelLoadingAssets()
+                state.nextStream.cancelLoadingAssets()
-                state.cancelLoadingStream(state.streamToLoad)
+                state.cancelLoadingStream(state.nextStream)
             }
         }
     }

Model/PlayerState.swift

@@ -1,6 +1,7 @@
 import AVFoundation
 import Foundation
 import Logging
+import UIKit
 
 final class PlayerState: ObservableObject {
     let logger = Logger(label: "net.arekf.Pearvidious.ps")
@@ -8,24 +9,40 @@ final class PlayerState: ObservableObject {
     var video: Video
 
     @Published private(set) var player: AVPlayer! = AVPlayer()
-    private(set) var composition = AVMutableComposition()
+    @Published private(set) var composition = AVMutableComposition()
+    @Published private(set) var nextComposition = AVMutableComposition()
+
+    private var comp: AVMutableComposition?
+
     @Published private(set) var currentStream: Stream!
 
-    @Published private(set) var streamToLoad: Stream!
+    @Published private(set) var nextStream: Stream!
     @Published private(set) var streamLoading = false
 
     @Published private(set) var currentTime: CMTime?
     @Published private(set) var savedTime: CMTime?
+
     @Published var currentSegment: Segment?
 
     var playerItem: AVPlayerItem {
         let playerItem = AVPlayerItem(asset: composition)
 
-        playerItem.externalMetadata = [
+        var externalMetadata = [
             makeMetadataItem(.commonIdentifierTitle, value: video.title),
             makeMetadataItem(.quickTimeMetadataGenre, value: video.genre),
-            makeMetadataItem(.commonIdentifierDescription, value: video.description)
+            makeMetadataItem(.commonIdentifierDescription, value: video.description),
         ]
+
+        if let thumbnailData = try? Data(contentsOf: video.thumbnailURL!),
+           let image = UIImage(data: thumbnailData),
+           let pngData = image.pngData()
+        {
+            let artworkItem = makeMetadataItem(.commonIdentifierArtwork, value: pngData)
+            externalMetadata.append(artworkItem)
+        }
+
+        playerItem.externalMetadata = externalMetadata
+
         playerItem.preferredForwardBufferDuration = 10
 
         return playerItem
@@ -46,17 +63,18 @@ final class PlayerState: ObservableObject {
     }
 
     func loadStream(_ stream: Stream?) {
-        guard streamToLoad != stream else {
+        guard nextStream != stream else {
             return
         }
 
-        streamToLoad?.cancelLoadingAssets()
+        nextStream?.cancelLoadingAssets()
+        removeTracksFromNextComposition()
 
         DispatchQueue.main.async {
             self.streamLoading = true
-            self.streamToLoad = stream
+            self.nextStream = stream
         }
-        logger.info("replace streamToLoad: \(streamToLoad?.description ?? "nil"), streamLoading \(streamLoading)")
+        logger.info("replace streamToLoad: \(nextStream?.description ?? "nil"), streamLoading \(streamLoading)")
     }
 
     func streamDidLoad(_ stream: Stream?) {
@@ -64,24 +82,24 @@ final class PlayerState: ObservableObject {
 
         currentStream?.cancelLoadingAssets()
         currentStream = stream
-        streamLoading = streamToLoad != stream
+        streamLoading = nextStream != stream
 
-        if streamToLoad == stream {
+        if nextStream == stream {
-            streamToLoad = nil
+            nextStream = nil
         }
 
         addTimeObserver()
     }
 
     func cancelLoadingStream(_ stream: Stream) {
-        guard streamToLoad == stream else {
+        guard nextStream == stream else {
             return
         }
 
-        streamToLoad = nil
+        nextStream = nil
         streamLoading = false
 
-        logger.info("cancel streamToLoad: \(streamToLoad?.description ?? "nil"), streamLoading \(streamLoading)")
+        logger.info("cancel streamToLoad: \(nextStream?.description ?? "nil"), streamLoading \(streamLoading)")
     }
 
     func playStream(_ stream: Stream) {
@@ -92,6 +110,7 @@ final class PlayerState: ObservableObject {
         logger.warning("loading \(stream.description) to player")
 
         saveTime()
+        replaceCompositionTracks()
 
         player.replaceCurrentItem(with: playerItem)
         streamDidLoad(stream)
@@ -99,6 +118,47 @@ final class PlayerState: ObservableObject {
         seekToSavedTime()
     }
 
+    func addTrackToNextComposition(_ asset: AVURLAsset, type: AVMediaType) {
+        guard let assetTrack = asset.tracks(withMediaType: type).first else {
+            return
+        }
+
+        if let track = nextComposition.tracks(withMediaType: type).first {
+            logger.info("removing \(type) track")
+            nextComposition.removeTrack(track)
+        }
+
+        let track = nextComposition.addMutableTrack(withMediaType: type, preferredTrackID: kCMPersistentTrackID_Invalid)!
+
+        try! track.insertTimeRange(
+            CMTimeRange(start: .zero, duration: CMTime(seconds: video.length, preferredTimescale: 1000)),
+            of: assetTrack,
+            at: .zero
+        )
+
+        logger.info("inserted \(type) track")
+    }
+
+    func replaceCompositionTracks() {
+        logger.warning("replacing compositions")
+
+        composition = AVMutableComposition()
+
+        nextComposition.tracks.forEach { track in
+            let newTrack = composition.addMutableTrack(withMediaType: track.mediaType, preferredTrackID: kCMPersistentTrackID_Invalid)!
+
+            try? newTrack.insertTimeRange(
+                CMTimeRange(start: .zero, duration: CMTime(seconds: video.length, preferredTimescale: 1000)),
+                of: track,
+                at: .zero
+            )
+        }
+    }
+
+    func removeTracksFromNextComposition() {
+        nextComposition.tracks.forEach { nextComposition.removeTrack($0) }
+    }
+
     func saveTime() {
         guard player != nil else {
             return
@@ -131,7 +191,7 @@ final class PlayerState: ObservableObject {
         player.currentItem?.tracks.forEach { $0.assetTrack?.asset?.cancelLoading() }
 
         currentStream?.cancelLoadingAssets()
-        streamToLoad?.cancelLoadingAssets()
+        nextStream?.cancelLoadingAssets()
 
         player.cancelPendingPrerolls()
         player.replaceCurrentItem(with: nil)

Model/Segment.swift

@@ -9,6 +9,18 @@ class Segment: ObservableObject, Hashable {
     let uuid: String
     let videoDuration: Int
 
+    var start: Double {
+        segment.first!
+    }
+
+    var end: Double {
+        segment.last!
+    }
+
+    var duration: Double {
+        end - start
+    }
+
     init(category: String, segment: [Double], uuid: String, videoDuration: Int) {
         self.category = category
         self.segment = segment
@@ -17,11 +29,11 @@ class Segment: ObservableObject, Hashable {
     }
 
     func timeInSegment(_ time: CMTime) -> Bool {
-        (segment.first! ... segment.last!).contains(time.seconds)
+        (start ... end).contains(time.seconds)
     }
 
     var skipTo: CMTime {
-        CMTime(seconds: segment.last!, preferredTimescale: 1)
+        CMTime(seconds: segment.last!, preferredTimescale: 1000)
     }
 
     func hash(into hasher: inout Hasher) {

Model/SponsorBlockSegmentsProvider.swift

@@ -29,7 +29,7 @@ final class SponsorBlockSegmentsProvider: ObservableObject {
     private var parameters: [String: String] {
         [
             "videoID": id,
-            "categories": JSON(categories).rawString(String.Encoding.utf8)!
+            "categories": JSON(categories).rawString(String.Encoding.utf8)!,
         ]
     }
 }

Pearvidious.xcodeproj/project.pbxproj

@@ -85,6 +85,9 @@
 		37C7A1D5267BFD9D0010EAD6 /* SponsorBlockSegment.swift in Sources */ = {isa = PBXBuildFile; fileRef = 37C7A1D4267BFD9D0010EAD6 /* SponsorBlockSegment.swift */; };
 		37C7A1D6267BFD9D0010EAD6 /* SponsorBlockSegment.swift in Sources */ = {isa = PBXBuildFile; fileRef = 37C7A1D4267BFD9D0010EAD6 /* SponsorBlockSegment.swift */; };
 		37C7A1D7267BFD9D0010EAD6 /* SponsorBlockSegment.swift in Sources */ = {isa = PBXBuildFile; fileRef = 37C7A1D4267BFD9D0010EAD6 /* SponsorBlockSegment.swift */; };
+		37C7A1D8267CACE10010EAD6 /* TrendingCategorySelectionView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3705B17D267B4DDE00704544 /* TrendingCategorySelectionView.swift */; };
+		37C7A1D9267CACE60010EAD6 /* VisualEffectView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3705B17B267B4D9A00704544 /* VisualEffectView.swift */; };
+		37C7A1DA267CACF50010EAD6 /* TrendingCountrySelectionView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3705B17F267B4DFB00704544 /* TrendingCountrySelectionView.swift */; };
 		37C7A9042679059200E721B4 /* AVKeyValueStatus+String.swift in Sources */ = {isa = PBXBuildFile; fileRef = 37C7A9032679059200E721B4 /* AVKeyValueStatus+String.swift */; };
 		37C7A905267905AE00E721B4 /* AVKeyValueStatus+String.swift in Sources */ = {isa = PBXBuildFile; fileRef = 37C7A9032679059200E721B4 /* AVKeyValueStatus+String.swift */; };
 		37C7A906267905AF00E721B4 /* AVKeyValueStatus+String.swift in Sources */ = {isa = PBXBuildFile; fileRef = 37C7A9032679059200E721B4 /* AVKeyValueStatus+String.swift */; };
@@ -655,9 +658,11 @@
 				37AAF29C26741B5F007FC770 /* SubscriptionVideosProvider.swift in Sources */,
 				37141668267A83F9006CA35D /* StreamAVPlayerViewController.swift in Sources */,
 				37EAD86B267B9C5600D9E01B /* SponsorBlockSegmentsProvider.swift in Sources */,
+				37C7A1DA267CACF50010EAD6 /* TrendingCountrySelectionView.swift in Sources */,
 				377FC7E6267A085600A6BBAF /* PlayerView.swift in Sources */,
 				37CEE4C12677B697005A1EFE /* Stream.swift in Sources */,
 				37141677267A9AAD006CA35D /* TrendingState.swift in Sources */,
+				37C7A1D9267CACE60010EAD6 /* VisualEffectView.swift in Sources */,
 				37D4B0E62671614900C925CA /* ContentView.swift in Sources */,
 				377FC7DC267A081800A6BBAF /* PopularVideosView.swift in Sources */,
 				3714167F267AB55D006CA35D /* TrendingVideosProvider.swift in Sources */,
@@ -682,6 +687,7 @@
 				37AAF2A026741C97007FC770 /* SubscriptionsView.swift in Sources */,
 				3714167B267AA1CF006CA35D /* TrendingCountriesProvider.swift in Sources */,
 				377FC7DF267A082200A6BBAF /* VideosView.swift in Sources */,
+				37C7A1D8267CACE10010EAD6 /* TrendingCategorySelectionView.swift in Sources */,
 				37D4B19726717E1500C925CA /* Video.swift in Sources */,
 				37D4B0E42671614900C925CA /* PearvidiousApp.swift in Sources */,
 				37CEE4B92677B63F005A1EFE /* StreamResolution.swift in Sources */,