Improve building AVPlayer composition

Model/PlayerState.swift

@@ -1,6 +1,7 @@
 import AVFoundation
 import Foundation
 import Logging
+import UIKit
 
 final class PlayerState: ObservableObject {
     let logger = Logger(label: "net.arekf.Pearvidious.ps")
@@ -8,24 +9,40 @@ final class PlayerState: ObservableObject {
     var video: Video
 
     @Published private(set) var player: AVPlayer! = AVPlayer()
-    private(set) var composition = AVMutableComposition()
+    @Published private(set) var composition = AVMutableComposition()
+    @Published private(set) var nextComposition = AVMutableComposition()
+
+    private var comp: AVMutableComposition?
+
     @Published private(set) var currentStream: Stream!
 
-    @Published private(set) var streamToLoad: Stream!
+    @Published private(set) var nextStream: Stream!
     @Published private(set) var streamLoading = false
 
     @Published private(set) var currentTime: CMTime?
     @Published private(set) var savedTime: CMTime?
+
     @Published var currentSegment: Segment?
 
     var playerItem: AVPlayerItem {
         let playerItem = AVPlayerItem(asset: composition)
 
-        playerItem.externalMetadata = [
+        var externalMetadata = [
             makeMetadataItem(.commonIdentifierTitle, value: video.title),
             makeMetadataItem(.quickTimeMetadataGenre, value: video.genre),
-            makeMetadataItem(.commonIdentifierDescription, value: video.description)
+            makeMetadataItem(.commonIdentifierDescription, value: video.description),
         ]
+
+        if let thumbnailData = try? Data(contentsOf: video.thumbnailURL!),
+           let image = UIImage(data: thumbnailData),
+           let pngData = image.pngData()
+        {
+            let artworkItem = makeMetadataItem(.commonIdentifierArtwork, value: pngData)
+            externalMetadata.append(artworkItem)
+        }
+
+        playerItem.externalMetadata = externalMetadata
+
         playerItem.preferredForwardBufferDuration = 10
 
         return playerItem
@@ -46,17 +63,18 @@ final class PlayerState: ObservableObject {
     }
 
     func loadStream(_ stream: Stream?) {
-        guard streamToLoad != stream else {
+        guard nextStream != stream else {
             return
         }
 
-        streamToLoad?.cancelLoadingAssets()
+        nextStream?.cancelLoadingAssets()
+        removeTracksFromNextComposition()
 
         DispatchQueue.main.async {
             self.streamLoading = true
-            self.streamToLoad = stream
+            self.nextStream = stream
         }
-        logger.info("replace streamToLoad: \(streamToLoad?.description ?? "nil"), streamLoading \(streamLoading)")
+        logger.info("replace streamToLoad: \(nextStream?.description ?? "nil"), streamLoading \(streamLoading)")
     }
 
     func streamDidLoad(_ stream: Stream?) {
@@ -64,24 +82,24 @@ final class PlayerState: ObservableObject {
 
         currentStream?.cancelLoadingAssets()
         currentStream = stream
-        streamLoading = streamToLoad != stream
+        streamLoading = nextStream != stream
 
-        if streamToLoad == stream {
-            streamToLoad = nil
+        if nextStream == stream {
+            nextStream = nil
         }
 
         addTimeObserver()
     }
 
     func cancelLoadingStream(_ stream: Stream) {
-        guard streamToLoad == stream else {
+        guard nextStream == stream else {
             return
         }
 
-        streamToLoad = nil
+        nextStream = nil
         streamLoading = false
 
-        logger.info("cancel streamToLoad: \(streamToLoad?.description ?? "nil"), streamLoading \(streamLoading)")
+        logger.info("cancel streamToLoad: \(nextStream?.description ?? "nil"), streamLoading \(streamLoading)")
     }
 
     func playStream(_ stream: Stream) {
@@ -92,6 +110,7 @@ final class PlayerState: ObservableObject {
         logger.warning("loading \(stream.description) to player")
 
         saveTime()
+        replaceCompositionTracks()
 
         player.replaceCurrentItem(with: playerItem)
         streamDidLoad(stream)
@@ -99,6 +118,47 @@ final class PlayerState: ObservableObject {
         seekToSavedTime()
     }
 
+    func addTrackToNextComposition(_ asset: AVURLAsset, type: AVMediaType) {
+        guard let assetTrack = asset.tracks(withMediaType: type).first else {
+            return
+        }
+
+        if let track = nextComposition.tracks(withMediaType: type).first {
+            logger.info("removing \(type) track")
+            nextComposition.removeTrack(track)
+        }
+
+        let track = nextComposition.addMutableTrack(withMediaType: type, preferredTrackID: kCMPersistentTrackID_Invalid)!
+
+        try! track.insertTimeRange(
+            CMTimeRange(start: .zero, duration: CMTime(seconds: video.length, preferredTimescale: 1000)),
+            of: assetTrack,
+            at: .zero
+        )
+
+        logger.info("inserted \(type) track")
+    }
+
+    func replaceCompositionTracks() {
+        logger.warning("replacing compositions")
+
+        composition = AVMutableComposition()
+
+        nextComposition.tracks.forEach { track in
+            let newTrack = composition.addMutableTrack(withMediaType: track.mediaType, preferredTrackID: kCMPersistentTrackID_Invalid)!
+
+            try? newTrack.insertTimeRange(
+                CMTimeRange(start: .zero, duration: CMTime(seconds: video.length, preferredTimescale: 1000)),
+                of: track,
+                at: .zero
+            )
+        }
+    }
+
+    func removeTracksFromNextComposition() {
+        nextComposition.tracks.forEach { nextComposition.removeTrack($0) }
+    }
+
     func saveTime() {
         guard player != nil else {
             return
@@ -131,7 +191,7 @@ final class PlayerState: ObservableObject {
         player.currentItem?.tracks.forEach { $0.assetTrack?.asset?.cancelLoading() }
 
         currentStream?.cancelLoadingAssets()
-        streamToLoad?.cancelLoadingAssets()
+        nextStream?.cancelLoadingAssets()
 
         player.cancelPendingPrerolls()
         player.replaceCurrentItem(with: nil)

Model/Segment.swift

@@ -9,6 +9,18 @@ class Segment: ObservableObject, Hashable {
     let uuid: String
     let videoDuration: Int
 
+    var start: Double {
+        segment.first!
+    }
+
+    var end: Double {
+        segment.last!
+    }
+
+    var duration: Double {
+        end - start
+    }
+
     init(category: String, segment: [Double], uuid: String, videoDuration: Int) {
         self.category = category
         self.segment = segment
@@ -17,11 +29,11 @@ class Segment: ObservableObject, Hashable {
     }
 
     func timeInSegment(_ time: CMTime) -> Bool {
-        (segment.first! ... segment.last!).contains(time.seconds)
+        (start ... end).contains(time.seconds)
     }
 
     var skipTo: CMTime {
-        CMTime(seconds: segment.last!, preferredTimescale: 1)
+        CMTime(seconds: segment.last!, preferredTimescale: 1000)
     }
 
     func hash(into hasher: inout Hasher) {

Model/SponsorBlockSegmentsProvider.swift

@@ -29,7 +29,7 @@ final class SponsorBlockSegmentsProvider: ObservableObject {
     private var parameters: [String: String] {
         [
             "videoID": id,
-            "categories": JSON(categories).rawString(String.Encoding.utf8)!
+            "categories": JSON(categories).rawString(String.Encoding.utf8)!,
         ]
     }
 }