yattee/Model/Player/Backends/AVPlayerBackend.swift

761 lines
23 KiB
Swift
Raw Normal View History

2023-05-20 20:49:10 +00:00
import AVKit
2022-02-16 20:23:11 +00:00
import Defaults
import Foundation
2022-08-23 21:29:50 +00:00
import Logging
2022-02-16 20:23:11 +00:00
import MediaPlayer
2022-02-27 20:31:17 +00:00
#if !os(macOS)
import UIKit
#endif
2023-05-20 20:49:10 +00:00
import SwiftUI
2022-02-16 20:23:11 +00:00
final class AVPlayerBackend: PlayerBackend {
static let assetKeysToLoad = ["tracks", "playable", "duration"]
2022-08-23 21:29:50 +00:00
private var logger = Logger(label: "avplayer-backend")
var model: PlayerModel { .shared }
var controls: PlayerControlsModel { .shared }
var playerTime: PlayerTimeModel { .shared }
var networkState: NetworkStateModel { .shared }
var seek: SeekModel { .shared }
2022-02-16 20:23:11 +00:00
var stream: Stream?
var video: Video?
2022-11-10 22:19:34 +00:00
var suggestedPlaybackRates: [Double] {
[0.5, 0.67, 0.8, 1, 1.25, 1.5, 2]
}
func canPlayAtRate(_ rate: Double) -> Bool {
suggestedPlaybackRates.contains(rate)
}
2022-02-16 20:23:11 +00:00
var currentTime: CMTime? {
avPlayer.currentTime()
}
var loadedVideo: Bool {
!avPlayer.currentItem.isNil
}
var isLoadingVideo = false
2022-02-16 20:23:11 +00:00
var isPlaying: Bool {
avPlayer.timeControlStatus == .playing
}
2022-11-10 17:11:28 +00:00
var videoWidth: Double? {
if let width = avPlayer.currentItem?.presentationSize.width {
return Double(width)
}
return nil
}
var videoHeight: Double? {
if let height = avPlayer.currentItem?.presentationSize.height {
return Double(height)
}
return nil
}
2022-07-09 00:21:04 +00:00
var aspectRatio: Double {
2022-07-10 01:15:15 +00:00
#if os(iOS)
guard let videoWidth, let videoHeight else {
return VideoPlayerView.defaultAspectRatio
}
return videoWidth / videoHeight
2022-07-09 00:21:04 +00:00
#else
2022-07-10 01:15:15 +00:00
VideoPlayerView.defaultAspectRatio
2022-07-09 00:21:04 +00:00
#endif
}
var isSeeking: Bool {
// TODO: implement this maybe?
false
}
2022-02-16 20:23:11 +00:00
var playerItemDuration: CMTime? {
avPlayer.currentItem?.asset.duration
}
private(set) var avPlayer = AVPlayer()
2022-08-18 22:40:46 +00:00
private(set) var playerLayer = AVPlayerLayer()
#if os(tvOS)
var controller: AppleAVPlayerViewController?
2023-05-20 20:49:10 +00:00
#elseif os(iOS)
var controller = AVPlayerViewController() { didSet {
controller.player = avPlayer
}}
2022-08-18 22:40:46 +00:00
#endif
2022-05-21 20:58:11 +00:00
var startPictureInPictureOnPlay = false
2022-08-26 20:17:21 +00:00
var startPictureInPictureOnSwitch = false
2022-02-16 20:23:11 +00:00
private var asset: AVURLAsset?
private var composition = AVMutableComposition()
private var loadedCompositionAssets = [AVMediaType]()
private var frequentTimeObserver: Any?
private var infrequentTimeObserver: Any?
private var playerTimeControlStatusObserver: Any?
private var statusObservation: NSKeyValueObservation?
private var timeObserverThrottle = Throttle(interval: 2)
2022-08-23 21:29:50 +00:00
internal var controlsUpdates = false
2022-09-01 23:05:31 +00:00
init() {
2022-02-16 20:23:11 +00:00
addFrequentTimeObserver()
addInfrequentTimeObserver()
addPlayerTimeControlStatusObserver()
2022-08-18 22:40:46 +00:00
playerLayer.player = avPlayer
2023-05-20 20:49:10 +00:00
#if os(iOS)
controller.player = avPlayer
#endif
2022-02-16 20:23:11 +00:00
}
func bestPlayable(_ streams: [Stream], maxResolution: ResolutionSetting) -> Stream? {
let sortedByResolution = streams
.filter { ($0.kind == .adaptive || $0.kind == .stream) && $0.resolution <= maxResolution.value }
.sorted { $0.resolution > $1.resolution }
return streams.first { $0.kind == .hls } ??
sortedByResolution.first { $0.kind == .stream } ??
sortedByResolution.first
2022-02-16 20:23:11 +00:00
}
func canPlay(_ stream: Stream) -> Bool {
stream.kind == .hls || stream.kind == .stream || (stream.kind == .adaptive && stream.format == .mp4)
2022-02-16 20:23:11 +00:00
}
func playStream(
_ stream: Stream,
of video: Video,
preservingTime: Bool,
2022-02-16 21:10:57 +00:00
upgrading _: Bool
2022-02-16 20:23:11 +00:00
) {
isLoadingVideo = true
2022-11-18 23:58:21 +00:00
if let url = stream.singleAssetURL {
2022-02-16 20:23:11 +00:00
model.logger.info("playing stream with one asset\(stream.kind == .hls ? " (HLS)" : ""): \(url)")
2022-11-10 17:11:28 +00:00
2022-11-12 02:05:56 +00:00
if video.isLocal, video.localStreamIsFile {
_ = url.startAccessingSecurityScopedResource()
2022-11-10 17:11:28 +00:00
}
2022-02-16 20:23:11 +00:00
loadSingleAsset(url, stream: stream, of: video, preservingTime: preservingTime)
} else {
model.logger.info("playing stream with many assets:")
model.logger.info("composition audio asset: \(stream.audioAsset.url)")
model.logger.info("composition video asset: \(stream.videoAsset.url)")
loadComposition(stream, of: video, preservingTime: preservingTime)
}
}
func play() {
guard avPlayer.timeControlStatus != .playing else {
return
}
avPlayer.play()
2022-08-26 20:17:21 +00:00
model.objectWillChange.send()
2022-02-16 20:23:11 +00:00
}
func pause() {
guard avPlayer.timeControlStatus != .paused else {
return
}
avPlayer.pause()
2022-08-26 20:17:21 +00:00
model.objectWillChange.send()
2022-02-16 20:23:11 +00:00
}
func togglePlay() {
isPlaying ? pause() : play()
}
func stop() {
avPlayer.replaceCurrentItem(with: nil)
}
func cancelLoads() {
asset?.cancelLoading()
composition.cancelLoading()
}
2022-08-29 11:55:23 +00:00
func seek(to time: CMTime, seekType _: SeekType, completionHandler: ((Bool) -> Void)?) {
2022-07-21 22:44:21 +00:00
guard !model.live else { return }
2022-02-16 20:23:11 +00:00
avPlayer.seek(
to: time,
2022-08-26 20:17:21 +00:00
toleranceBefore: .zero,
2022-02-16 20:23:11 +00:00
toleranceAfter: .zero,
completionHandler: completionHandler ?? { _ in }
)
}
func setRate(_ rate: Double) {
avPlayer.rate = Float(rate)
2022-02-16 20:23:11 +00:00
}
func closeItem() {
avPlayer.replaceCurrentItem(with: nil)
2022-08-26 20:17:21 +00:00
video = nil
stream = nil
2022-02-16 20:23:11 +00:00
}
2022-08-18 22:40:46 +00:00
func closePiP() {
model.pipController?.stopPictureInPicture()
}
2022-02-16 20:23:11 +00:00
private func loadSingleAsset(
_ url: URL,
stream: Stream,
of video: Video,
preservingTime: Bool = false
) {
asset?.cancelLoading()
asset = AVURLAsset(url: url)
asset?.loadValuesAsynchronously(forKeys: Self.assetKeysToLoad) { [weak self] in
var error: NSError?
switch self?.asset?.statusOfValue(forKey: "duration", error: &error) {
case .loaded:
DispatchQueue.main.async { [weak self] in
self?.insertPlayerItem(stream, for: video, preservingTime: preservingTime)
}
case .failed:
DispatchQueue.main.async { [weak self] in
self?.model.playerError = error
}
default:
return
}
}
}
private func loadComposition(
_ stream: Stream,
of video: Video,
preservingTime: Bool = false
) {
loadedCompositionAssets = []
loadCompositionAsset(stream.audioAsset, stream: stream, type: .audio, of: video, preservingTime: preservingTime, model: model)
loadCompositionAsset(stream.videoAsset, stream: stream, type: .video, of: video, preservingTime: preservingTime, model: model)
}
private func loadCompositionAsset(
_ asset: AVURLAsset,
stream: Stream,
type: AVMediaType,
of video: Video,
preservingTime: Bool = false,
model: PlayerModel
) {
asset.loadValuesAsynchronously(forKeys: Self.assetKeysToLoad) { [weak self] in
2022-09-28 14:27:01 +00:00
guard let self else {
2022-02-16 20:23:11 +00:00
return
}
model.logger.info("loading \(type.rawValue) track")
let assetTracks = asset.tracks(withMediaType: type)
guard let compositionTrack = self.composition.addMutableTrack(
withMediaType: type,
preferredTrackID: kCMPersistentTrackID_Invalid
) else {
model.logger.critical("composition \(type.rawValue) addMutableTrack FAILED")
return
}
guard let assetTrack = assetTracks.first else {
model.logger.critical("asset \(type.rawValue) track FAILED")
return
}
try! compositionTrack.insertTimeRange(
CMTimeRange(start: .zero, duration: CMTime.secondsInDefaultTimescale(video.length)),
of: assetTrack,
at: .zero
)
model.logger.critical("\(type.rawValue) LOADED")
guard model.streamSelection == stream else {
model.logger.critical("IGNORING LOADED")
return
}
self.loadedCompositionAssets.append(type)
if self.loadedCompositionAssets.count == 2 {
self.insertPlayerItem(stream, for: video, preservingTime: preservingTime)
}
}
}
private func insertPlayerItem(
_ stream: Stream,
for video: Video,
preservingTime: Bool = false
) {
removeItemDidPlayToEndTimeObserver()
model.playerItem = playerItem(stream)
guard model.playerItem != nil else {
return
}
addItemDidPlayToEndTimeObserver()
attachMetadata()
2022-02-16 20:23:11 +00:00
DispatchQueue.main.async {
self.stream = stream
self.video = video
self.model.stream = stream
self.composition = AVMutableComposition()
self.asset = nil
}
let startPlaying = {
#if !os(macOS)
try? AVAudioSession.sharedInstance().setActive(true)
#endif
2022-07-10 01:15:15 +00:00
self.setRate(self.model.currentRate)
guard let item = self.model.playerItem, self.isAutoplaying(item) else { return }
2022-02-16 20:23:11 +00:00
2022-07-10 01:15:15 +00:00
DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) { [weak self] in
2022-09-28 14:27:01 +00:00
guard let self else {
2022-07-10 01:15:15 +00:00
return
}
if !preservingTime,
2022-08-26 20:17:21 +00:00
!self.model.transitioningToPiP,
2022-07-10 01:15:15 +00:00
let segment = self.model.sponsorBlock.segments.first,
segment.start < 3,
self.model.lastSkipped.isNil
{
self.avPlayer.seek(
to: segment.endTime,
toleranceBefore: .secondsInDefaultTimescale(1),
toleranceAfter: .zero
) { finished in
guard finished else {
return
2022-02-16 20:23:11 +00:00
}
2022-07-10 01:15:15 +00:00
self.model.lastSkipped = segment
2022-02-16 20:23:11 +00:00
self.model.play()
}
2022-07-10 01:15:15 +00:00
} else {
self.model.play()
2022-02-16 20:23:11 +00:00
}
}
}
let replaceItemAndSeek = {
guard video == self.model.currentVideo else {
return
}
2022-11-10 17:11:28 +00:00
2022-02-16 20:23:11 +00:00
self.avPlayer.replaceCurrentItem(with: self.model.playerItem)
self.seekToPreservedTime { finished in
guard finished else {
return
}
2022-08-22 17:04:01 +00:00
DispatchQueue.main.async { [weak self] in
self?.model.preservedTime = nil
}
2022-02-16 20:23:11 +00:00
startPlaying()
}
}
if preservingTime {
if model.preservedTime.isNil {
model.saveTime {
replaceItemAndSeek()
startPlaying()
}
} else {
replaceItemAndSeek()
startPlaying()
}
} else {
avPlayer.replaceCurrentItem(with: model.playerItem)
startPlaying()
}
}
private func seekToPreservedTime(completionHandler: @escaping (Bool) -> Void = { _ in }) {
guard let time = model.preservedTime else {
return
}
avPlayer.seek(
to: time,
toleranceBefore: .secondsInDefaultTimescale(1),
toleranceAfter: .zero,
completionHandler: completionHandler
)
}
private func playerItem(_: Stream) -> AVPlayerItem? {
2022-09-28 14:27:01 +00:00
if let asset {
2022-02-16 20:23:11 +00:00
return AVPlayerItem(asset: asset)
} else {
return AVPlayerItem(asset: composition)
}
}
private func attachMetadata() {
guard let video = model.currentVideo else { return }
2022-02-16 20:23:11 +00:00
#if !os(macOS)
var externalMetadata = [
2022-11-10 17:11:28 +00:00
makeMetadataItem(.commonIdentifierTitle, value: video.displayTitle),
makeMetadataItem(.commonIdentifierArtist, value: video.displayAuthor),
2022-02-16 20:23:11 +00:00
makeMetadataItem(.quickTimeMetadataGenre, value: video.genre ?? ""),
makeMetadataItem(.commonIdentifierDescription, value: video.description ?? "")
]
if let thumbnailURL = video.thumbnailURL(quality: .medium) {
let task = URLSession.shared.dataTask(with: thumbnailURL) { [weak self] thumbnailData, _, _ in
2022-09-28 14:27:01 +00:00
guard let thumbnailData else { return }
let image = UIImage(data: thumbnailData)
if let pngData = image?.pngData() {
if let artworkItem = self?.makeMetadataItem(.commonIdentifierArtwork, value: pngData) {
externalMetadata.append(artworkItem)
}
}
self?.avPlayer.currentItem?.externalMetadata = externalMetadata
}
task.resume()
2022-02-16 20:23:11 +00:00
}
#endif
if let item = model.playerItem {
#if !os(macOS)
item.externalMetadata = externalMetadata
#endif
item.preferredForwardBufferDuration = 5
observePlayerItemStatus(item)
}
2022-02-16 20:23:11 +00:00
}
#if !os(macOS)
private func makeMetadataItem(_ identifier: AVMetadataIdentifier, value: Any) -> AVMetadataItem {
let item = AVMutableMetadataItem()
item.identifier = identifier
item.value = value as? NSCopying & NSObjectProtocol
item.extendedLanguageTag = "und"
return item.copy() as! AVMetadataItem
}
#endif
func isAutoplaying(_ item: AVPlayerItem) -> Bool {
avPlayer.currentItem == item
}
private func observePlayerItemStatus(_ item: AVPlayerItem) {
statusObservation?.invalidate()
statusObservation = item.observe(\.status, options: [.old, .new]) { [weak self] playerItem, _ in
2022-09-28 14:27:01 +00:00
guard let self else {
2022-02-16 20:23:11 +00:00
return
}
isLoadingVideo = false
2022-02-16 20:23:11 +00:00
switch playerItem.status {
case .readyToPlay:
if self.model.activeBackend == .appleAVPlayer,
2022-08-13 14:46:45 +00:00
self.isAutoplaying(playerItem)
{
self.model.updateAspectRatio()
2022-08-26 20:17:21 +00:00
if self.startPictureInPictureOnPlay,
let controller = self.model.pipController,
controller.isPictureInPicturePossible
{
self.tryStartingPictureInPicture()
} else {
self.model.play()
}
} else if self.startPictureInPictureOnPlay {
self.model.stream = self.stream
self.model.streamSelection = self.stream
if self.model.activeBackend != .appleAVPlayer {
self.startPictureInPictureOnSwitch = true
let seconds = self.model.mpvBackend.currentTime?.seconds ?? 0
2023-05-20 20:49:10 +00:00
self.seek(to: seconds, seekType: .backendSync) { finished in
guard finished else { return }
DispatchQueue.main.async {
self.model.pause()
self.model.changeActiveBackend(from: .mpv, to: .appleAVPlayer, changingStream: false)
2023-05-20 20:49:10 +00:00
Delay.by(3) {
self.startPictureInPictureOnPlay = false
}
}
2022-08-26 20:17:21 +00:00
}
}
2022-02-16 20:23:11 +00:00
}
case .failed:
2022-08-18 22:36:16 +00:00
DispatchQueue.main.async {
self.model.playerError = item.error
}
2022-02-16 20:23:11 +00:00
default:
return
}
}
}
private func addItemDidPlayToEndTimeObserver() {
NotificationCenter.default.addObserver(
self,
selector: #selector(itemDidPlayToEndTime),
name: NSNotification.Name.AVPlayerItemDidPlayToEndTime,
2022-05-21 20:58:11 +00:00
object: model.playerItem
2022-02-16 20:23:11 +00:00
)
}
private func removeItemDidPlayToEndTimeObserver() {
NotificationCenter.default.removeObserver(
self,
name: NSNotification.Name.AVPlayerItemDidPlayToEndTime,
2022-05-21 20:58:11 +00:00
object: model.playerItem
2022-02-16 20:23:11 +00:00
)
}
@objc func itemDidPlayToEndTime() {
2022-07-10 22:24:56 +00:00
eofPlaybackModeAction()
2022-02-16 20:23:11 +00:00
}
private func addFrequentTimeObserver() {
let interval = CMTime.secondsInDefaultTimescale(0.5)
frequentTimeObserver = avPlayer.addPeriodicTimeObserver(
forInterval: interval,
queue: .main
) { [weak self] _ in
2022-09-28 14:27:01 +00:00
guard let self, self.model.activeBackend == .appleAVPlayer else {
2022-02-16 20:23:11 +00:00
return
}
guard !self.model.currentItem.isNil else {
return
}
self.model.updateNowPlayingInfo()
2022-02-16 20:23:11 +00:00
#if os(macOS)
MPNowPlayingInfoCenter.default().playbackState = self.avPlayer.timeControlStatus == .playing ? .playing : .paused
2022-02-16 20:23:11 +00:00
#endif
2022-09-01 18:10:53 +00:00
if self.controls.isPlaying != self.isPlaying {
DispatchQueue.main.async {
self.controls.isPlaying = self.isPlaying
}
}
2022-02-16 20:23:11 +00:00
if let currentTime = self.currentTime {
self.model.handleSegments(at: currentTime)
}
2022-08-20 20:28:31 +00:00
#if !os(macOS)
guard UIApplication.shared.applicationState != .background else {
print("not performing controls updates in background")
return
}
#endif
if self.controlsUpdates {
self.updateControls()
2022-08-20 20:28:31 +00:00
}
2022-02-16 20:23:11 +00:00
}
}
private func addInfrequentTimeObserver() {
let interval = CMTime.secondsInDefaultTimescale(5)
infrequentTimeObserver = avPlayer.addPeriodicTimeObserver(
forInterval: interval,
queue: .main
) { [weak self] _ in
2022-09-28 14:27:01 +00:00
guard let self else {
2022-02-16 20:23:11 +00:00
return
}
guard !self.model.currentItem.isNil else {
return
}
self.timeObserverThrottle.execute {
self.model.updateWatch()
}
}
}
private func addPlayerTimeControlStatusObserver() {
playerTimeControlStatusObserver = avPlayer.observe(\.timeControlStatus) { [weak self] player, _ in
2022-09-28 14:27:01 +00:00
guard let self,
2022-09-01 18:10:53 +00:00
self.avPlayer == player,
self.model.activeBackend == .appleAVPlayer
2022-02-16 20:23:11 +00:00
else {
return
}
let isPlaying = player.timeControlStatus == .playing
if self.controls.isPlaying != isPlaying {
DispatchQueue.main.async {
self.controls.isPlaying = player.timeControlStatus == .playing
}
2022-02-16 20:23:11 +00:00
}
if player.timeControlStatus != .waitingToPlayAtSpecifiedRate {
DispatchQueue.main.async { [weak self] in
self?.model.objectWillChange.send()
}
}
2022-05-20 21:23:14 +00:00
if player.timeControlStatus == .playing {
2022-08-31 19:24:46 +00:00
self.model.objectWillChange.send()
if player.rate != Float(self.model.currentRate) {
player.rate = Float(self.model.currentRate)
2022-05-20 21:23:14 +00:00
}
2022-02-16 20:23:11 +00:00
}
#if os(macOS)
if player.timeControlStatus == .playing {
ScreenSaverManager.shared.disable(reason: "Yattee is playing video")
} else {
ScreenSaverManager.shared.enable()
}
#endif
self.timeObserverThrottle.execute {
self.model.updateWatch()
}
}
}
2022-03-27 11:42:20 +00:00
func startControlsUpdates() {
2022-08-23 21:29:50 +00:00
guard model.presentingPlayer, model.controls.presentingControls, !model.controls.presentingOverlays else {
logger.info("ignored controls update start")
return
}
logger.info("starting controls updates")
controlsUpdates = true
2022-08-26 20:17:21 +00:00
model.objectWillChange.send()
}
func stopControlsUpdates() {
controlsUpdates = false
2022-08-26 20:17:21 +00:00
model.objectWillChange.send()
}
2022-08-20 20:31:03 +00:00
func startMusicMode() {
if model.playingInPictureInPicture {
closePiP()
}
playerLayer.player = nil
toggleVisualTracksEnabled(false)
}
func stopMusicMode() {
playerLayer.player = avPlayer
toggleVisualTracksEnabled(true)
}
func toggleVisualTracksEnabled(_ value: Bool) {
if let item = avPlayer.currentItem {
for playerItemTrack in item.tracks {
if let assetTrack = playerItemTrack.assetTrack,
assetTrack.hasMediaCharacteristic(AVMediaCharacteristic.visual)
{
playerItemTrack.isEnabled = value
}
}
}
}
func didChangeTo() {
2022-08-26 20:17:21 +00:00
if startPictureInPictureOnSwitch {
tryStartingPictureInPicture()
} else if model.musicMode {
2022-08-20 20:31:03 +00:00
startMusicMode()
} else {
stopMusicMode()
}
}
2023-05-20 20:49:10 +00:00
var isStartingPiP: Bool {
startPictureInPictureOnPlay || startPictureInPictureOnSwitch
}
2022-08-26 20:17:21 +00:00
func tryStartingPictureInPicture() {
guard let controller = model.pipController else { return }
var opened = false
for delay in [0.1, 0.3, 0.5, 1, 2, 3, 5] {
Delay.by(delay) {
guard !opened else { return }
if controller.isPictureInPicturePossible {
opened = true
controller.startPictureInPicture()
} else {
print("PiP not possible, waited \(delay) seconds")
}
}
}
2023-05-20 20:49:10 +00:00
Delay.by(5) {
self.startPictureInPictureOnSwitch = false
}
}
func setPlayerInLayer(_ playerIsPresented: Bool) {
if playerIsPresented {
bindPlayerToLayer()
} else {
removePlayerFromLayer()
}
}
func removePlayerFromLayer() {
playerLayer.player = nil
#if os(iOS)
controller.player = nil
#endif
}
func bindPlayerToLayer() {
playerLayer.player = avPlayer
#if os(iOS)
controller.player = avPlayer
#endif
2022-08-26 20:17:21 +00:00
}
func getTimeUpdates() {}
2022-03-27 11:42:20 +00:00
func setNeedsDrawing(_: Bool) {}
func setSize(_: Double, _: Double) {}
2022-06-24 23:39:29 +00:00
func setNeedsNetworkStateUpdates(_: Bool) {}
2022-02-16 20:23:11 +00:00
}