From 5e85fd294c079d27611a4ee0afdf88b0dd7a5842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toni=20F=C3=B6rster?= Date: Sat, 7 Sep 2024 22:22:09 +0200 Subject: [PATCH] MPV: improved A/V sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - use displays refresh rate - execute needs drawing with higher priority - run create() with higher priority - determine the number of threads used for rendering - enable VSYNC and change video-sync to display-resample - iOS/tvOS: set new display refresh rate on change - run setSize with higher priority - add more options to MPVClient - get refresh rate updates - sync refresh rate to fps - update CADisplayLink to current refresh rate - update refresh rate on macOS - Add experimental feature to sync display with content fps Signed-off-by: Toni Förster --- .../AdvancedSettingsGroupExporter.swift | 1 + .../AdvancedSettingsGroupImporter.swift | 4 + Model/Player/Backends/MPVBackend.swift | 71 +++++++++++++- Model/Player/Backends/MPVClient.swift | 79 +++++++++++++++- Model/Player/PlayerModel.swift | 28 +++--- Shared/Defaults.swift | 1 + Shared/Player/MPV/MPVOGLView.swift | 93 ++++++++++++++++--- Shared/Settings/AdvancedSettings.swift | 7 ++ 8 files changed, 250 insertions(+), 34 deletions(-) diff --git a/Model/Import Export Settings/Exporters/AdvancedSettingsGroupExporter.swift b/Model/Import Export Settings/Exporters/AdvancedSettingsGroupExporter.swift index 9aa5145b..7b394148 100644 --- a/Model/Import Export Settings/Exporters/AdvancedSettingsGroupExporter.swift +++ b/Model/Import Export Settings/Exporters/AdvancedSettingsGroupExporter.swift @@ -13,6 +13,7 @@ final class AdvancedSettingsGroupExporter: SettingsGroupExporter { "mpvDeinterlace": Defaults[.mpvDeinterlace], "mpvHWdec": Defaults[.mpvHWdec], "mpvDemuxerLavfProbeInfo": Defaults[.mpvDemuxerLavfProbeInfo], + "mpvSetRefreshToContentFPS": Defaults[.mpvSetRefreshToContentFPS], "mpvInitialAudioSync": Defaults[.mpvInitialAudioSync], "showCacheStatus": Defaults[.showCacheStatus], "feedCacheSize": Defaults[.feedCacheSize] diff --git a/Model/Import Export Settings/Importers/AdvancedSettingsGroupImporter.swift b/Model/Import Export Settings/Importers/AdvancedSettingsGroupImporter.swift index 5c2b2113..8d9347ae 100644 --- a/Model/Import Export Settings/Importers/AdvancedSettingsGroupImporter.swift +++ b/Model/Import Export Settings/Importers/AdvancedSettingsGroupImporter.swift @@ -41,6 +41,10 @@ struct AdvancedSettingsGroupImporter { Defaults[.mpvDemuxerLavfProbeInfo] = mpvDemuxerLavfProbeInfo } + if let mpvSetRefreshToContentFPS = json["mpvSetRefreshToContentFPS"].bool { + Defaults[.mpvSetRefreshToContentFPS] = mpvSetRefreshToContentFPS + } + if let mpvInitialAudioSync = json["mpvInitialAudioSync"].bool { Defaults[.mpvInitialAudioSync] = mpvInitialAudioSync } diff --git a/Model/Player/Backends/MPVBackend.swift b/Model/Player/Backends/MPVBackend.swift index 942589ba..c876d169 100644 --- a/Model/Player/Backends/MPVBackend.swift +++ b/Model/Player/Backends/MPVBackend.swift @@ -11,6 +11,7 @@ import SwiftUI final class MPVBackend: PlayerBackend { static var timeUpdateInterval = 0.5 static var networkStateUpdateInterval = 0.1 + static var refreshRateUpdateInterval = 0.5 private var logger = Logger(label: "mpv-backend") @@ -89,6 +90,7 @@ final class MPVBackend: PlayerBackend { private var clientTimer: Repeater! private var networkStateTimer: Repeater! + private var refreshRateTimer: Repeater! private var onFileLoaded: (() -> Void)? @@ -184,21 +186,24 @@ final class MPVBackend: PlayerBackend { } init() { - // swiftlint:disable shorthand_optional_binding clientTimer = .init(interval: .seconds(Self.timeUpdateInterval), mode: .infinite) { [weak self] _ in - guard let self = self, self.model.activeBackend == .mpv else { + guard let self, self.model.activeBackend == .mpv else { return } self.getTimeUpdates() } networkStateTimer = .init(interval: .seconds(Self.networkStateUpdateInterval), mode: .infinite) { [weak self] _ in - guard let self = self, self.model.activeBackend == .mpv else { + guard let self, self.model.activeBackend == .mpv else { return } self.updateNetworkState() } - // swiftlint:enable shorthand_optional_binding + + refreshRateTimer = .init(interval: .seconds(Self.refreshRateUpdateInterval), mode: .infinite) { [weak self] _ in + guard let self, self.model.activeBackend == .mpv else { return } + self.checkAndUpdateRefreshRate() + } } typealias AreInIncreasingOrder = (Stream, Stream) -> Bool @@ -343,8 +348,17 @@ final class MPVBackend: PlayerBackend { startClientUpdates() } + func startRefreshRateUpdates() { + refreshRateTimer.start() + } + + func stopRefreshRateUpdates() { + refreshRateTimer.pause() + } + func play() { startClientUpdates() + startRefreshRateUpdates() if controls.presentingControls { startControlsUpdates() @@ -372,6 +386,7 @@ final class MPVBackend: PlayerBackend { func pause() { stopClientUpdates() + stopRefreshRateUpdates() client?.pause() isPaused = true @@ -391,6 +406,8 @@ final class MPVBackend: PlayerBackend { } func stop() { + stopClientUpdates() + stopRefreshRateUpdates() client?.stop() isPlaying = false isPaused = false @@ -472,6 +489,52 @@ final class MPVBackend: PlayerBackend { } } + private func checkAndUpdateRefreshRate() { + guard let screenRefreshRate = client?.getScreenRefreshRate() else { + logger.warning("Failed to get screen refresh rate.") + return + } + + let contentFps = client?.currentContainerFps ?? screenRefreshRate + + guard Defaults[.mpvSetRefreshToContentFPS] else { + // If the current refresh rate doesn't match the screen refresh rate, reset it + if client?.currentRefreshRate != screenRefreshRate { + client?.updateRefreshRate(to: screenRefreshRate) + client?.currentRefreshRate = screenRefreshRate + #if !os(macOS) + notifyViewToUpdateDisplayLink(with: screenRefreshRate) + #endif + logger.info("Reset refresh rate to screen's rate: \(screenRefreshRate) Hz") + } + return + } + + // Adjust the refresh rate to match the content if it differs + if screenRefreshRate != contentFps { + client?.updateRefreshRate(to: contentFps) + client?.currentRefreshRate = contentFps + #if !os(macOS) + notifyViewToUpdateDisplayLink(with: contentFps) + #endif + logger.info("Adjusted screen refresh rate to match content: \(contentFps) Hz") + } else if client?.currentRefreshRate != screenRefreshRate { + // Ensure the refresh rate is set back to the screen's rate if no adjustment is needed + client?.updateRefreshRate(to: screenRefreshRate) + client?.currentRefreshRate = screenRefreshRate + #if !os(macOS) + notifyViewToUpdateDisplayLink(with: screenRefreshRate) + #endif + logger.info("Checked and reset refresh rate to screen's rate: \(screenRefreshRate) Hz") + } + } + + #if !os(macOS) + private func notifyViewToUpdateDisplayLink(with refreshRate: Int) { + NotificationCenter.default.post(name: .updateDisplayLinkFrameRate, object: nil, userInfo: ["refreshRate": refreshRate]) + } + #endif + func handle(_ event: UnsafePointer!) { logger.info(.init(stringLiteral: "RECEIVED event: \(String(cString: mpv_event_name(event.pointee.event_id)))")) diff --git a/Model/Player/Backends/MPVClient.swift b/Model/Player/Backends/MPVClient.swift index 0ef5832b..27fa92be 100644 --- a/Model/Player/Backends/MPVClient.swift +++ b/Model/Player/Backends/MPVClient.swift @@ -6,6 +6,8 @@ import Logging #if !os(macOS) import Siesta import UIKit +#else + import AppKit #endif final class MPVClient: ObservableObject { @@ -29,6 +31,7 @@ final class MPVClient: ObservableObject { var backend: MPVBackend! var seeking = false + var currentRefreshRate = 60 func create(frame: CGRect? = nil) { #if !os(macOS) @@ -76,6 +79,27 @@ final class MPVClient: ObservableObject { checkError(mpv_set_option_string(mpv, "user-agent", UserAgentManager.shared.userAgent)) checkError(mpv_set_option_string(mpv, "initial-audio-sync", Defaults[.mpvInitialAudioSync] ? "yes" : "no")) + // Enable VSYNC – needed for `video-sync` + checkError(mpv_set_option_string(mpv, "opengl-swapinterval", "1")) + checkError(mpv_set_option_string(mpv, "video-sync", "display-resample")) + checkError(mpv_set_option_string(mpv, "interpolation", "yes")) + checkError(mpv_set_option_string(mpv, "tscale", "mitchell")) + checkError(mpv_set_option_string(mpv, "tscale-window", "blackman")) + checkError(mpv_set_option_string(mpv, "vd-lavc-framedrop", "nonref")) + checkError(mpv_set_option_string(mpv, "display-fps-override", "\(String(getScreenRefreshRate()))")) + + // CPU // + + // Determine number of threads based on system core count + let numberOfCores = ProcessInfo.processInfo.processorCount + let threads = numberOfCores * 2 + + // Log the number of cores and threads + logger.info("Number of CPU cores: \(numberOfCores)") + + // Set the number of threads dynamically + checkError(mpv_set_option_string(mpv, "vd-lavc-threads", "\(threads)")) + // GPU // checkError(mpv_set_option_string(mpv, "hwdec", Defaults[.mpvHWdec])) @@ -83,7 +107,6 @@ final class MPVClient: ObservableObject { // We set set everything to OpenGL so MPV doesn't have to probe for other APIs. checkError(mpv_set_option_string(mpv, "gpu-api", "opengl")) - checkError(mpv_set_option_string(mpv, "opengl-swapinterval", "0")) #if !os(macOS) checkError(mpv_set_option_string(mpv, "opengl-es", "yes")) @@ -114,7 +137,7 @@ final class MPVClient: ObservableObject { get_proc_address_ctx: nil ) - queue = DispatchQueue(label: "mpv") + queue = DispatchQueue(label: "mpv", qos: .userInteractive, attributes: [.concurrent]) withUnsafeMutablePointer(to: &initParams) { initParams in var params = [ @@ -320,6 +343,17 @@ final class MPVClient: ObservableObject { mpv.isNil ? false : getFlag("eof-reached") } + var currentContainerFps: Int { + guard !mpv.isNil else { return 30 } + let fps = getDouble("container-fps") + return Int(fps.rounded()) + } + + func logCurrentFps() { + let fps = currentContainerFps + logger.info("Current container FPS: \(fps)") + } + func seek(relative time: CMTime, completionHandler: ((Bool) -> Void)? = nil) { guard !seeking else { logger.warning("ignoring seek, another in progress") @@ -363,7 +397,7 @@ final class MPVClient: ObservableObject { return } - DispatchQueue.main.async { [weak self] in + DispatchQueue.main.async(qos: .userInteractive) { [weak self] in guard let self else { return } let model = self.backend.model let aspectRatio = self.aspectRatio > 0 && self.aspectRatio < VideoPlayerView.defaultAspectRatio ? self.aspectRatio : VideoPlayerView.defaultAspectRatio @@ -442,6 +476,45 @@ final class MPVClient: ObservableObject { } } + func updateRefreshRate(to refreshRate: Int) { + setString("display-fps-override", "\(String(refreshRate))") + logger.info("Updated refresh rate during playback to: \(refreshRate) Hz") + } + + // Retrieve the screen's current refresh rate dynamically. + func getScreenRefreshRate() -> Int { + var refreshRate = 60 // Default to 60 Hz in case of failure + + #if os(macOS) + // macOS implementation using NSScreen + if let screen = NSScreen.main, + let displayID = screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? CGDirectDisplayID, + let mode = CGDisplayCopyDisplayMode(displayID), + mode.refreshRate > 0 + { + refreshRate = Int(mode.refreshRate) + print("Screen refresh rate: \(refreshRate) Hz") + } else { + print("Failed to get refresh rate from NSScreen.") + } + #else + // iOS implementation using UIScreen with a failover + let mainScreen = UIScreen.main + refreshRate = mainScreen.maximumFramesPerSecond + + // Failover: if maximumFramesPerSecond is 0 or an unexpected value + if refreshRate <= 0 { + refreshRate = 60 // Fallback to 60 Hz + print("Failed to get refresh rate from UIScreen, falling back to 60 Hz.") + } else { + print("Screen refresh rate: \(refreshRate) Hz") + } + #endif + + currentRefreshRate = refreshRate + return refreshRate + } + func addVideoTrack(_ url: URL) { command("video-add", args: [url.absoluteString]) } diff --git a/Model/Player/PlayerModel.swift b/Model/Player/PlayerModel.swift index a7ba1c11..b2e4a4df 100644 --- a/Model/Player/PlayerModel.swift +++ b/Model/Player/PlayerModel.swift @@ -532,8 +532,8 @@ final class PlayerModel: ObservableObject { } private func handlePresentationChange() { - #if !os(iOS) - // TODO: Check whether this is neede on tvOS and macOS + #if os(macOS) + // TODO: Check whether this is needed on macOS backend.setNeedsDrawing(presentingPlayer) #endif @@ -1007,23 +1007,21 @@ final class PlayerModel: ObservableObject { } #else func handleEnterForeground() { + DispatchQueue.global(qos: .userInteractive).async { [weak self] in + guard let self = self else { return } + + if !self.musicMode, self.activeBackend == .mpv { + self.mpvBackend.addVideoTrackFromStream() + self.mpvBackend.setVideoToAuto() + self.mpvBackend.controls.resetTimer() + } else if !self.musicMode, self.activeBackend == .appleAVPlayer { + self.avPlayerBackend.bindPlayerToLayer() + } + } #if os(iOS) OrientationTracker.shared.startDeviceOrientationTracking() #endif - #if os(tvOS) - // TODO: Not sure if this is realy needed on tvOS, maybe it can be removed. - setNeedsDrawing(presentingPlayer) - #endif - - if !musicMode, activeBackend == .mpv { - mpvBackend.addVideoTrackFromStream() - mpvBackend.setVideoToAuto() - mpvBackend.controls.resetTimer() - } else if !musicMode, activeBackend == .appleAVPlayer { - avPlayerBackend.bindPlayerToLayer() - } - guard closePiPAndOpenPlayerOnEnteringForeground, playingInPictureInPicture else { return } diff --git a/Shared/Defaults.swift b/Shared/Defaults.swift index 05cf1844..3d717db9 100644 --- a/Shared/Defaults.swift +++ b/Shared/Defaults.swift @@ -368,6 +368,7 @@ extension Defaults.Keys { static let mpvHWdec = Key("hwdec", default: "auto-safe") static let mpvDemuxerLavfProbeInfo = Key("mpvDemuxerLavfProbeInfo", default: "no") static let mpvInitialAudioSync = Key("mpvInitialAudioSync", default: true) + static let mpvSetRefreshToContentFPS = Key("mpvSetRefreshToContentFPS", default: false) static let showCacheStatus = Key("showCacheStatus", default: false) static let feedCacheSize = Key("feedCacheSize", default: "50") diff --git a/Shared/Player/MPV/MPVOGLView.swift b/Shared/Player/MPV/MPVOGLView.swift index 6b818758..e3604d1e 100644 --- a/Shared/Player/MPV/MPVOGLView.swift +++ b/Shared/Player/MPV/MPVOGLView.swift @@ -6,9 +6,10 @@ import OpenGLES final class MPVOGLView: GLKView { private var logger = Logger(label: "stream.yattee.mpv.oglview") private var defaultFBO: GLint? + private var displayLink: CADisplayLink? var mpvGL: UnsafeMutableRawPointer? - var queue = DispatchQueue(label: "stream.yattee.opengl") + var queue = DispatchQueue(label: "stream.yattee.opengl", qos: .userInteractive) var needsDrawing = true override init(frame: CGRect) { @@ -29,6 +30,69 @@ final class MPVOGLView: GLKView { enableSetNeedsDisplay = false fillBlack() + setupDisplayLink() + setupNotifications() + } + + required init?(coder aDecoder: NSCoder) { + super.init(coder: aDecoder) + setupDisplayLink() + setupNotifications() + } + + private func setupDisplayLink() { + displayLink = CADisplayLink(target: self, selector: #selector(updateFrame)) + displayLink?.add(to: .main, forMode: .common) + } + + // Set up observers to detect display changes and custom refresh rate updates. + private func setupNotifications() { + NotificationCenter.default.addObserver(self, selector: #selector(updateDisplayLinkFromNotification(_:)), name: .updateDisplayLinkFrameRate, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(screenDidChange), name: UIScreen.didConnectNotification, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(screenDidChange), name: UIScreen.didDisconnectNotification, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(screenDidChange), name: UIScreen.modeDidChangeNotification, object: nil) + } + + @objc private func screenDidChange(_: Notification) { + // Update the display link refresh rate when the screen configuration changes + updateDisplayLinkFrameRate() + } + + // Update the display link frame rate from the notification. + @objc private func updateDisplayLinkFromNotification(_ notification: Notification) { + guard let userInfo = notification.userInfo, + let refreshRate = userInfo["refreshRate"] as? Int else { return } + displayLink?.preferredFramesPerSecond = refreshRate + logger.info("Updated CADisplayLink frame rate to: \(refreshRate) from backend notification.") + } + + // Update the display link's preferred frame rate based on the current screen refresh rate. + private func updateDisplayLinkFrameRate() { + guard let displayLink else { return } + let refreshRate = getScreenRefreshRate() + displayLink.preferredFramesPerSecond = refreshRate + logger.info("Updated CADisplayLink preferred frames per second to: \(refreshRate)") + } + + // Retrieve the screen's current refresh rate dynamically. + private func getScreenRefreshRate() -> Int { + // Use the main screen's maximumFramesPerSecond property + let refreshRate = UIScreen.main.maximumFramesPerSecond + logger.info("Screen refresh rate: \(refreshRate) Hz") + return refreshRate + } + + @objc private func updateFrame() { + // Trigger the drawing process if needed + if needsDrawing { + setNeedsDisplay() + } + } + + deinit { + // Invalidate the display link and remove observers to avoid memory leaks + displayLink?.invalidate() + NotificationCenter.default.removeObserver(self) } func fillBlack() { @@ -37,35 +101,40 @@ final class MPVOGLView: GLKView { } override func draw(_: CGRect) { - guard needsDrawing, let mpvGL else { - return - } + guard needsDrawing, let mpvGL else { return } + // Bind the default framebuffer glGetIntegerv(UInt32(GL_FRAMEBUFFER_BINDING), &defaultFBO!) + // Get the current viewport dimensions var dims: [GLint] = [0, 0, 0, 0] glGetIntegerv(GLenum(GL_VIEWPORT), &dims) + // Set up the OpenGL FBO data var data = mpv_opengl_fbo( fbo: Int32(defaultFBO!), w: Int32(dims[2]), h: Int32(dims[3]), internal_format: 0 ) + + // Flip Y coordinate for proper rendering var flip: CInt = 1 - withUnsafeMutablePointer(to: &flip) { flip in - withUnsafeMutablePointer(to: &data) { data in + + // Render with the provided OpenGL FBO parameters + withUnsafeMutablePointer(to: &flip) { flipPtr in + withUnsafeMutablePointer(to: &data) { dataPtr in var params = [ - mpv_render_param(type: MPV_RENDER_PARAM_OPENGL_FBO, data: data), - mpv_render_param(type: MPV_RENDER_PARAM_FLIP_Y, data: flip), + mpv_render_param(type: MPV_RENDER_PARAM_OPENGL_FBO, data: dataPtr), + mpv_render_param(type: MPV_RENDER_PARAM_FLIP_Y, data: flipPtr), mpv_render_param() ] mpv_render_context_render(OpaquePointer(mpvGL), ¶ms) } } } - - required init?(coder aDecoder: NSCoder) { - super.init(coder: aDecoder) - } +} + +extension Notification.Name { + static let updateDisplayLinkFrameRate = Notification.Name("updateDisplayLinkFrameRate") } diff --git a/Shared/Settings/AdvancedSettings.swift b/Shared/Settings/AdvancedSettings.swift index 0e1d25f3..e0579e5a 100644 --- a/Shared/Settings/AdvancedSettings.swift +++ b/Shared/Settings/AdvancedSettings.swift @@ -11,6 +11,7 @@ struct AdvancedSettings: View { @Default(.mpvHWdec) private var mpvHWdec @Default(.mpvDemuxerLavfProbeInfo) private var mpvDemuxerLavfProbeInfo @Default(.mpvInitialAudioSync) private var mpvInitialAudioSync + @Default(.mpvSetRefreshToContentFPS) private var mpvSetRefreshToContentFPS @Default(.showCacheStatus) private var showCacheStatus @Default(.feedCacheSize) private var feedCacheSize @Default(.showPlayNowInBackendContextMenu) private var showPlayNowInBackendContextMenu @@ -245,6 +246,12 @@ struct AdvancedSettings: View { #endif } + Toggle(isOn: $mpvSetRefreshToContentFPS) { + HStack { + Text("Sync refresh rate with content FPS – EXPERIMENTAL") + } + } + if mpvEnableLogging { logButton }