MPV: improved A/V sync

- use displays refresh rate
- execute needs drawing with higher priority
- run create() with higher priority
- determine the number of threads used for rendering
- enable VSYNC and change video-sync  to display-resample
- iOS/tvOS: set new display refresh rate on change
- run setSize with higher priority
- add more options to MPVClient
- get refresh rate updates
- sync refresh rate to fps
- update CADisplayLink to current refresh rate
- update refresh rate on macOS
- Add experimental feature to sync display  with content fps

Signed-off-by: Toni Förster <toni.foerster@gmail.com>

Model/Import Export Settings/Exporters/AdvancedSettingsGroupExporter.swift

@@ -13,6 +13,7 @@ final class AdvancedSettingsGroupExporter: SettingsGroupExporter {
             "mpvDeinterlace": Defaults[.mpvDeinterlace],
             "mpvHWdec": Defaults[.mpvHWdec],
             "mpvDemuxerLavfProbeInfo": Defaults[.mpvDemuxerLavfProbeInfo],
+            "mpvSetRefreshToContentFPS": Defaults[.mpvSetRefreshToContentFPS],
             "mpvInitialAudioSync": Defaults[.mpvInitialAudioSync],
             "showCacheStatus": Defaults[.showCacheStatus],
             "feedCacheSize": Defaults[.feedCacheSize]

Model/Import Export Settings/Importers/AdvancedSettingsGroupImporter.swift

@@ -41,6 +41,10 @@ struct AdvancedSettingsGroupImporter {
             Defaults[.mpvDemuxerLavfProbeInfo] = mpvDemuxerLavfProbeInfo
         }
 
+        if let mpvSetRefreshToContentFPS = json["mpvSetRefreshToContentFPS"].bool {
+            Defaults[.mpvSetRefreshToContentFPS] = mpvSetRefreshToContentFPS
+        }
+
         if let mpvInitialAudioSync = json["mpvInitialAudioSync"].bool {
             Defaults[.mpvInitialAudioSync] = mpvInitialAudioSync
         }

Model/Player/Backends/MPVBackend.swift

@@ -11,6 +11,7 @@ import SwiftUI
 final class MPVBackend: PlayerBackend {
     static var timeUpdateInterval = 0.5
     static var networkStateUpdateInterval = 0.1
+    static var refreshRateUpdateInterval = 0.5
 
     private var logger = Logger(label: "mpv-backend")
 
@@ -89,6 +90,7 @@ final class MPVBackend: PlayerBackend {
 
     private var clientTimer: Repeater!
     private var networkStateTimer: Repeater!
+    private var refreshRateTimer: Repeater!
 
     private var onFileLoaded: (() -> Void)?
 
@@ -184,21 +186,24 @@ final class MPVBackend: PlayerBackend {
     }
 
     init() {
-        // swiftlint:disable shorthand_optional_binding
         clientTimer = .init(interval: .seconds(Self.timeUpdateInterval), mode: .infinite) { [weak self] _ in
-            guard let self = self, self.model.activeBackend == .mpv else {
+            guard let self, self.model.activeBackend == .mpv else {
                 return
             }
             self.getTimeUpdates()
         }
 
         networkStateTimer = .init(interval: .seconds(Self.networkStateUpdateInterval), mode: .infinite) { [weak self] _ in
-            guard let self = self, self.model.activeBackend == .mpv else {
+            guard let self, self.model.activeBackend == .mpv else {
                 return
             }
             self.updateNetworkState()
         }
-        // swiftlint:enable shorthand_optional_binding
+
+        refreshRateTimer = .init(interval: .seconds(Self.refreshRateUpdateInterval), mode: .infinite) { [weak self] _ in
+            guard let self, self.model.activeBackend == .mpv else { return }
+            self.checkAndUpdateRefreshRate()
+        }
     }
 
     typealias AreInIncreasingOrder = (Stream, Stream) -> Bool
@@ -343,8 +348,17 @@ final class MPVBackend: PlayerBackend {
         startClientUpdates()
     }
 
+    func startRefreshRateUpdates() {
+        refreshRateTimer.start()
+    }
+
+    func stopRefreshRateUpdates() {
+        refreshRateTimer.pause()
+    }
+
     func play() {
         startClientUpdates()
+        startRefreshRateUpdates()
 
         if controls.presentingControls {
             startControlsUpdates()
@@ -372,6 +386,7 @@ final class MPVBackend: PlayerBackend {
 
     func pause() {
         stopClientUpdates()
+        stopRefreshRateUpdates()
 
         client?.pause()
         isPaused = true
@@ -391,6 +406,8 @@ final class MPVBackend: PlayerBackend {
     }
 
     func stop() {
+        stopClientUpdates()
+        stopRefreshRateUpdates()
         client?.stop()
         isPlaying = false
         isPaused = false
@@ -472,6 +489,52 @@ final class MPVBackend: PlayerBackend {
         }
     }
 
+    private func checkAndUpdateRefreshRate() {
+        guard let screenRefreshRate = client?.getScreenRefreshRate() else {
+            logger.warning("Failed to get screen refresh rate.")
+            return
+        }
+
+        let contentFps = client?.currentContainerFps ?? screenRefreshRate
+
+        guard Defaults[.mpvSetRefreshToContentFPS] else {
+            // If the current refresh rate doesn't match the screen refresh rate, reset it
+            if client?.currentRefreshRate != screenRefreshRate {
+                client?.updateRefreshRate(to: screenRefreshRate)
+                client?.currentRefreshRate = screenRefreshRate
+                #if !os(macOS)
+                    notifyViewToUpdateDisplayLink(with: screenRefreshRate)
+                #endif
+                logger.info("Reset refresh rate to screen's rate: \(screenRefreshRate) Hz")
+            }
+            return
+        }
+
+        // Adjust the refresh rate to match the content if it differs
+        if screenRefreshRate != contentFps {
+            client?.updateRefreshRate(to: contentFps)
+            client?.currentRefreshRate = contentFps
+            #if !os(macOS)
+                notifyViewToUpdateDisplayLink(with: contentFps)
+            #endif
+            logger.info("Adjusted screen refresh rate to match content: \(contentFps) Hz")
+        } else if client?.currentRefreshRate != screenRefreshRate {
+            // Ensure the refresh rate is set back to the screen's rate if no adjustment is needed
+            client?.updateRefreshRate(to: screenRefreshRate)
+            client?.currentRefreshRate = screenRefreshRate
+            #if !os(macOS)
+                notifyViewToUpdateDisplayLink(with: screenRefreshRate)
+            #endif
+            logger.info("Checked and reset refresh rate to screen's rate: \(screenRefreshRate) Hz")
+        }
+    }
+
+    #if !os(macOS)
+        private func notifyViewToUpdateDisplayLink(with refreshRate: Int) {
+            NotificationCenter.default.post(name: .updateDisplayLinkFrameRate, object: nil, userInfo: ["refreshRate": refreshRate])
+        }
+    #endif
+
     func handle(_ event: UnsafePointer<mpv_event>!) {
         logger.info(.init(stringLiteral: "RECEIVED  event: \(String(cString: mpv_event_name(event.pointee.event_id)))"))
 

Model/Player/Backends/MPVClient.swift

@@ -6,6 +6,8 @@ import Logging
 #if !os(macOS)
     import Siesta
     import UIKit
+#else
+    import AppKit
 #endif
 
 final class MPVClient: ObservableObject {
@@ -29,6 +31,7 @@ final class MPVClient: ObservableObject {
     var backend: MPVBackend!
 
     var seeking = false
+    var currentRefreshRate = 60
 
     func create(frame: CGRect? = nil) {
         #if !os(macOS)
@@ -76,6 +79,27 @@ final class MPVClient: ObservableObject {
         checkError(mpv_set_option_string(mpv, "user-agent", UserAgentManager.shared.userAgent))
         checkError(mpv_set_option_string(mpv, "initial-audio-sync", Defaults[.mpvInitialAudioSync] ? "yes" : "no"))
 
+        // Enable VSYNC – needed for `video-sync`
+        checkError(mpv_set_option_string(mpv, "opengl-swapinterval", "1"))
+        checkError(mpv_set_option_string(mpv, "video-sync", "display-resample"))
+        checkError(mpv_set_option_string(mpv, "interpolation", "yes"))
+        checkError(mpv_set_option_string(mpv, "tscale", "mitchell"))
+        checkError(mpv_set_option_string(mpv, "tscale-window", "blackman"))
+        checkError(mpv_set_option_string(mpv, "vd-lavc-framedrop", "nonref"))
+        checkError(mpv_set_option_string(mpv, "display-fps-override", "\(String(getScreenRefreshRate()))"))
+
+        // CPU //
+
+        // Determine number of threads based on system core count
+        let numberOfCores = ProcessInfo.processInfo.processorCount
+        let threads = numberOfCores * 2
+
+        // Log the number of cores and threads
+        logger.info("Number of CPU cores: \(numberOfCores)")
+
+        // Set the number of threads dynamically
+        checkError(mpv_set_option_string(mpv, "vd-lavc-threads", "\(threads)"))
+
         // GPU //
 
         checkError(mpv_set_option_string(mpv, "hwdec", Defaults[.mpvHWdec]))
@@ -83,7 +107,6 @@ final class MPVClient: ObservableObject {
 
         // We set set everything to OpenGL so MPV doesn't have to probe for other APIs.
         checkError(mpv_set_option_string(mpv, "gpu-api", "opengl"))
-        checkError(mpv_set_option_string(mpv, "opengl-swapinterval", "0"))
 
         #if !os(macOS)
             checkError(mpv_set_option_string(mpv, "opengl-es", "yes"))
@@ -114,7 +137,7 @@ final class MPVClient: ObservableObject {
             get_proc_address_ctx: nil
         )
 
-        queue = DispatchQueue(label: "mpv")
+        queue = DispatchQueue(label: "mpv", qos: .userInteractive, attributes: [.concurrent])
 
         withUnsafeMutablePointer(to: &initParams) { initParams in
             var params = [
@@ -320,6 +343,17 @@ final class MPVClient: ObservableObject {
         mpv.isNil ? false : getFlag("eof-reached")
     }
 
+    var currentContainerFps: Int {
+        guard !mpv.isNil else { return 30 }
+        let fps = getDouble("container-fps")
+        return Int(fps.rounded())
+    }
+
+    func logCurrentFps() {
+        let fps = currentContainerFps
+        logger.info("Current container FPS: \(fps)")
+    }
+
     func seek(relative time: CMTime, completionHandler: ((Bool) -> Void)? = nil) {
         guard !seeking else {
             logger.warning("ignoring seek, another in progress")
@@ -363,7 +397,7 @@ final class MPVClient: ObservableObject {
                 return
             }
 
-            DispatchQueue.main.async { [weak self] in
+            DispatchQueue.main.async(qos: .userInteractive) { [weak self] in
                 guard let self else { return }
                 let model = self.backend.model
                 let aspectRatio = self.aspectRatio > 0 && self.aspectRatio < VideoPlayerView.defaultAspectRatio ? self.aspectRatio : VideoPlayerView.defaultAspectRatio
@@ -442,6 +476,45 @@ final class MPVClient: ObservableObject {
         }
     }
 
+    func updateRefreshRate(to refreshRate: Int) {
+        setString("display-fps-override", "\(String(refreshRate))")
+        logger.info("Updated refresh rate during playback to: \(refreshRate) Hz")
+    }
+
+    // Retrieve the screen's current refresh rate dynamically.
+    func getScreenRefreshRate() -> Int {
+        var refreshRate = 60 // Default to 60 Hz in case of failure
+
+        #if os(macOS)
+            // macOS implementation using NSScreen
+            if let screen = NSScreen.main,
+               let displayID = screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? CGDirectDisplayID,
+               let mode = CGDisplayCopyDisplayMode(displayID),
+               mode.refreshRate > 0
+            {
+                refreshRate = Int(mode.refreshRate)
+                print("Screen refresh rate: \(refreshRate) Hz")
+            } else {
+                print("Failed to get refresh rate from NSScreen.")
+            }
+        #else
+            // iOS implementation using UIScreen with a failover
+            let mainScreen = UIScreen.main
+            refreshRate = mainScreen.maximumFramesPerSecond
+
+            // Failover: if maximumFramesPerSecond is 0 or an unexpected value
+            if refreshRate <= 0 {
+                refreshRate = 60 // Fallback to 60 Hz
+                print("Failed to get refresh rate from UIScreen, falling back to 60 Hz.")
+            } else {
+                print("Screen refresh rate: \(refreshRate) Hz")
+            }
+        #endif
+
+        currentRefreshRate = refreshRate
+        return refreshRate
+    }
+
     func addVideoTrack(_ url: URL) {
         command("video-add", args: [url.absoluteString])
     }

Model/Player/PlayerModel.swift

@@ -532,8 +532,8 @@ final class PlayerModel: ObservableObject {
     }
 
     private func handlePresentationChange() {
-        #if !os(iOS)
+        #if os(macOS)
-            // TODO: Check whether this is neede on tvOS and macOS
+            // TODO: Check whether this is needed on macOS
             backend.setNeedsDrawing(presentingPlayer)
         #endif
 
@@ -1007,23 +1007,21 @@ final class PlayerModel: ObservableObject {
         }
     #else
         func handleEnterForeground() {
+            DispatchQueue.global(qos: .userInteractive).async { [weak self] in
+                guard let self = self else { return }
+
+                if !self.musicMode, self.activeBackend == .mpv {
+                    self.mpvBackend.addVideoTrackFromStream()
+                    self.mpvBackend.setVideoToAuto()
+                    self.mpvBackend.controls.resetTimer()
+                } else if !self.musicMode, self.activeBackend == .appleAVPlayer {
+                    self.avPlayerBackend.bindPlayerToLayer()
+                }
+            }
             #if os(iOS)
                 OrientationTracker.shared.startDeviceOrientationTracking()
             #endif
 
-            #if os(tvOS)
-                // TODO: Not sure if this is realy needed on tvOS, maybe it can be removed.
-                setNeedsDrawing(presentingPlayer)
-            #endif
-
-            if !musicMode, activeBackend == .mpv {
-                mpvBackend.addVideoTrackFromStream()
-                mpvBackend.setVideoToAuto()
-                mpvBackend.controls.resetTimer()
-            } else if !musicMode, activeBackend == .appleAVPlayer {
-                avPlayerBackend.bindPlayerToLayer()
-            }
-
             guard closePiPAndOpenPlayerOnEnteringForeground, playingInPictureInPicture else {
                 return
             }

Shared/Defaults.swift

@@ -368,6 +368,7 @@ extension Defaults.Keys {
     static let mpvHWdec = Key<String>("hwdec", default: "auto-safe")
     static let mpvDemuxerLavfProbeInfo = Key<String>("mpvDemuxerLavfProbeInfo", default: "no")
     static let mpvInitialAudioSync = Key<Bool>("mpvInitialAudioSync", default: true)
+    static let mpvSetRefreshToContentFPS = Key<Bool>("mpvSetRefreshToContentFPS", default: false)
 
     static let showCacheStatus = Key<Bool>("showCacheStatus", default: false)
     static let feedCacheSize = Key<String>("feedCacheSize", default: "50")

Shared/Player/MPV/MPVOGLView.swift

@@ -6,9 +6,10 @@ import OpenGLES
 final class MPVOGLView: GLKView {
     private var logger = Logger(label: "stream.yattee.mpv.oglview")
     private var defaultFBO: GLint?
+    private var displayLink: CADisplayLink?
 
     var mpvGL: UnsafeMutableRawPointer?
-    var queue = DispatchQueue(label: "stream.yattee.opengl")
+    var queue = DispatchQueue(label: "stream.yattee.opengl", qos: .userInteractive)
     var needsDrawing = true
 
     override init(frame: CGRect) {
@@ -29,6 +30,69 @@ final class MPVOGLView: GLKView {
         enableSetNeedsDisplay = false
 
         fillBlack()
+        setupDisplayLink()
+        setupNotifications()
+    }
+
+    required init?(coder aDecoder: NSCoder) {
+        super.init(coder: aDecoder)
+        setupDisplayLink()
+        setupNotifications()
+    }
+
+    private func setupDisplayLink() {
+        displayLink = CADisplayLink(target: self, selector: #selector(updateFrame))
+        displayLink?.add(to: .main, forMode: .common)
+    }
+
+    // Set up observers to detect display changes and custom refresh rate updates.
+    private func setupNotifications() {
+        NotificationCenter.default.addObserver(self, selector: #selector(updateDisplayLinkFromNotification(_:)), name: .updateDisplayLinkFrameRate, object: nil)
+        NotificationCenter.default.addObserver(self, selector: #selector(screenDidChange), name: UIScreen.didConnectNotification, object: nil)
+        NotificationCenter.default.addObserver(self, selector: #selector(screenDidChange), name: UIScreen.didDisconnectNotification, object: nil)
+        NotificationCenter.default.addObserver(self, selector: #selector(screenDidChange), name: UIScreen.modeDidChangeNotification, object: nil)
+    }
+
+    @objc private func screenDidChange(_: Notification) {
+        // Update the display link refresh rate when the screen configuration changes
+        updateDisplayLinkFrameRate()
+    }
+
+    // Update the display link frame rate from the notification.
+    @objc private func updateDisplayLinkFromNotification(_ notification: Notification) {
+        guard let userInfo = notification.userInfo,
+              let refreshRate = userInfo["refreshRate"] as? Int else { return }
+        displayLink?.preferredFramesPerSecond = refreshRate
+        logger.info("Updated CADisplayLink frame rate to: \(refreshRate) from backend notification.")
+    }
+
+    // Update the display link's preferred frame rate based on the current screen refresh rate.
+    private func updateDisplayLinkFrameRate() {
+        guard let displayLink else { return }
+        let refreshRate = getScreenRefreshRate()
+        displayLink.preferredFramesPerSecond = refreshRate
+        logger.info("Updated CADisplayLink preferred frames per second to: \(refreshRate)")
+    }
+
+    // Retrieve the screen's current refresh rate dynamically.
+    private func getScreenRefreshRate() -> Int {
+        // Use the main screen's maximumFramesPerSecond property
+        let refreshRate = UIScreen.main.maximumFramesPerSecond
+        logger.info("Screen refresh rate: \(refreshRate) Hz")
+        return refreshRate
+    }
+
+    @objc private func updateFrame() {
+        // Trigger the drawing process if needed
+        if needsDrawing {
+            setNeedsDisplay()
+        }
+    }
+
+    deinit {
+        // Invalidate the display link and remove observers to avoid memory leaks
+        displayLink?.invalidate()
+        NotificationCenter.default.removeObserver(self)
     }
 
     func fillBlack() {
@@ -37,35 +101,40 @@ final class MPVOGLView: GLKView {
     }
 
     override func draw(_: CGRect) {
-        guard needsDrawing, let mpvGL else {
+        guard needsDrawing, let mpvGL else { return }
-            return
-        }
 
+        // Bind the default framebuffer
         glGetIntegerv(UInt32(GL_FRAMEBUFFER_BINDING), &defaultFBO!)
 
+        // Get the current viewport dimensions
         var dims: [GLint] = [0, 0, 0, 0]
         glGetIntegerv(GLenum(GL_VIEWPORT), &dims)
 
+        // Set up the OpenGL FBO data
         var data = mpv_opengl_fbo(
             fbo: Int32(defaultFBO!),
             w: Int32(dims[2]),
             h: Int32(dims[3]),
             internal_format: 0
         )
+
+        // Flip Y coordinate for proper rendering
         var flip: CInt = 1
-        withUnsafeMutablePointer(to: &flip) { flip in
+
-            withUnsafeMutablePointer(to: &data) { data in
+        // Render with the provided OpenGL FBO parameters
+        withUnsafeMutablePointer(to: &flip) { flipPtr in
+            withUnsafeMutablePointer(to: &data) { dataPtr in
                 var params = [
-                    mpv_render_param(type: MPV_RENDER_PARAM_OPENGL_FBO, data: data),
+                    mpv_render_param(type: MPV_RENDER_PARAM_OPENGL_FBO, data: dataPtr),
-                    mpv_render_param(type: MPV_RENDER_PARAM_FLIP_Y, data: flip),
+                    mpv_render_param(type: MPV_RENDER_PARAM_FLIP_Y, data: flipPtr),
                     mpv_render_param()
                 ]
                 mpv_render_context_render(OpaquePointer(mpvGL), &params)
             }
         }
     }
-
+}
-    required init?(coder aDecoder: NSCoder) {
+
-        super.init(coder: aDecoder)
+extension Notification.Name {
-    }
+    static let updateDisplayLinkFrameRate = Notification.Name("updateDisplayLinkFrameRate")
 }

Shared/Settings/AdvancedSettings.swift

@@ -11,6 +11,7 @@ struct AdvancedSettings: View {
     @Default(.mpvHWdec) private var mpvHWdec
     @Default(.mpvDemuxerLavfProbeInfo) private var mpvDemuxerLavfProbeInfo
     @Default(.mpvInitialAudioSync) private var mpvInitialAudioSync
+    @Default(.mpvSetRefreshToContentFPS) private var mpvSetRefreshToContentFPS
     @Default(.showCacheStatus) private var showCacheStatus
     @Default(.feedCacheSize) private var feedCacheSize
     @Default(.showPlayNowInBackendContextMenu) private var showPlayNowInBackendContextMenu
@@ -245,6 +246,12 @@ struct AdvancedSettings: View {
                 #endif
             }
 
+            Toggle(isOn: $mpvSetRefreshToContentFPS) {
+                HStack {
+                    Text("Sync refresh rate with content FPS – EXPERIMENTAL")
+                }
+            }
+
             if mpvEnableLogging {
                 logButton
             }