From 515f4ba06fea556f6b6e6b74ca77b3826a8df612 Mon Sep 17 00:00:00 2001 From: Geoffrey McRae Date: Thu, 6 Aug 2026 16:05:05 +1000 Subject: [PATCH] [client/host/idd/obs] scheduler: validate phase feedback Bump KVMFR to version 28 and attach generation, epoch, and deadline identities to cadence deliveries and their timing records. Accept phase corrections only for the exact completed publication. Keep VM and client clocks independent by exchanging relative phase errors. Make submission, completion, and work-estimate accounting best effort so scheduler state cannot delay GPU submission or frame publication. --- client/include/interface/transport.h | 7 +- client/src/frame_scheduler.c | 82 +++++---- client/src/frame_scheduler.h | 3 +- client/src/main.c | 46 ++--- client/transports/LGMP/lgmp.c | 63 +++++-- common/include/common/KVMFR.h | 38 +++- host/src/app.c | 15 +- idd/LGIdd/CFrameBufferResource.h | 39 ++-- idd/LGIdd/CFrameScheduler.cpp | 260 ++++++++++++++++++++++----- idd/LGIdd/CFrameScheduler.h | 46 ++++- idd/LGIdd/CIndirectDeviceContext.cpp | 72 ++++++-- idd/LGIdd/CIndirectDeviceContext.h | 9 +- idd/LGIdd/CSwapChainProcessor.cpp | 88 ++++++--- obs/frame_scheduler.c | 70 ++++---- obs/frame_scheduler.h | 6 +- obs/lg.c | 79 ++++++-- 16 files changed, 688 insertions(+), 235 deletions(-) diff --git a/client/include/interface/transport.h b/client/include/interface/transport.h index ea631ea6..6934ca21 100644 --- a/client/include/interface/transport.h +++ b/client/include/interface/transport.h @@ -96,6 +96,10 @@ typedef uint32_t LG_TransportFrameFlags; typedef struct LG_TransportFrameTiming { bool valid; /* producer fields below are available and coherent */ + bool phaseValid; + uint32_t scheduleGeneration; + uint32_t scheduleEpoch; + uint32_t scheduleDeadlineSerial; uint64_t captureTime; uint64_t postProcessTime; uint64_t copyTime; @@ -134,8 +138,8 @@ typedef struct LG_TransportFrame { uint64_t serial; uint64_t timestamp; - uint32_t scheduleGeneration; uint32_t scheduleEpoch; + uint32_t scheduleDeadlineSerial; LG_TransportFrameFlags flags; bool scheduleOwner; // Backend-owned immutable metadata, valid until releaseFrame. @@ -216,6 +220,7 @@ typedef struct LG_TransportControl int64_t phaseError; uint32_t feedbackFrameSerial; uint32_t feedbackScheduleEpoch; + uint32_t feedbackDeadlineSerial; uint32_t lease; } frameSchedule; diff --git a/client/src/frame_scheduler.c b/client/src/frame_scheduler.c index 533cc9fa..41fa8d2d 100644 --- a/client/src/frame_scheduler.c +++ b/client/src/frame_scheduler.c @@ -52,6 +52,7 @@ static struct int64_t phaseError; uint32_t feedbackFrameSerial; uint32_t feedbackScheduleEpoch; + uint32_t feedbackDeadlineSerial; unsigned feedbackSamples; bool feedbackDirty; @@ -92,23 +93,26 @@ static bool sendSchedule(LG_TransportFrameScheduleFlags flags, int64_t phaseError; uint32_t feedbackFrameSerial; uint32_t feedbackScheduleEpoch; + uint32_t feedbackDeadlineSerial; LG_LOCK(l_frameScheduler.lock); - phaseError = l_frameScheduler.phaseError; - feedbackFrameSerial = l_frameScheduler.feedbackFrameSerial; - feedbackScheduleEpoch = l_frameScheduler.feedbackScheduleEpoch; + phaseError = l_frameScheduler.phaseError; + feedbackFrameSerial = l_frameScheduler.feedbackFrameSerial; + feedbackScheduleEpoch = l_frameScheduler.feedbackScheduleEpoch; + feedbackDeadlineSerial = l_frameScheduler.feedbackDeadlineSerial; LG_UNLOCK(l_frameScheduler.lock); const LG_TransportControl control = { .type = LG_TRANSPORT_CONTROL_FRAME_SCHEDULE, .frameSchedule = { - .generation = l_frameScheduler.generation, - .flags = flags, - .period = period, - .targetSlack = FRAME_SCHEDULER_TARGET_SLACK_NS, - .phaseError = phaseError, - .feedbackFrameSerial = feedbackFrameSerial, - .feedbackScheduleEpoch = feedbackScheduleEpoch, - .lease = FRAME_SCHEDULER_LEASE_MS, + .generation = l_frameScheduler.generation, + .flags = flags, + .period = period, + .targetSlack = FRAME_SCHEDULER_TARGET_SLACK_NS, + .phaseError = phaseError, + .feedbackFrameSerial = feedbackFrameSerial, + .feedbackScheduleEpoch = feedbackScheduleEpoch, + .feedbackDeadlineSerial = feedbackDeadlineSerial, + .lease = FRAME_SCHEDULER_LEASE_MS, }, }; @@ -127,7 +131,8 @@ static bool sendSchedule(LG_TransportFrameScheduleFlags flags, l_frameScheduler.immediatePending = false; LG_LOCK(l_frameScheduler.lock); if (l_frameScheduler.feedbackFrameSerial == feedbackFrameSerial && - l_frameScheduler.feedbackScheduleEpoch == feedbackScheduleEpoch) + l_frameScheduler.feedbackScheduleEpoch == feedbackScheduleEpoch && + l_frameScheduler.feedbackDeadlineSerial == feedbackDeadlineSerial) l_frameScheduler.feedbackDirty = false; LG_UNLOCK(l_frameScheduler.lock); return true; @@ -156,11 +161,12 @@ void frameScheduler_start(LG_TransportFeatureFlags features) ++l_frameScheduler.generation; LG_LOCK(l_frameScheduler.lock); - l_frameScheduler.phaseError = 0; - l_frameScheduler.feedbackFrameSerial = 0; - l_frameScheduler.feedbackScheduleEpoch = 0; - l_frameScheduler.feedbackSamples = 0; - l_frameScheduler.feedbackDirty = false; + l_frameScheduler.phaseError = 0; + l_frameScheduler.feedbackFrameSerial = 0; + l_frameScheduler.feedbackScheduleEpoch = 0; + l_frameScheduler.feedbackDeadlineSerial = 0; + l_frameScheduler.feedbackSamples = 0; + l_frameScheduler.feedbackDirty = false; LG_UNLOCK(l_frameScheduler.lock); } @@ -194,11 +200,12 @@ void frameScheduler_update(void) l_frameScheduler.period = 0; l_frameScheduler.immediatePending = true; LG_LOCK(l_frameScheduler.lock); - l_frameScheduler.phaseError = 0; - l_frameScheduler.feedbackFrameSerial = 0; - l_frameScheduler.feedbackScheduleEpoch = 0; - l_frameScheduler.feedbackSamples = 0; - l_frameScheduler.feedbackDirty = false; + l_frameScheduler.phaseError = 0; + l_frameScheduler.feedbackFrameSerial = 0; + l_frameScheduler.feedbackScheduleEpoch = 0; + l_frameScheduler.feedbackDeadlineSerial = 0; + l_frameScheduler.feedbackSamples = 0; + l_frameScheduler.feedbackDirty = false; LG_UNLOCK(l_frameScheduler.lock); } return; @@ -219,11 +226,12 @@ void frameScheduler_update(void) ++l_frameScheduler.generation; l_frameScheduler.immediatePending = true; LG_LOCK(l_frameScheduler.lock); - l_frameScheduler.phaseError = 0; - l_frameScheduler.feedbackFrameSerial = 0; - l_frameScheduler.feedbackScheduleEpoch = 0; - l_frameScheduler.feedbackSamples = 0; - l_frameScheduler.feedbackDirty = false; + l_frameScheduler.phaseError = 0; + l_frameScheduler.feedbackFrameSerial = 0; + l_frameScheduler.feedbackScheduleEpoch = 0; + l_frameScheduler.feedbackDeadlineSerial = 0; + l_frameScheduler.feedbackSamples = 0; + l_frameScheduler.feedbackDirty = false; LG_UNLOCK(l_frameScheduler.lock); } else @@ -257,9 +265,10 @@ void frameScheduler_update(void) } void frameScheduler_feedback(uint64_t frameSerial, uint32_t generation, - uint32_t scheduleEpoch, uint64_t measuredPhase) + uint32_t scheduleEpoch, uint32_t deadlineSerial, + uint64_t measuredPhase) { - if (!generation || !scheduleEpoch) + if (!generation || !scheduleEpoch || !deadlineSerial) return; LG_LOCK(l_frameScheduler.lock); @@ -270,6 +279,14 @@ void frameScheduler_feedback(uint64_t frameSerial, uint32_t generation, return; } + if (l_frameScheduler.feedbackFrameSerial == (uint32_t)frameSerial && + l_frameScheduler.feedbackScheduleEpoch == scheduleEpoch && + l_frameScheduler.feedbackDeadlineSerial == deadlineSerial) + { + LG_UNLOCK(l_frameScheduler.lock); + return; + } + const int64_t period = (int64_t)l_frameScheduler.period; if (!period) { @@ -301,8 +318,9 @@ void frameScheduler_feedback(uint64_t frameSerial, uint32_t generation, if (l_frameScheduler.feedbackSamples < 32) ++l_frameScheduler.feedbackSamples; - l_frameScheduler.feedbackFrameSerial = (uint32_t)frameSerial; - l_frameScheduler.feedbackScheduleEpoch = scheduleEpoch; - l_frameScheduler.feedbackDirty = true; + l_frameScheduler.feedbackFrameSerial = (uint32_t)frameSerial; + l_frameScheduler.feedbackScheduleEpoch = scheduleEpoch; + l_frameScheduler.feedbackDeadlineSerial = deadlineSerial; + l_frameScheduler.feedbackDirty = true; LG_UNLOCK(l_frameScheduler.lock); } diff --git a/client/src/frame_scheduler.h b/client/src/frame_scheduler.h index 168a2821..0ed22dab 100644 --- a/client/src/frame_scheduler.h +++ b/client/src/frame_scheduler.h @@ -31,6 +31,7 @@ void frameScheduler_start(LG_TransportFeatureFlags features); void frameScheduler_stop(void); void frameScheduler_update(void); void frameScheduler_feedback(uint64_t frameSerial, uint32_t generation, - uint32_t scheduleEpoch, uint64_t measuredPhase); + uint32_t scheduleEpoch, uint32_t deadlineSerial, + uint64_t measuredPhase); #endif diff --git a/client/src/main.c b/client/src/main.c index 1c2e55a4..663729e4 100644 --- a/client/src/main.c +++ b/client/src/main.c @@ -217,9 +217,10 @@ struct FrameTimingRecord uint64_t frameSerial; uint32_t scheduleGeneration; uint32_t scheduleEpoch; + uint32_t scheduleDeadlineSerial; unsigned readyMask; bool producerValid; - bool scheduleOwner; + bool phaseValid; uint64_t captureTime; uint64_t postProcessTime; @@ -330,9 +331,8 @@ static void frameTimingCancel(LG_RendererFrameToken token) } static void frameTimingQueue(LG_RendererFrameToken token, uint64_t frameSerial, - uint32_t scheduleGeneration, uint32_t scheduleEpoch, bool scheduleOwner, - uint64_t importTime, uint64_t importWaitTime, uint64_t dispatchStart, - uint64_t queueStart) + const LG_TransportFrameTiming * timing, uint64_t importTime, + uint64_t importWaitTime, uint64_t dispatchStart, uint64_t queueStart) { INTERLOCKED_SECTION(l_frameTiming.lock, { struct FrameTimingRecord * record = frameTimingRecord(token); @@ -341,14 +341,16 @@ static void frameTimingQueue(LG_RendererFrameToken token, uint64_t frameSerial, const uint64_t elapsed = queueStart > dispatchStart ? queueStart - dispatchStart : 0; const uint64_t accounted = importTime + importWaitTime; - record->importTime = importTime; - record->importWaitTime = importWaitTime; - record->dispatchTime = elapsed > accounted ? elapsed - accounted : 0; - record->queueStart = queueStart; - record->frameSerial = frameSerial; - record->scheduleGeneration = scheduleGeneration; - record->scheduleEpoch = scheduleEpoch; - record->scheduleOwner = scheduleOwner; + record->importTime = importTime; + record->importWaitTime = importWaitTime; + record->dispatchTime = elapsed > accounted ? + elapsed - accounted : 0; + record->queueStart = queueStart; + record->frameSerial = frameSerial; + record->scheduleGeneration = timing->scheduleGeneration; + record->scheduleEpoch = timing->scheduleEpoch; + record->scheduleDeadlineSerial = timing->scheduleDeadlineSerial; + record->phaseValid = timing->phaseValid; if (record->timestamp < queueStart) record->timestamp = queueStart; } @@ -398,7 +400,8 @@ static void frameTimingFinishRender(const LG_RendererFrameTiming * timing, uint64_t feedbackQueueStart = 0; uint32_t feedbackGeneration = 0; uint32_t feedbackEpoch = 0; - bool feedbackOwner = false; + uint32_t feedbackDeadline = 0; + bool feedbackValid = false; LG_LOCK(l_frameTiming.lock); if (l_frameTiming.retireToken <= timing->frameToken) @@ -421,7 +424,8 @@ static void frameTimingFinishRender(const LG_RendererFrameTiming * timing, feedbackFrameSerial = record->frameSerial; feedbackGeneration = record->scheduleGeneration; feedbackEpoch = record->scheduleEpoch; - feedbackOwner = record->scheduleOwner; + feedbackDeadline = record->scheduleDeadlineSerial; + feedbackValid = record->phaseValid; feedbackQueueStart = record->queueStart; if (unlikely( @@ -445,13 +449,13 @@ static void frameTimingFinishRender(const LG_RendererFrameTiming * timing, } LG_UNLOCK(l_frameTiming.lock); - if (g_state.jitRender && feedbackOwner && feedbackFrameSerial && - feedbackGeneration && feedbackEpoch && feedbackQueueStart && - prepareStart >= feedbackQueueStart) + if (g_state.jitRender && feedbackValid && feedbackFrameSerial && + feedbackGeneration && feedbackEpoch && feedbackDeadline && + feedbackQueueStart && prepareStart >= feedbackQueueStart) { frameScheduler_feedback( feedbackFrameSerial, feedbackGeneration, feedbackEpoch, - prepareStart - feedbackQueueStart); + feedbackDeadline, prepareStart - feedbackQueueStart); } } @@ -1143,9 +1147,9 @@ int main_frameThread(void * unused) atomic_store_explicit(&l_testFrameSerial, frame.serial, memory_order_release); #endif - frameTimingQueue(frameToken, frame.serial, frame.scheduleGeneration, - frame.scheduleEpoch, frame.scheduleOwner, g_state.frameImportTime, - g_state.frameImportWaitTime, dispatchStart, queueStart); + frameTimingQueue(frameToken, frame.serial, &timing, + g_state.frameImportTime, g_state.frameImportWaitTime, + dispatchStart, queueStart); if (g_state.jitRender) { diff --git a/client/transports/LGMP/lgmp.c b/client/transports/LGMP/lgmp.c index 03455136..1de5ba5a 100644 --- a/client/transports/LGMP/lgmp.c +++ b/client/transports/LGMP/lgmp.c @@ -721,6 +721,18 @@ static bool lgmp_frameSerialNewer(uint32_t lhs, uint32_t rhs) (uint32_t)(lhs - rhs) < UINT32_C(0x80000000); } +static bool lgmp_frameMessageHasMatchingDeadline( + const struct LGMPFrameMessage * message) +{ + if (!message->owner) + return false; + + const uint32_t epoch = (uint32_t)(message->message.udata >> 32); + const uint32_t deadlineSerial = (uint32_t)message->message.udata; + return deadlineSerial && epoch == message->frame->scheduleEpoch && + deadlineSerial == message->frame->scheduleDeadlineSerial; +} + static void lgmp_selectNewestFrameMessage( struct LGMPFrameMessage * candidate, struct LGMPFrameMessage ** selected) @@ -732,7 +744,9 @@ static void lgmp_selectNewestFrameMessage( lgmp_frameSerialNewer(candidate->frame->frameSerial, (*selected)->frame->frameSerial) || (candidate->frame->frameSerial == (*selected)->frame->frameSerial && - candidate->owner && !(*selected)->owner)) + ((candidate->owner && !(*selected)->owner) || + (lgmp_frameMessageHasMatchingDeadline(candidate) && + !lgmp_frameMessageHasMatchingDeadline(*selected))))) *selected = candidate; } @@ -917,9 +931,9 @@ static LG_TransportStatus lgmp_nextFrameLocked(LG_Transport * this, if (selected->owner) { const uint64_t scheduleToken = selected->message.udata; - result->scheduleGeneration = (uint32_t)(scheduleToken >> 32); - result->scheduleEpoch = (uint32_t)scheduleToken; - result->scheduleOwner = true; + result->scheduleEpoch = (uint32_t)(scheduleToken >> 32); + result->scheduleDeadlineSerial = (uint32_t)scheduleToken; + result->scheduleOwner = true; } if (frame->flags & FRAME_FLAG_BLOCK_SCREENSAVER) result->flags |= LG_TRANSPORT_FRAME_BLOCK_SCREENSAVER; @@ -1057,11 +1071,20 @@ static void lgmp_getFrameTiming(LG_Transport * this, return; } - timing->valid = true; - timing->captureTime = lease->frame->captureTime; - timing->postProcessTime = lease->frame->postProcessTime; - timing->copyTime = lease->frame->copyTime; - timing->readyTime = lease->frame->readyTime; + timing->valid = true; + timing->scheduleGeneration = lease->frame->scheduleGeneration; + timing->scheduleEpoch = lease->frame->scheduleEpoch; + timing->scheduleDeadlineSerial = lease->frame->scheduleDeadlineSerial; + timing->phaseValid = frame->scheduleOwner && + timing->scheduleGeneration && timing->scheduleEpoch && + timing->scheduleDeadlineSerial && + timing->scheduleEpoch == frame->scheduleEpoch && + timing->scheduleDeadlineSerial == frame->scheduleDeadlineSerial && + (lease->frame->timingFlags & KVMFR_FRAME_TIMING_PHASE_VALID); + timing->captureTime = lease->frame->captureTime; + timing->postProcessTime = lease->frame->postProcessTime; + timing->copyTime = lease->frame->copyTime; + timing->readyTime = lease->frame->readyTime; LG_UNLOCK(this->frameLock); } @@ -1244,17 +1267,19 @@ static LG_TransportStatus lgmp_sendControl(LG_Transport * this, case LG_TRANSPORT_CONTROL_FRAME_SCHEDULE: { const KVMFRFrameSchedule message = { - .msg.type = KVMFR_MESSAGE_FRAME_SCHEDULE, - .clientID = this->clientID, - .generation = control->frameSchedule.generation, - .flags = control->frameSchedule.flags, - .period = control->frameSchedule.period, - .targetSlack = control->frameSchedule.targetSlack, - .phaseError = control->frameSchedule.phaseError, - .feedbackFrameSerial = control->frameSchedule.feedbackFrameSerial, - .feedbackScheduleEpoch = + .msg.type = KVMFR_MESSAGE_FRAME_SCHEDULE, + .clientID = this->clientID, + .generation = control->frameSchedule.generation, + .flags = control->frameSchedule.flags, + .period = control->frameSchedule.period, + .targetSlack = control->frameSchedule.targetSlack, + .phaseError = control->frameSchedule.phaseError, + .feedbackFrameSerial = control->frameSchedule.feedbackFrameSerial, + .feedbackScheduleEpoch = control->frameSchedule.feedbackScheduleEpoch, - .lease = control->frameSchedule.lease, + .feedbackDeadlineSerial = + control->frameSchedule.feedbackDeadlineSerial, + .lease = control->frameSchedule.lease, }; memcpy(buffer, &message, sizeof(message)); size = sizeof(message); diff --git a/common/include/common/KVMFR.h b/common/include/common/KVMFR.h index 8e3ad0a5..ca4f02ae 100644 --- a/common/include/common/KVMFR.h +++ b/common/include/common/KVMFR.h @@ -30,7 +30,7 @@ #include "LGMPConfig.h" #define KVMFR_MAGIC "KVMFR---" -#define KVMFR_VERSION 27 +#define KVMFR_VERSION 28 // Fallback used by producers that cannot report the source display's SDR // white level. IDD frames override this with IDDCX_METADATA2::SdrWhiteLevel. @@ -166,6 +166,13 @@ enum typedef uint32_t KVMFRFrameFlags; +enum +{ + KVMFR_FRAME_TIMING_PHASE_VALID = 0x1 +}; + +typedef uint32_t KVMFRFrameTimingFlags; + typedef struct KVMFRFrame { /* @@ -211,9 +218,16 @@ typedef struct KVMFRFrame uint32_t timingSerial; uint32_t timingValid; + // Producer cadence identity. Phase feedback is permitted only when the + // matching frame has KVMFR_FRAME_TIMING_PHASE_VALID set. + KVMFRFrameTimingFlags timingFlags; + uint32_t scheduleGeneration; + uint32_t scheduleEpoch; + uint32_t scheduleDeadlineSerial; + // Keep the conditional HDR block and damage rectangles on separate cache // lines from the producer timing fields. - uint8_t timingReserved[24]; + uint8_t timingReserved[8]; // HDR static metadata (valid when FRAME_FLAG_HDR_METADATA is set) // Display color primaries in 0.00002 units (SMPTE ST 2086 format) @@ -237,6 +251,10 @@ KVMFRFrame; #if defined(__cplusplus) static_assert(offsetof(KVMFRFrame, captureTime) == 64, "KVMFRFrame hot fields must fit in one cache line"); +static_assert(offsetof(KVMFRFrame, timingFlags) == 104, + "KVMFRFrame timing flags layout changed"); +static_assert(offsetof(KVMFRFrame, scheduleDeadlineSerial) == 116, + "KVMFRFrame schedule identity layout changed"); static_assert(offsetof(KVMFRFrame, hdrDisplayPrimary) == 128, "KVMFRFrame HDR metadata must be cache-line aligned"); static_assert(offsetof(KVMFRFrame, damageRects) == 192, @@ -244,6 +262,10 @@ static_assert(offsetof(KVMFRFrame, damageRects) == 192, #elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L _Static_assert(offsetof(KVMFRFrame, captureTime) == 64, "KVMFRFrame hot fields must fit in one cache line"); +_Static_assert(offsetof(KVMFRFrame, timingFlags) == 104, + "KVMFRFrame timing flags layout changed"); +_Static_assert(offsetof(KVMFRFrame, scheduleDeadlineSerial) == 116, + "KVMFRFrame schedule identity layout changed"); _Static_assert(offsetof(KVMFRFrame, hdrDisplayPrimary) == 128, "KVMFRFrame HDR metadata must be cache-line aligned"); _Static_assert(offsetof(KVMFRFrame, damageRects) == 192, @@ -297,15 +319,25 @@ typedef struct KVMFRFrameSchedule int64_t phaseError; // ready-to-render phase error (ns) uint32_t feedbackFrameSerial; uint32_t feedbackScheduleEpoch; + // Identifies the exact producer deadline paired with feedbackFrameSerial. + uint32_t feedbackDeadlineSerial; uint32_t lease; // lease duration (ms) - uint8_t reserved[12]; + uint8_t reserved[8]; } KVMFRFrameSchedule; #if defined(__cplusplus) +static_assert(offsetof(KVMFRFrameSchedule, feedbackDeadlineSerial) == 48, + "KVMFR frame schedule feedback identity layout changed"); +static_assert(offsetof(KVMFRFrameSchedule, lease) == 52, + "KVMFR frame schedule lease layout changed"); static_assert(sizeof(KVMFRFrameSchedule) == 64, "KVMFR frame schedule must fit in one LGMP control message"); #elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +_Static_assert(offsetof(KVMFRFrameSchedule, feedbackDeadlineSerial) == 48, + "KVMFR frame schedule feedback identity layout changed"); +_Static_assert(offsetof(KVMFRFrameSchedule, lease) == 52, + "KVMFR frame schedule lease layout changed"); _Static_assert(sizeof(KVMFRFrameSchedule) == 64, "KVMFR frame schedule must fit in one LGMP control message"); #endif diff --git a/host/src/app.c b/host/src/app.c index a14bf953..359db0e6 100644 --- a/host/src/app.c +++ b/host/src/app.c @@ -398,11 +398,16 @@ static bool sendFrame(CaptureResult result, bool * restart) // fi->offset is initialized at startup fi->flags = flags; fi->sdrWhiteLevel = sdrWhiteLevel; - fi->captureTime = frame.captureTime; - fi->postProcessTime = 0; - fi->copyTime = 0; - fi->readyTime = 0; - fi->timingSerial = 0; + + fi->captureTime = frame.captureTime; + fi->postProcessTime = 0; + fi->copyTime = 0; + fi->readyTime = 0; + fi->timingSerial = 0; + fi->timingFlags = 0; + fi->scheduleGeneration = 0; + fi->scheduleEpoch = 0; + fi->scheduleDeadlineSerial = 0; __atomic_store_n(&fi->timingValid, 0, __ATOMIC_RELAXED); if (frame.hdrMetadata) { diff --git a/idd/LGIdd/CFrameBufferResource.h b/idd/LGIdd/CFrameBufferResource.h index bf5678d5..7b6437e0 100644 --- a/idd/LGIdd/CFrameBufferResource.h +++ b/idd/LGIdd/CFrameBufferResource.h @@ -26,6 +26,8 @@ #include #include +#include "CFrameScheduler.h" + class CSwapChainProcessor; using namespace Microsoft::WRL; @@ -33,19 +35,22 @@ using namespace Microsoft::WRL; class CFrameBufferResource { private: - unsigned m_frameIndex = 0; - uint8_t * m_base = nullptr; - size_t m_size = 0; - size_t m_frameSize = 0; - uint64_t m_captureTime = 0; - uint64_t m_postProcessStart = 0; - uint64_t m_copyStart = 0; - unsigned m_timingEffectIndex = 0; - uint64_t m_timingToken = 0; - bool m_fullCopy = false; - unsigned m_candidateIndex = 0; - ComPtr m_res; - void * m_map = nullptr; + unsigned m_frameIndex = 0; + uint8_t * m_base = nullptr; + size_t m_size = 0; + size_t m_frameSize = 0; + uint64_t m_captureTime = 0; + uint64_t m_postProcessStart = 0; + uint64_t m_copyStart = 0; + + CFrameScheduler::Schedule m_schedule = {}; + + unsigned m_timingEffectIndex = 0; + uint64_t m_timingToken = 0; + bool m_fullCopy = false; + unsigned m_candidateIndex = 0; + ComPtr m_res; + void * m_map = nullptr; public: bool Init(CSwapChainProcessor * swapChain, unsigned frameIndex, uint8_t * base, size_t size); @@ -65,6 +70,14 @@ class CFrameBufferResource uint64_t GetCaptureTime () const { return m_captureTime; } uint64_t GetPostProcessStart() const { return m_postProcessStart; } uint64_t GetCopyStart () const { return m_copyStart; } + void SetSchedule(const CFrameScheduler::Schedule& schedule) + { + m_schedule = schedule; + } + const CFrameScheduler::Schedule& GetSchedule() const + { + return m_schedule; + } void SetPostProcessSample( unsigned effectIndex, uint64_t token, bool fullCopy) diff --git a/idd/LGIdd/CFrameScheduler.cpp b/idd/LGIdd/CFrameScheduler.cpp index 32fbe406..15d5839c 100644 --- a/idd/LGIdd/CFrameScheduler.cpp +++ b/idd/LGIdd/CFrameScheduler.cpp @@ -22,6 +22,8 @@ #include "CDebug.h" +#include + static const uint64_t MIN_SOURCE_PERIOD_NS = 100000ULL; static const uint64_t MIN_SCHEDULE_PERIOD_NS = 2000000ULL; static const uint64_t MAX_PERIOD_NS = 1000000000ULL; @@ -82,7 +84,21 @@ CFrameScheduler::Client * CFrameScheduler::FindClient(uint32_t clientID) return nullptr; } -bool CFrameScheduler::ElectOwner(uint64_t now) +CFrameScheduler::Publication * CFrameScheduler::FindPublication( + const Schedule& schedule, uint32_t frameSerial) +{ + for (Publication& publication : m_publications) + if (publication.generation == schedule.generation && + publication.epoch == schedule.epoch && + publication.deadlineSerial == schedule.deadlineSerial && + publication.frameSerial == frameSerial && + publication.deadline == schedule.deadline) + return &publication; + + return nullptr; +} + +bool CFrameScheduler::ElectOwner(uint64_t now, uint32_t resetClientID) { Client * fastest = nullptr; Client * incumbent = FindClient(m_schedule.clientID); @@ -122,7 +138,8 @@ bool CFrameScheduler::ElectOwner(uint64_t now) const uint32_t oldEpoch = m_schedule.epoch; const uint64_t oldPeriod = m_schedule.period; const uint64_t oldSlack = m_schedule.targetSlack; - if (incumbent && incumbent->active && + if (incumbent && incumbent->clientID != resetClientID && + incumbent->active && incumbent->generation == oldGeneration) incumbent->nextDelivery = m_nextDeadline; @@ -142,7 +159,11 @@ bool CFrameScheduler::ElectOwner(uint64_t now) } const bool ownerChanged = oldClientID != m_schedule.clientID; - if (ownerChanged || oldGeneration != m_schedule.generation) + const bool ownerReset = resetClientID && + resetClientID == m_schedule.clientID; + const bool identityChanged = ownerChanged || ownerReset || + oldGeneration != m_schedule.generation; + if (identityChanged) { if (m_scheduling) { @@ -153,6 +174,8 @@ bool CFrameScheduler::ElectOwner(uint64_t now) m_nextDeadline = m_scheduling ? fastest->nextDelivery : 0; if (m_scheduling && !m_nextDeadline) m_nextDeadline = now + m_schedule.period; + m_deadlineSerial = m_scheduling ? 1 : 0; + m_pendingCorrection = 0; if (m_scheduling) { ++m_forceRequestTicket; @@ -165,9 +188,10 @@ bool CFrameScheduler::ElectOwner(uint64_t now) } if (fastest) - fastest->lastFeedbackFrameSerial = 0; + fastest->lastFeedbackDeadlineSerial = 0; - m_lastPublishedFrameSerial = 0; + memset(m_publications, 0, sizeof(m_publications)); + m_publicationIndex = 0; m_lastPhaseError = 0; m_lastLog = now; m_lastLogAcquired = m_acquiredFrames; @@ -182,7 +206,7 @@ bool CFrameScheduler::ElectOwner(uint64_t now) DEBUG_INFO("Frame timing owner released; using push delivery"); } - return ownerChanged || oldGeneration != m_schedule.generation || + return identityChanged || oldPeriod != m_schedule.period || oldSlack != m_schedule.targetSlack || clientExpired; } @@ -199,12 +223,15 @@ void CFrameScheduler::Reset() m_forceAckTicket = m_forceRequestTicket; m_republishAckTicket = m_republishRequestTicket; - m_lastArrival = 0; - m_guestPeriod = 0; - m_workEstimate = 0; - m_nextDeadline = 0; + m_lastArrival = 0; + m_guestPeriod = 0; + m_workEstimate = 0; + m_nextDeadline = 0; + m_deadlineSerial = 0; + m_pendingCorrection = 0; - m_lastPublishedFrameSerial = 0; + memset(m_publications, 0, sizeof(m_publications)); + m_publicationIndex = 0; m_lastPhaseError = 0; m_acquiredFrames = 0; @@ -351,16 +378,18 @@ bool CFrameScheduler::UpdateSchedule(uint32_t sourceClientID, return false; } + const bool explicitReset = + (schedule.flags & KVMFR_FRAME_SCHEDULE_RESET) != 0; const bool reset = client->generation != schedule.generation || - (schedule.flags & KVMFR_FRAME_SCHEDULE_RESET); + explicitReset; bool wake = reset || !client->active || client->period != schedule.period || client->targetSlack != schedule.targetSlack; if (reset) { - client->lastFeedbackFrameSerial = 0; - client->immediate = false; - client->nextDelivery = now + schedule.period; + client->lastFeedbackDeadlineSerial = 0; + client->immediate = false; + client->nextDelivery = now + schedule.period; } client->generation = schedule.generation; client->period = schedule.period; @@ -372,7 +401,7 @@ bool CFrameScheduler::UpdateSchedule(uint32_t sourceClientID, client->immediate = true; wake = true; } - wake |= ElectOwner(now); + wake |= ElectOwner(now, explicitReset ? schedule.clientID : 0); if (m_scheduling && client->clientID == m_schedule.clientID && client->generation == m_schedule.generation && (schedule.flags & KVMFR_FRAME_SCHEDULE_IMMEDIATE)) @@ -395,14 +424,30 @@ bool CFrameScheduler::ApplyFeedback(Client& client, schedule.generation != m_schedule.generation || schedule.feedbackScheduleEpoch != m_schedule.epoch || !schedule.feedbackFrameSerial || - (client.lastFeedbackFrameSerial && - static_cast(schedule.feedbackFrameSerial - - client.lastFeedbackFrameSerial) <= 0) || - !m_lastPublishedFrameSerial || - static_cast(schedule.feedbackFrameSerial - - m_lastPublishedFrameSerial) > 0) + !schedule.feedbackDeadlineSerial || + (client.lastFeedbackDeadlineSerial && + static_cast(schedule.feedbackDeadlineSerial - + client.lastFeedbackDeadlineSerial) <= 0)) return false; + Publication * publication = nullptr; + for (Publication& candidate : m_publications) + if (candidate.generation == schedule.generation && + candidate.epoch == schedule.feedbackScheduleEpoch && + candidate.deadlineSerial == schedule.feedbackDeadlineSerial && + candidate.frameSerial == schedule.feedbackFrameSerial) + { + publication = &candidate; + break; + } + + if (!publication || !publication->committed || + !publication->completed || !publication->phaseValid || + publication->accepted) + return false; + + publication->accepted = true; + int64_t correction = schedule.phaseError / 4; const int64_t limit = static_cast(m_schedule.period / 4); if (correction > limit) @@ -410,18 +455,50 @@ bool CFrameScheduler::ApplyFeedback(Client& client, else if (correction < -limit) correction = -limit; - if (correction >= 0) - m_nextDeadline += static_cast(correction); + m_pendingCorrection += correction; + m_lastPhaseError = schedule.phaseError; + client.lastFeedbackDeadlineSerial = schedule.feedbackDeadlineSerial; + return false; +} + +void CFrameScheduler::AdvanceCurrentDeadline() +{ + m_nextDeadline += m_schedule.period; + if (m_pendingCorrection >= 0) + m_nextDeadline += static_cast(m_pendingCorrection); else { - const uint64_t advance = static_cast(-correction); - m_nextDeadline = m_nextDeadline > advance ? - m_nextDeadline - advance : 0; + const uint64_t correction = + static_cast(-m_pendingCorrection); + m_nextDeadline = m_nextDeadline > correction ? + m_nextDeadline - correction : 0; } - m_lastPhaseError = schedule.phaseError; - client.lastFeedbackFrameSerial = schedule.feedbackFrameSerial; - client.nextDelivery = m_nextDeadline; - return correction != 0; + m_pendingCorrection = 0; + AdvanceDeadlineSerial(1); +} + +void CFrameScheduler::AdvanceDeadlineSerial(uint64_t count) +{ + if (!count || !m_deadlineSerial) + return; + + const uint64_t position = + static_cast(m_deadlineSerial - 1) + count; + uint64_t epochAdvances = position / UINT32_MAX; + m_deadlineSerial = static_cast(position % UINT32_MAX) + 1; + if (!epochAdvances) + return; + + for (; epochAdvances; --epochAdvances) + if (!++m_epoch) + ++m_epoch; + m_schedule.epoch = m_epoch; + + Client * client = FindClient(m_schedule.clientID); + if (client) + client->lastFeedbackDeadlineSerial = 0; + memset(m_publications, 0, sizeof(m_publications)); + m_publicationIndex = 0; } void CFrameScheduler::AdvanceDeadline(uint64_t now) @@ -435,6 +512,7 @@ void CFrameScheduler::AdvanceDeadline(uint64_t now) const uint64_t periods = (target - m_nextDeadline) / m_schedule.period + 1; m_nextDeadline += periods * m_schedule.period; + AdvanceDeadlineSerial(periods); } void CFrameScheduler::AdvanceDelivery(Client& client, uint64_t now) @@ -520,6 +598,9 @@ bool CFrameScheduler::GetPublishTarget(uint64_t now, uint64_t& target, schedule.republishTicket = m_republishRequestTicket; republish = schedule.republishTicket != m_republishAckTicket; + schedule.deadline = m_nextDeadline; + schedule.deadlineSerial = m_deadlineSerial; + const uint64_t lead = PublicationLead(m_schedule.targetSlack, m_workEstimate); const uint64_t periodicTarget = m_nextDeadline > lead ? @@ -528,22 +609,45 @@ bool CFrameScheduler::GetPublishTarget(uint64_t now, uint64_t& target, if (schedule.forceTicket != m_forceAckTicket) { periodic = periodicTarget <= now; + if (periodic) + target = periodicTarget; + schedule.deliveryDeadlineSerial = periodic ? m_deadlineSerial : 0; + schedule.phaseEligible = periodic; ReleaseSRWLockExclusive(&m_lock); return true; } - periodic = true; - if (periodicTarget <= now) - { - ReleaseSRWLockExclusive(&m_lock); - return true; - } - - target = periodicTarget; + periodic = true; + schedule.deliveryDeadlineSerial = m_deadlineSerial; + schedule.phaseEligible = true; + target = periodicTarget; ReleaseSRWLockExclusive(&m_lock); return true; } +void CFrameScheduler::FrameMissed(const Schedule& schedule, uint64_t now, + bool periodic) +{ + if (!periodic) + return; + + AcquireSRWLockExclusive(&m_lock); + if (m_scheduling && schedule.clientID == m_schedule.clientID && + schedule.generation == m_schedule.generation && + schedule.epoch == m_schedule.epoch && + schedule.deadlineSerial == m_deadlineSerial && + schedule.deadline == m_nextDeadline) + { + AdvanceCurrentDeadline(); + AdvanceDeadline(now); + + Client * client = FindClient(m_schedule.clientID); + if (client) + client->nextDelivery = m_nextDeadline; + } + ReleaseSRWLockExclusive(&m_lock); +} + void CFrameScheduler::FrameSuperseded() { AcquireSRWLockExclusive(&m_lock); @@ -551,14 +655,52 @@ void CFrameScheduler::FrameSuperseded() ReleaseSRWLockExclusive(&m_lock); } +bool CFrameScheduler::TryFrameSubmitted(const Schedule& schedule, + uint32_t frameSerial) +{ + if (!schedule.phaseEligible || !schedule.deliveryDeadlineSerial || + schedule.deliveryDeadlineSerial != schedule.deadlineSerial) + return false; + + if (!TryAcquireSRWLockExclusive(&m_lock)) + return false; + + bool registered = false; + if (m_scheduling && schedule.clientID == m_schedule.clientID && + schedule.generation == m_schedule.generation && + schedule.epoch == m_schedule.epoch && + schedule.deadlineSerial == m_deadlineSerial && + schedule.deadline == m_nextDeadline) + { + Publication& publication = + m_publications[m_publicationIndex++ % PUBLICATION_HISTORY_SIZE]; + publication = {}; + publication.generation = schedule.generation; + publication.epoch = schedule.epoch; + publication.deadlineSerial = schedule.deadlineSerial; + publication.frameSerial = frameSerial; + publication.deadline = schedule.deadline; + publication.committed = true; + registered = true; + } + ReleaseSRWLockExclusive(&m_lock); + return registered; +} + void CFrameScheduler::FramePublished(const Schedule& schedule, uint32_t frameSerial, uint64_t now, bool periodic) { AcquireSRWLockExclusive(&m_lock); if (m_scheduling && schedule.clientID == m_schedule.clientID && schedule.generation == m_schedule.generation && - schedule.epoch == m_schedule.epoch) + schedule.epoch == m_schedule.epoch && + schedule.deadlineSerial == m_deadlineSerial && + schedule.deadline == m_nextDeadline) { + Publication * publication = FindPublication(schedule, frameSerial); + if (publication) + publication->committed = true; + if (schedule.forceTicket > m_forceAckTicket) m_forceAckTicket = schedule.forceTicket; if (schedule.republishTicket > m_republishAckTicket) @@ -572,11 +714,10 @@ void CFrameScheduler::FramePublished(const Schedule& schedule, client->lastDeliveredFrameSerial = frameSerial; client->deliveredFrameValid = true; } - m_lastPublishedFrameSerial = frameSerial; ++m_publishedFrames; if (periodic) { - m_nextDeadline += m_schedule.period; + AdvanceCurrentDeadline(); AdvanceDeadline(now); } if (client) @@ -585,6 +726,34 @@ void CFrameScheduler::FramePublished(const Schedule& schedule, ReleaseSRWLockExclusive(&m_lock); } +bool CFrameScheduler::TryFrameCompleted(const Schedule& schedule, + uint32_t frameSerial, uint64_t completedAt) +{ + if (!schedule.phaseEligible || !schedule.deliveryDeadlineSerial || + schedule.deliveryDeadlineSerial != schedule.deadlineSerial || + !schedule.deadline) + return false; + + if (!TryAcquireSRWLockExclusive(&m_lock)) + return false; + + bool phaseValid = false; + if (m_scheduling && schedule.clientID == m_schedule.clientID && + schedule.generation == m_schedule.generation && + schedule.epoch == m_schedule.epoch) + { + Publication * publication = FindPublication(schedule, frameSerial); + if (publication && publication->committed) + { + publication->completed = true; + publication->phaseValid = completedAt <= publication->deadline; + phaseValid = publication->phaseValid; + } + } + ReleaseSRWLockExclusive(&m_lock); + return phaseValid; +} + void CFrameScheduler::FrameRepublished(const Schedule& schedule, uint32_t frameSerial) { @@ -605,7 +774,6 @@ void CFrameScheduler::FrameRepublished(const Schedule& schedule, client->deliveredFrameValid = true; client->nextDelivery = m_nextDeadline; } - m_lastPublishedFrameSerial = frameSerial; ++m_publishedFrames; } ReleaseSRWLockExclusive(&m_lock); @@ -765,12 +933,14 @@ void CFrameScheduler::LogStatistics(uint64_t now) ReleaseSRWLockExclusive(&m_lock); } -void CFrameScheduler::RecordFrameTiming(uint64_t duration) +void CFrameScheduler::TryRecordFrameTiming(uint64_t duration) { if (!duration) return; - AcquireSRWLockExclusive(&m_lock); + if (!TryAcquireSRWLockExclusive(&m_lock)) + return; + if (!m_workEstimate || duration > m_workEstimate) m_workEstimate = duration; else diff --git a/idd/LGIdd/CFrameScheduler.h b/idd/LGIdd/CFrameScheduler.h index 826b9919..22458089 100644 --- a/idd/LGIdd/CFrameScheduler.h +++ b/idd/LGIdd/CFrameScheduler.h @@ -39,8 +39,12 @@ public: uint32_t epoch; uint64_t period; uint64_t targetSlack; + uint64_t deadline; uint64_t forceTicket; uint64_t republishTicket; + uint32_t deadlineSerial; + uint32_t deliveryDeadlineSerial; + bool phaseEligible; }; private: @@ -52,7 +56,7 @@ private: uint64_t targetSlack; uint64_t expiry; uint64_t nextDelivery; - uint32_t lastFeedbackFrameSerial; + uint32_t lastFeedbackDeadlineSerial; uint32_t lastDeliveredFrameSerial; bool subscribed; bool ownerCapable; @@ -62,6 +66,21 @@ private: bool deliveredFrameValid; }; + struct Publication + { + uint32_t generation; + uint32_t epoch; + uint32_t deadlineSerial; + uint32_t frameSerial; + uint64_t deadline; + bool committed; + bool completed; + bool phaseValid; + bool accepted; + }; + + static const unsigned PUBLICATION_HISTORY_SIZE = 128; + mutable SRWLOCK m_lock = SRWLOCK_INIT; HANDLE m_wakeEvent = nullptr; Client m_clients[LGMP_MAX_CLIENTS] = {}; @@ -75,12 +94,15 @@ private: uint64_t m_republishRequestTicket = 0; uint64_t m_republishAckTicket = 0; - uint64_t m_lastArrival = 0; - uint64_t m_guestPeriod = 0; - uint64_t m_workEstimate = 0; - uint64_t m_nextDeadline = 0; + uint64_t m_lastArrival = 0; + uint64_t m_guestPeriod = 0; + uint64_t m_workEstimate = 0; + uint64_t m_nextDeadline = 0; + uint32_t m_deadlineSerial = 0; + int64_t m_pendingCorrection = 0; - uint32_t m_lastPublishedFrameSerial = 0; + Publication m_publications[PUBLICATION_HISTORY_SIZE] = {}; + unsigned m_publicationIndex = 0; int64_t m_lastPhaseError = 0; uint64_t m_acquiredFrames = 0; @@ -92,8 +114,12 @@ private: uint64_t m_lastLogPublished = 0; Client * FindClient(uint32_t clientID); - bool ElectOwner(uint64_t now); + Publication * FindPublication(const Schedule& schedule, + uint32_t frameSerial); + bool ElectOwner(uint64_t now, uint32_t resetClientID = 0); bool ApplyFeedback(Client& client, const KVMFRFrameSchedule& schedule); + void AdvanceCurrentDeadline(); + void AdvanceDeadlineSerial(uint64_t count); void AdvanceDeadline(uint64_t now); static void AdvanceDelivery(Client& client, uint64_t now); void WakePublisher() const; @@ -115,10 +141,14 @@ public: void ForceFrame(); bool GetPublishTarget(uint64_t now, uint64_t& target, Schedule& schedule, bool& periodic, bool& republish); + void FrameMissed(const Schedule& schedule, uint64_t now, bool periodic); void FrameSuperseded(); + bool TryFrameSubmitted(const Schedule& schedule, uint32_t frameSerial); void FramePublished(const Schedule& schedule, uint32_t frameSerial, uint64_t now, bool periodic); void FrameRepublished(const Schedule& schedule, uint32_t frameSerial); + bool TryFrameCompleted(const Schedule& schedule, uint32_t frameSerial, + uint64_t completedAt); unsigned GetSecondaryRecipients(const uint32_t * clientIDs, unsigned count, uint32_t frameSerial, uint64_t now, uint32_t * recipients) const; @@ -128,6 +158,6 @@ public: void FrameDelivered(const uint32_t * clientIDs, unsigned count, uint32_t frameSerial, uint64_t now); void NotifyPublisher() const { WakePublisher(); } - void RecordFrameTiming(uint64_t duration); + void TryRecordFrameTiming(uint64_t duration); void LogStatistics(uint64_t now); }; diff --git a/idd/LGIdd/CIndirectDeviceContext.cpp b/idd/LGIdd/CIndirectDeviceContext.cpp index 54e16151..107d9bb5 100644 --- a/idd/LGIdd/CIndirectDeviceContext.cpp +++ b/idd/LGIdd/CIndirectDeviceContext.cpp @@ -47,8 +47,8 @@ static const struct LGMPQueueConfig POINTER_QUEUE_CONFIG = static uint64_t FrameScheduleToken( const CFrameScheduler::Schedule& schedule) { - return static_cast(schedule.generation) << 32 | - schedule.epoch; + return static_cast(schedule.epoch) << 32 | + schedule.deliveryDeadlineSerial; } static bool FrameScheduleMatches( @@ -1884,11 +1884,16 @@ CIndirectDeviceContext::PreparedFrameBuffer CIndirectDeviceContext::PrepareFrame // fi->offset is initialized at startup fi->flags = flags; fi->sdrWhiteLevel = dstFormat.sdrWhiteLevel; - fi->captureTime = 0; - fi->postProcessTime = 0; - fi->copyTime = 0; - fi->readyTime = 0; - fi->timingSerial = 0; + + fi->captureTime = 0; + fi->postProcessTime = 0; + fi->copyTime = 0; + fi->readyTime = 0; + fi->timingSerial = 0; + fi->timingFlags = 0; + fi->scheduleGeneration = 0; + fi->scheduleEpoch = 0; + fi->scheduleDeadlineSerial = 0; InterlockedExchange((volatile LONG *)&fi->timingValid, 0); fi->rotation = FRAME_ROT_0; fi->type = dstFormat.format; @@ -1953,6 +1958,12 @@ bool CIndirectDeviceContext::PublishFrameBuffer(unsigned frameIndex, return false; } + KVMFRFrame * frame = m_frame[frameIndex]; + frame->timingFlags = 0; + frame->scheduleGeneration = schedule.generation; + frame->scheduleEpoch = schedule.epoch; + frame->scheduleDeadlineSerial = schedule.deliveryDeadlineSerial; + LGMP_STATUS status = LGMP_OK; bool published = false; if (schedule.clientID) @@ -2036,7 +2047,10 @@ bool CIndirectDeviceContext::RepublishFrameBuffer( return false; } - const uint64_t scheduleToken = FrameScheduleToken(schedule); + CFrameScheduler::Schedule deliverySchedule = schedule; + deliverySchedule.deliveryDeadlineSerial = 0; + deliverySchedule.phaseEligible = false; + const uint64_t scheduleToken = FrameScheduleToken(deliverySchedule); const uint32_t frameSerial = m_frame[frameIndex]->frameSerial; if (HasMatchingOwnerDelivery(schedule.clientID, static_cast(frameIndex), scheduleToken)) @@ -2057,7 +2071,7 @@ bool CIndirectDeviceContext::RepublishFrameBuffer( if (ownerQueueIndex < 0) { const bool published = PostSharedOwnerFrame( - static_cast(frameIndex), schedule); + static_cast(frameIndex), deliverySchedule); ReleaseSRWLockExclusive(&m_framePublishLock); if (published) m_frameScheduler.FrameRepublished(schedule, frameSerial); @@ -2106,6 +2120,16 @@ void CIndirectDeviceContext::CommitFrameBuffer(unsigned frameIndex, CFrameScheduler::Nanotime(), periodic); } +bool CIndirectDeviceContext::TryFrameSubmitted(unsigned frameIndex, + const CFrameScheduler::Schedule& schedule) +{ + if (frameIndex >= LGMP_Q_FRAME_BUFFER_LEN) + return false; + + return m_frameScheduler.TryFrameSubmitted( + schedule, m_frame[frameIndex]->frameSerial); +} + void CIndirectDeviceContext::ObserveFrame(uint64_t now) { m_frameScheduler.ObserveFrame(now); @@ -2124,14 +2148,20 @@ bool CIndirectDeviceContext::GetPublishTarget(uint64_t now, now, target, schedule, periodic, republish); } +void CIndirectDeviceContext::FrameMissed( + const CFrameScheduler::Schedule& schedule, uint64_t now, bool periodic) +{ + m_frameScheduler.FrameMissed(schedule, now, periodic); +} + void CIndirectDeviceContext::FrameSuperseded() { m_frameScheduler.FrameSuperseded(); } -void CIndirectDeviceContext::RecordFrameTiming(uint64_t duration) +void CIndirectDeviceContext::TryRecordFrameTiming(uint64_t duration) { - m_frameScheduler.RecordFrameTiming(duration); + m_frameScheduler.TryRecordFrameTiming(duration); } void CIndirectDeviceContext::AbortFrameBuffer(unsigned frameIndex) @@ -2183,17 +2213,23 @@ void CIndirectDeviceContext::CompleteFrameBuffer( void CIndirectDeviceContext::SetFrameTiming(unsigned frameIndex, uint64_t captureTime, uint64_t postProcessTime, uint64_t copyTime, - uint64_t readyTime) + uint64_t readyTime, const CFrameScheduler::Schedule& schedule, + uint64_t completedAt) { if (frameIndex >= LGMP_Q_FRAME_BUFFER_LEN) return; - KVMFRFrame * frame = m_frame[frameIndex]; - frame->captureTime = captureTime; - frame->postProcessTime = postProcessTime; - frame->copyTime = copyTime; - frame->readyTime = readyTime; - frame->timingSerial = frame->frameSerial; + KVMFRFrame * frame = m_frame[frameIndex]; + const bool phaseValid = m_frameScheduler.TryFrameCompleted( + schedule, frame->frameSerial, completedAt); + + frame->captureTime = captureTime; + frame->postProcessTime = postProcessTime; + frame->copyTime = copyTime; + frame->readyTime = readyTime; + frame->timingFlags = phaseValid ? + KVMFR_FRAME_TIMING_PHASE_VALID : 0; + frame->timingSerial = frame->frameSerial; InterlockedExchange((volatile LONG *)&frame->timingValid, 1); } diff --git a/idd/LGIdd/CIndirectDeviceContext.h b/idd/LGIdd/CIndirectDeviceContext.h index 2eaa377a..74cd64ad 100644 --- a/idd/LGIdd/CIndirectDeviceContext.h +++ b/idd/LGIdd/CIndirectDeviceContext.h @@ -282,13 +282,16 @@ public: bool PublishFrameBuffer(unsigned frameIndex, const CFrameScheduler::Schedule& schedule); bool RepublishFrameBuffer(const CFrameScheduler::Schedule& schedule); + bool TryFrameSubmitted(unsigned frameIndex, + const CFrameScheduler::Schedule& schedule); void CommitFrameBuffer(unsigned frameIndex, const CFrameScheduler::Schedule& schedule, bool periodic); void AbortFrameBuffer(unsigned frameIndex); void FailFrameBuffer(unsigned frameIndex); void CompleteFrameBuffer(unsigned frameIndex, bool succeeded); void SetFrameTiming(unsigned frameIndex, uint64_t captureTime, - uint64_t postProcessTime, uint64_t copyTime, uint64_t readyTime); + uint64_t postProcessTime, uint64_t copyTime, uint64_t readyTime, + const CFrameScheduler::Schedule& schedule, uint64_t completedAt); void WriteFrameBuffer(unsigned frameIndex, void* src, size_t offset, size_t len, bool setWritePos) const; void FinalizeFrameBuffer(unsigned frameIndex) const; @@ -296,12 +299,14 @@ public: void ForceFrame(); bool GetPublishTarget(uint64_t now, uint64_t& target, CFrameScheduler::Schedule& schedule, bool& periodic, bool& republish); + void FrameMissed(const CFrameScheduler::Schedule& schedule, + uint64_t now, bool periodic); void FrameSuperseded(); HANDLE GetFrameScheduleEvent() const { return m_frameScheduler.GetWakeEvent(); } - void RecordFrameTiming(uint64_t duration); + void TryRecordFrameTiming(uint64_t duration); void SendCursor(const IDARG_OUT_QUERY_HWCURSOR & info, const BYTE * data, UINT sdrWhiteLevel); diff --git a/idd/LGIdd/CSwapChainProcessor.cpp b/idd/LGIdd/CSwapChainProcessor.cpp index 58f24a1d..33037b68 100644 --- a/idd/LGIdd/CSwapChainProcessor.cpp +++ b/idd/LGIdd/CSwapChainProcessor.cpp @@ -327,6 +327,16 @@ void CSwapChainProcessor::PublisherThread() } const uint64_t replayNow = CFrameScheduler::Nanotime(); + uint64_t cadenceTarget = 0; + if (schedule.deliveryDeadlineSerial && periodic && + schedule.deadline <= replayNow) + { + m_devContext->FrameMissed(schedule, replayNow, periodic); + continue; + } + if (schedule.deliveryDeadlineSerial && periodic) + cadenceTarget = schedule.deadline; + uint64_t replayTarget; if (m_devContext->GetSharedFrameTarget(replayNow, replayTarget)) { @@ -335,18 +345,28 @@ void CSwapChainProcessor::PublisherThread() { if (m_devContext->ReplaySharedFrame(replayNow, retry)) continue; - if (!retry) + if (retry) + replayTarget = replayNow + PUBLISH_RETRY_NS; + else { - if (m_publishTimer.Get()) - CancelWaitableTimer(m_publishTimer.Get()); - if (WaitForMultipleObjects( - ARRAYSIZE(idleHandles), idleHandles, FALSE, INFINITE) == - WAIT_OBJECT_0) - break; - continue; + if (cadenceTarget) + replayTarget = cadenceTarget; + else + { + if (m_publishTimer.Get()) + CancelWaitableTimer(m_publishTimer.Get()); + if (WaitForMultipleObjects( + ARRAYSIZE(idleHandles), idleHandles, FALSE, INFINITE) == + WAIT_OBJECT_0) + break; + continue; + } } } + if (cadenceTarget && cadenceTarget < replayTarget) + replayTarget = cadenceTarget; + const uint64_t delay = replayTarget > replayNow ? replayTarget - replayNow : PUBLISH_RETRY_NS; ArmPublishTimer(m_publishTimer.Get(), delay); @@ -357,6 +377,17 @@ void CSwapChainProcessor::PublisherThread() continue; } + if (cadenceTarget) + { + ArmPublishTimer( + m_publishTimer.Get(), cadenceTarget - replayNow); + if (WaitForMultipleObjects( + ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) == + WAIT_OBJECT_0) + break; + continue; + } + if (m_publishTimer.Get()) CancelWaitableTimer(m_publishTimer.Get()); if (WaitForMultipleObjects( @@ -711,10 +742,7 @@ void CSwapChainProcessor::CompletionFunction( const bool gpuTimingValid = slot->GetGPUTimes(gpuCopyStart, gpuCopyEnd); - // Publish readiness before sampling the endpoint. Timing has its own valid - // flag and is published immediately afterwards. - sc->m_devContext->FinalizeFrameBuffer(fbRes->GetFrameIndex()); - const uint64_t readyEnd = CFrameScheduler::Nanotime(); + const uint64_t copyReady = CFrameScheduler::Nanotime(); const uint64_t postProcessStart = fbRes->GetPostProcessStart(); uint64_t postProcessTime = prepareCopyStart - postProcessStart; @@ -726,31 +754,41 @@ void CSwapChainProcessor::CompletionFunction( prepareCopyTime = prepareGPUEnd - prepareGPUStart; } - uint64_t publishCopyTime = readyEnd - publishStart; + uint64_t publishCopyTime = copyReady - publishStart; if (gpuTimingValid && gpuCopyStart >= publishStart && - gpuCopyEnd >= gpuCopyStart && gpuCopyEnd <= readyEnd) + gpuCopyEnd >= gpuCopyStart && gpuCopyEnd <= copyReady) publishCopyTime = gpuCopyEnd - gpuCopyStart + indirectCopyTime; const uint64_t copyTime = prepareCopyTime + publishCopyTime; - const uint64_t elapsed = readyEnd - postProcessStart; - const uint64_t measured = postProcessTime + copyTime; - const uint64_t readyTime = elapsed > measured ? elapsed - measured : 0; + + // Make the framebuffer readable before phase bookkeeping. If the scheduler + // lock is busy, the frame is still delivered and only this phase sample is + // discarded. + sc->m_devContext->FinalizeFrameBuffer(fbRes->GetFrameIndex()); + const uint64_t publishedAt = CFrameScheduler::Nanotime(); + const uint64_t elapsed = publishedAt - postProcessStart; + const uint64_t measured = postProcessTime + copyTime; + const uint64_t readyTime = elapsed > measured ? elapsed - measured : 0; + + sc->m_devContext->SetFrameTiming(fbRes->GetFrameIndex(), + fbRes->GetCaptureTime(), postProcessTime, copyTime, readyTime, + fbRes->GetSchedule(), publishedAt); + sc->m_devContext->TryRecordFrameTiming( + publishedAt - publishStart); // Use matching wall-clock boundaries for both modes. The split excludes the // cadence hold while including the indirect CPU copy only when it occurs. const uint64_t timingToken = fbRes->GetTimingToken(); if (timingToken && timingStart && prepareReady >= timingStart && - readyEnd >= publishStart) + copyReady >= publishStart) { const uint64_t totalTime = - (prepareReady - timingStart) + (readyEnd - publishStart); + (prepareReady - timingStart) + (copyReady - publishStart); sc->m_postProcessors[candidateIndex].RecordTiming( fbRes->GetTimingEffectIndex(), timingToken, fbRes->IsFullCopy(), totalTime); } - sc->m_devContext->RecordFrameTiming(readyEnd - publishStart); - sc->m_devContext->SetFrameTiming(fbRes->GetFrameIndex(), - fbRes->GetCaptureTime(), postProcessTime, copyTime, readyTime); + sc->m_devContext->CompleteFrameBuffer(fbRes->GetFrameIndex(), true); sc->ReleaseCandidate(candidateIndex); } @@ -1270,6 +1308,12 @@ bool CSwapChainProcessor::PublishNewestCandidate( restoreCandidates(); return false; } + CFrameScheduler::Schedule frameSchedule = schedule; + // Phase accounting must never hold up D3D submission. Keep the immutable + // delivery identity and discard only this feedback sample on contention. + if (!m_devContext->TryFrameSubmitted(buffer.frameIndex, schedule)) + frameSchedule.phaseEligible = false; + fbRes->SetSchedule(frameSchedule); // Retire the candidate damage before submission. The completion callback // may run before Execute returns and make this candidate reusable. diff --git a/obs/frame_scheduler.c b/obs/frame_scheduler.c index 122ffb11..0caa2478 100644 --- a/obs/frame_scheduler.c +++ b/obs/frame_scheduler.c @@ -61,13 +61,14 @@ void lgFrameSchedulerSetPeriod(LGFrameScheduler * scheduler, return; ++scheduler->generation; - scheduler->resetPending = true; - scheduler->immediatePending = true; - scheduler->phaseError = 0; - scheduler->feedbackFrameSerial = 0; - scheduler->feedbackScheduleEpoch = 0; - scheduler->feedbackSamples = 0; - scheduler->feedbackDirty = false; + scheduler->resetPending = true; + scheduler->immediatePending = true; + scheduler->phaseError = 0; + scheduler->feedbackFrameSerial = 0; + scheduler->feedbackScheduleEpoch = 0; + scheduler->feedbackDeadlineSerial = 0; + scheduler->feedbackSamples = 0; + scheduler->feedbackDirty = false; } void lgFrameSchedulerRequestImmediate(LGFrameScheduler * scheduler) @@ -78,31 +79,35 @@ void lgFrameSchedulerRequestImmediate(LGFrameScheduler * scheduler) void lgFrameSchedulerObserveFrame(LGFrameScheduler * scheduler, uint32_t frameSerial, uint32_t generation, uint32_t scheduleEpoch, - uint64_t readyTime) + uint32_t deadlineSerial, uint64_t readyTime) { - if (!generation || !scheduleEpoch || + if (!generation || !scheduleEpoch || !deadlineSerial || (scheduler->readyFrameSerial == frameSerial && scheduler->readyGeneration == generation && - scheduler->readyScheduleEpoch == scheduleEpoch)) + scheduler->readyScheduleEpoch == scheduleEpoch && + scheduler->readyDeadlineSerial == deadlineSerial)) return; - scheduler->readyFrameSerial = frameSerial; - scheduler->readyGeneration = generation; - scheduler->readyScheduleEpoch = scheduleEpoch; - scheduler->readyTime = readyTime; + scheduler->readyFrameSerial = frameSerial; + scheduler->readyGeneration = generation; + scheduler->readyScheduleEpoch = scheduleEpoch; + scheduler->readyDeadlineSerial = deadlineSerial; + scheduler->readyTime = readyTime; } void lgFrameSchedulerFeedback(LGFrameScheduler * scheduler, uint32_t frameSerial, uint32_t generation, uint32_t scheduleEpoch, - uint64_t tickTime) + uint32_t deadlineSerial, uint64_t tickTime) { if (!scheduler->active || generation != scheduler->generation || - !scheduleEpoch || + !scheduleEpoch || !deadlineSerial || (frameSerial == scheduler->feedbackFrameSerial && - scheduleEpoch == scheduler->feedbackScheduleEpoch) || + scheduleEpoch == scheduler->feedbackScheduleEpoch && + deadlineSerial == scheduler->feedbackDeadlineSerial) || scheduler->readyFrameSerial != frameSerial || scheduler->readyGeneration != generation || - scheduler->readyScheduleEpoch != scheduleEpoch) + scheduler->readyScheduleEpoch != scheduleEpoch || + scheduler->readyDeadlineSerial != deadlineSerial) return; if (scheduler->feedbackScheduleEpoch && @@ -132,7 +137,8 @@ void lgFrameSchedulerFeedback(LGFrameScheduler * scheduler, ++scheduler->feedbackSamples; scheduler->feedbackFrameSerial = frameSerial; - scheduler->feedbackScheduleEpoch = scheduleEpoch; + scheduler->feedbackScheduleEpoch = scheduleEpoch; + scheduler->feedbackDeadlineSerial = deadlineSerial; scheduler->feedbackDirty = true; } @@ -159,17 +165,20 @@ void lgFrameSchedulerUpdate(LGFrameScheduler * scheduler, scheduler->feedbackFrameSerial; const uint32_t feedbackScheduleEpoch = scheduler->feedbackScheduleEpoch; + const uint32_t feedbackDeadlineSerial = + scheduler->feedbackDeadlineSerial; const KVMFRFrameSchedule message = { - .msg.type = KVMFR_MESSAGE_FRAME_SCHEDULE, - .clientID = scheduler->clientID, - .generation = scheduler->generation, - .flags = flags, - .period = scheduler->period, - .targetSlack = FRAME_SCHEDULER_TARGET_SLACK_NS, - .phaseError = scheduler->phaseError, - .feedbackFrameSerial = feedbackFrameSerial, - .feedbackScheduleEpoch = feedbackScheduleEpoch, - .lease = FRAME_SCHEDULER_LEASE_MS, + .msg.type = KVMFR_MESSAGE_FRAME_SCHEDULE, + .clientID = scheduler->clientID, + .generation = scheduler->generation, + .flags = flags, + .period = scheduler->period, + .targetSlack = FRAME_SCHEDULER_TARGET_SLACK_NS, + .phaseError = scheduler->phaseError, + .feedbackFrameSerial = feedbackFrameSerial, + .feedbackScheduleEpoch = feedbackScheduleEpoch, + .feedbackDeadlineSerial = feedbackDeadlineSerial, + .lease = FRAME_SCHEDULER_LEASE_MS, }; if (lgmpClientSendData(queue, &message, sizeof(message), NULL) != LGMP_OK) @@ -180,6 +189,7 @@ void lgFrameSchedulerUpdate(LGFrameScheduler * scheduler, scheduler->immediatePending = false; scheduler->lastSend = now; if (scheduler->feedbackFrameSerial == feedbackFrameSerial && - scheduler->feedbackScheduleEpoch == feedbackScheduleEpoch) + scheduler->feedbackScheduleEpoch == feedbackScheduleEpoch && + scheduler->feedbackDeadlineSerial == feedbackDeadlineSerial) scheduler->feedbackDirty = false; } diff --git a/obs/frame_scheduler.h b/obs/frame_scheduler.h index 8fa14a36..d7272343 100644 --- a/obs/frame_scheduler.h +++ b/obs/frame_scheduler.h @@ -41,11 +41,13 @@ typedef struct LGFrameScheduler int64_t phaseError; uint32_t feedbackFrameSerial; uint32_t feedbackScheduleEpoch; + uint32_t feedbackDeadlineSerial; unsigned feedbackSamples; uint32_t readyFrameSerial; uint32_t readyGeneration; uint32_t readyScheduleEpoch; + uint32_t readyDeadlineSerial; uint64_t readyTime; } LGFrameScheduler; @@ -57,10 +59,10 @@ void lgFrameSchedulerSetPeriod(LGFrameScheduler * scheduler, void lgFrameSchedulerRequestImmediate(LGFrameScheduler * scheduler); void lgFrameSchedulerObserveFrame(LGFrameScheduler * scheduler, uint32_t frameSerial, uint32_t generation, uint32_t scheduleEpoch, - uint64_t readyTime); + uint32_t deadlineSerial, uint64_t readyTime); void lgFrameSchedulerFeedback(LGFrameScheduler * scheduler, uint32_t frameSerial, uint32_t generation, uint32_t scheduleEpoch, - uint64_t tickTime); + uint32_t deadlineSerial, uint64_t tickTime); void lgFrameSchedulerUpdate(LGFrameScheduler * scheduler, PLGMPClientQueue queue, uint64_t now); diff --git a/obs/lg.c b/obs/lg.c index 78b48f75..8137bad3 100644 --- a/obs/lg.c +++ b/obs/lg.c @@ -43,6 +43,8 @@ #include "cursor.effect.h" #include "frame_scheduler.h" +#define LG_FRAME_TIMING_SPIN_COUNT 4096 + /* scRGB reference white in cd/m² (OBS GS_CS_709_SCRGB is defined as * 1.0 = 80 cd/m²). PQ encodes an absolute 0..10000 cd/m² range, so linear * PQ output is scaled by 10000 / 80 to reach the scRGB convention. */ @@ -180,8 +182,8 @@ typedef struct { LGMPMessage msg; PLGMPClientQueue queue; - uint32_t generation; uint32_t scheduleEpoch; + uint32_t scheduleDeadlineSerial; bool owner; } LGFrameMessage; @@ -210,6 +212,18 @@ static bool lgFrameSerialNewer(uint32_t lhs, uint32_t rhs) return (int32_t)(lhs - rhs) > 0; } +static bool lgFrameMessageHasMatchingDeadline( + bool owner, const LGMPMessage * message, const KVMFRFrame * frame) +{ + if (!owner) + return false; + + const uint32_t epoch = (uint32_t)(message->udata >> 32); + const uint32_t deadlineSerial = (uint32_t)message->udata; + return deadlineSerial && epoch == frame->scheduleEpoch && + deadlineSerial == frame->scheduleDeadlineSerial; +} + static LGMP_STATUS lgFrameProcessNewest(LGPlugin * this, LGFrameMessage * result) { @@ -260,7 +274,10 @@ static LGMP_STATUS lgFrameProcessNewest(LGPlugin * this, const bool replace = lgFrameSerialNewer( frame->frameSerial, selected->frameSerial) || (frame->frameSerial == selected->frameSerial && - owner && !result->owner); + ((owner && !result->owner) || + (lgFrameMessageHasMatchingDeadline(owner, &msg, frame) && + !lgFrameMessageHasMatchingDeadline( + result->owner, &result->msg, selected)))); PLGMPClientQueue discard = queues[i].queue; if (replace) { @@ -277,13 +294,43 @@ static LGMP_STATUS lgFrameProcessNewest(LGPlugin * this, if (result->owner) { - result->generation = (uint32_t)(result->msg.udata >> 32); - result->scheduleEpoch = (uint32_t)result->msg.udata; + result->scheduleEpoch = (uint32_t)(result->msg.udata >> 32); + result->scheduleDeadlineSerial = (uint32_t)result->msg.udata; } return result->queue ? LGMP_OK : LGMP_ERR_QUEUE_EMPTY; } +static bool lgFrameTimingReady(const KVMFRFrame * frame) +{ + return __atomic_load_n(&frame->timingValid, __ATOMIC_ACQUIRE) && + frame->timingSerial == frame->frameSerial; +} + +static bool lgFramePhaseIdentity(const LGFrameMessage * message, + const KVMFRFrame * frame, uint32_t * generation) +{ + if (!message->owner || !message->scheduleEpoch || + !message->scheduleDeadlineSerial) + return false; + + for (unsigned i = 0; + !lgFrameTimingReady(frame) && i < LG_FRAME_TIMING_SPIN_COUNT; + ++i) + { + } + + if (!lgFrameTimingReady(frame) || + !(frame->timingFlags & KVMFR_FRAME_TIMING_PHASE_VALID) || + !frame->scheduleGeneration || + frame->scheduleEpoch != message->scheduleEpoch || + frame->scheduleDeadlineSerial != message->scheduleDeadlineSerial) + return false; + + *generation = frame->scheduleGeneration; + return true; +} + static void lgFrameUnsubscribeOwnerQueues(LGPlugin * this) { for (unsigned int i = 0; i < LGMP_Q_FRAME_LEN; ++i) @@ -666,9 +713,11 @@ static void * frameThread(void * data) fb, (size_t)frame->dataHeight * frame->pitch)) { now = os_gettime_ns(); - lgFrameSchedulerObserveFrame(&this->frameScheduler, - frame->frameSerial, frameMessage.generation, - frameMessage.scheduleEpoch, now); + uint32_t generation; + if (lgFramePhaseIdentity(&frameMessage, frame, &generation)) + lgFrameSchedulerObserveFrame(&this->frameScheduler, + frame->frameSerial, generation, frameMessage.scheduleEpoch, + frameMessage.scheduleDeadlineSerial, now); } } } @@ -1557,12 +1606,16 @@ static void lgVideoTick(void * data, float seconds) } const uint64_t readyTime = os_gettime_ns(); - lgFrameSchedulerObserveFrame(&this->frameScheduler, - frame->frameSerial, frameMessage.generation, - frameMessage.scheduleEpoch, readyTime); - lgFrameSchedulerFeedback(&this->frameScheduler, - frame->frameSerial, frameMessage.generation, - frameMessage.scheduleEpoch, tickTime); + uint32_t generation; + if (lgFramePhaseIdentity(&frameMessage, frame, &generation)) + { + lgFrameSchedulerObserveFrame(&this->frameScheduler, + frame->frameSerial, generation, frameMessage.scheduleEpoch, + frameMessage.scheduleDeadlineSerial, readyTime); + lgFrameSchedulerFeedback(&this->frameScheduler, + frame->frameSerial, generation, frameMessage.scheduleEpoch, + frameMessage.scheduleDeadlineSerial, tickTime); + } } this->frameSerial = frame->frameSerial;