diff --git a/client/include/interface/transport.h b/client/include/interface/transport.h index ea631ea6..6934ca21 100644 --- a/client/include/interface/transport.h +++ b/client/include/interface/transport.h @@ -96,6 +96,10 @@ typedef uint32_t LG_TransportFrameFlags; typedef struct LG_TransportFrameTiming { bool valid; /* producer fields below are available and coherent */ + bool phaseValid; + uint32_t scheduleGeneration; + uint32_t scheduleEpoch; + uint32_t scheduleDeadlineSerial; uint64_t captureTime; uint64_t postProcessTime; uint64_t copyTime; @@ -134,8 +138,8 @@ typedef struct LG_TransportFrame { uint64_t serial; uint64_t timestamp; - uint32_t scheduleGeneration; uint32_t scheduleEpoch; + uint32_t scheduleDeadlineSerial; LG_TransportFrameFlags flags; bool scheduleOwner; // Backend-owned immutable metadata, valid until releaseFrame. @@ -216,6 +220,7 @@ typedef struct LG_TransportControl int64_t phaseError; uint32_t feedbackFrameSerial; uint32_t feedbackScheduleEpoch; + uint32_t feedbackDeadlineSerial; uint32_t lease; } frameSchedule; diff --git a/client/src/frame_scheduler.c b/client/src/frame_scheduler.c index 533cc9fa..41fa8d2d 100644 --- a/client/src/frame_scheduler.c +++ b/client/src/frame_scheduler.c @@ -52,6 +52,7 @@ static struct int64_t phaseError; uint32_t feedbackFrameSerial; uint32_t feedbackScheduleEpoch; + uint32_t feedbackDeadlineSerial; unsigned feedbackSamples; bool feedbackDirty; @@ -92,23 +93,26 @@ static bool sendSchedule(LG_TransportFrameScheduleFlags flags, int64_t phaseError; uint32_t feedbackFrameSerial; uint32_t feedbackScheduleEpoch; + uint32_t feedbackDeadlineSerial; LG_LOCK(l_frameScheduler.lock); - phaseError = l_frameScheduler.phaseError; - feedbackFrameSerial = l_frameScheduler.feedbackFrameSerial; - feedbackScheduleEpoch = l_frameScheduler.feedbackScheduleEpoch; + phaseError = l_frameScheduler.phaseError; + feedbackFrameSerial = l_frameScheduler.feedbackFrameSerial; + feedbackScheduleEpoch = l_frameScheduler.feedbackScheduleEpoch; + feedbackDeadlineSerial = l_frameScheduler.feedbackDeadlineSerial; LG_UNLOCK(l_frameScheduler.lock); const LG_TransportControl control = { .type = LG_TRANSPORT_CONTROL_FRAME_SCHEDULE, .frameSchedule = { - .generation = l_frameScheduler.generation, - .flags = flags, - .period = period, - .targetSlack = FRAME_SCHEDULER_TARGET_SLACK_NS, - .phaseError = phaseError, - .feedbackFrameSerial = feedbackFrameSerial, - .feedbackScheduleEpoch = feedbackScheduleEpoch, - .lease = FRAME_SCHEDULER_LEASE_MS, + .generation = l_frameScheduler.generation, + .flags = flags, + .period = period, + .targetSlack = FRAME_SCHEDULER_TARGET_SLACK_NS, + .phaseError = phaseError, + .feedbackFrameSerial = feedbackFrameSerial, + .feedbackScheduleEpoch = feedbackScheduleEpoch, + .feedbackDeadlineSerial = feedbackDeadlineSerial, + .lease = FRAME_SCHEDULER_LEASE_MS, }, }; @@ -127,7 +131,8 @@ static bool sendSchedule(LG_TransportFrameScheduleFlags flags, l_frameScheduler.immediatePending = false; LG_LOCK(l_frameScheduler.lock); if (l_frameScheduler.feedbackFrameSerial == feedbackFrameSerial && - l_frameScheduler.feedbackScheduleEpoch == feedbackScheduleEpoch) + l_frameScheduler.feedbackScheduleEpoch == feedbackScheduleEpoch && + l_frameScheduler.feedbackDeadlineSerial == feedbackDeadlineSerial) l_frameScheduler.feedbackDirty = false; LG_UNLOCK(l_frameScheduler.lock); return true; @@ -156,11 +161,12 @@ void frameScheduler_start(LG_TransportFeatureFlags features) ++l_frameScheduler.generation; LG_LOCK(l_frameScheduler.lock); - l_frameScheduler.phaseError = 0; - l_frameScheduler.feedbackFrameSerial = 0; - l_frameScheduler.feedbackScheduleEpoch = 0; - l_frameScheduler.feedbackSamples = 0; - l_frameScheduler.feedbackDirty = false; + l_frameScheduler.phaseError = 0; + l_frameScheduler.feedbackFrameSerial = 0; + l_frameScheduler.feedbackScheduleEpoch = 0; + l_frameScheduler.feedbackDeadlineSerial = 0; + l_frameScheduler.feedbackSamples = 0; + l_frameScheduler.feedbackDirty = false; LG_UNLOCK(l_frameScheduler.lock); } @@ -194,11 +200,12 @@ void frameScheduler_update(void) l_frameScheduler.period = 0; l_frameScheduler.immediatePending = true; LG_LOCK(l_frameScheduler.lock); - l_frameScheduler.phaseError = 0; - l_frameScheduler.feedbackFrameSerial = 0; - l_frameScheduler.feedbackScheduleEpoch = 0; - l_frameScheduler.feedbackSamples = 0; - l_frameScheduler.feedbackDirty = false; + l_frameScheduler.phaseError = 0; + l_frameScheduler.feedbackFrameSerial = 0; + l_frameScheduler.feedbackScheduleEpoch = 0; + l_frameScheduler.feedbackDeadlineSerial = 0; + l_frameScheduler.feedbackSamples = 0; + l_frameScheduler.feedbackDirty = false; LG_UNLOCK(l_frameScheduler.lock); } return; @@ -219,11 +226,12 @@ void frameScheduler_update(void) ++l_frameScheduler.generation; l_frameScheduler.immediatePending = true; LG_LOCK(l_frameScheduler.lock); - l_frameScheduler.phaseError = 0; - l_frameScheduler.feedbackFrameSerial = 0; - l_frameScheduler.feedbackScheduleEpoch = 0; - l_frameScheduler.feedbackSamples = 0; - l_frameScheduler.feedbackDirty = false; + l_frameScheduler.phaseError = 0; + l_frameScheduler.feedbackFrameSerial = 0; + l_frameScheduler.feedbackScheduleEpoch = 0; + l_frameScheduler.feedbackDeadlineSerial = 0; + l_frameScheduler.feedbackSamples = 0; + l_frameScheduler.feedbackDirty = false; LG_UNLOCK(l_frameScheduler.lock); } else @@ -257,9 +265,10 @@ void frameScheduler_update(void) } void frameScheduler_feedback(uint64_t frameSerial, uint32_t generation, - uint32_t scheduleEpoch, uint64_t measuredPhase) + uint32_t scheduleEpoch, uint32_t deadlineSerial, + uint64_t measuredPhase) { - if (!generation || !scheduleEpoch) + if (!generation || !scheduleEpoch || !deadlineSerial) return; LG_LOCK(l_frameScheduler.lock); @@ -270,6 +279,14 @@ void frameScheduler_feedback(uint64_t frameSerial, uint32_t generation, return; } + if (l_frameScheduler.feedbackFrameSerial == (uint32_t)frameSerial && + l_frameScheduler.feedbackScheduleEpoch == scheduleEpoch && + l_frameScheduler.feedbackDeadlineSerial == deadlineSerial) + { + LG_UNLOCK(l_frameScheduler.lock); + return; + } + const int64_t period = (int64_t)l_frameScheduler.period; if (!period) { @@ -301,8 +318,9 @@ void frameScheduler_feedback(uint64_t frameSerial, uint32_t generation, if (l_frameScheduler.feedbackSamples < 32) ++l_frameScheduler.feedbackSamples; - l_frameScheduler.feedbackFrameSerial = (uint32_t)frameSerial; - l_frameScheduler.feedbackScheduleEpoch = scheduleEpoch; - l_frameScheduler.feedbackDirty = true; + l_frameScheduler.feedbackFrameSerial = (uint32_t)frameSerial; + l_frameScheduler.feedbackScheduleEpoch = scheduleEpoch; + l_frameScheduler.feedbackDeadlineSerial = deadlineSerial; + l_frameScheduler.feedbackDirty = true; LG_UNLOCK(l_frameScheduler.lock); } diff --git a/client/src/frame_scheduler.h b/client/src/frame_scheduler.h index 168a2821..0ed22dab 100644 --- a/client/src/frame_scheduler.h +++ b/client/src/frame_scheduler.h @@ -31,6 +31,7 @@ void frameScheduler_start(LG_TransportFeatureFlags features); void frameScheduler_stop(void); void frameScheduler_update(void); void frameScheduler_feedback(uint64_t frameSerial, uint32_t generation, - uint32_t scheduleEpoch, uint64_t measuredPhase); + uint32_t scheduleEpoch, uint32_t deadlineSerial, + uint64_t measuredPhase); #endif diff --git a/client/src/main.c b/client/src/main.c index 1c2e55a4..663729e4 100644 --- a/client/src/main.c +++ b/client/src/main.c @@ -217,9 +217,10 @@ struct FrameTimingRecord uint64_t frameSerial; uint32_t scheduleGeneration; uint32_t scheduleEpoch; + uint32_t scheduleDeadlineSerial; unsigned readyMask; bool producerValid; - bool scheduleOwner; + bool phaseValid; uint64_t captureTime; uint64_t postProcessTime; @@ -330,9 +331,8 @@ static void frameTimingCancel(LG_RendererFrameToken token) } static void frameTimingQueue(LG_RendererFrameToken token, uint64_t frameSerial, - uint32_t scheduleGeneration, uint32_t scheduleEpoch, bool scheduleOwner, - uint64_t importTime, uint64_t importWaitTime, uint64_t dispatchStart, - uint64_t queueStart) + const LG_TransportFrameTiming * timing, uint64_t importTime, + uint64_t importWaitTime, uint64_t dispatchStart, uint64_t queueStart) { INTERLOCKED_SECTION(l_frameTiming.lock, { struct FrameTimingRecord * record = frameTimingRecord(token); @@ -341,14 +341,16 @@ static void frameTimingQueue(LG_RendererFrameToken token, uint64_t frameSerial, const uint64_t elapsed = queueStart > dispatchStart ? queueStart - dispatchStart : 0; const uint64_t accounted = importTime + importWaitTime; - record->importTime = importTime; - record->importWaitTime = importWaitTime; - record->dispatchTime = elapsed > accounted ? elapsed - accounted : 0; - record->queueStart = queueStart; - record->frameSerial = frameSerial; - record->scheduleGeneration = scheduleGeneration; - record->scheduleEpoch = scheduleEpoch; - record->scheduleOwner = scheduleOwner; + record->importTime = importTime; + record->importWaitTime = importWaitTime; + record->dispatchTime = elapsed > accounted ? + elapsed - accounted : 0; + record->queueStart = queueStart; + record->frameSerial = frameSerial; + record->scheduleGeneration = timing->scheduleGeneration; + record->scheduleEpoch = timing->scheduleEpoch; + record->scheduleDeadlineSerial = timing->scheduleDeadlineSerial; + record->phaseValid = timing->phaseValid; if (record->timestamp < queueStart) record->timestamp = queueStart; } @@ -398,7 +400,8 @@ static void frameTimingFinishRender(const LG_RendererFrameTiming * timing, uint64_t feedbackQueueStart = 0; uint32_t feedbackGeneration = 0; uint32_t feedbackEpoch = 0; - bool feedbackOwner = false; + uint32_t feedbackDeadline = 0; + bool feedbackValid = false; LG_LOCK(l_frameTiming.lock); if (l_frameTiming.retireToken <= timing->frameToken) @@ -421,7 +424,8 @@ static void frameTimingFinishRender(const LG_RendererFrameTiming * timing, feedbackFrameSerial = record->frameSerial; feedbackGeneration = record->scheduleGeneration; feedbackEpoch = record->scheduleEpoch; - feedbackOwner = record->scheduleOwner; + feedbackDeadline = record->scheduleDeadlineSerial; + feedbackValid = record->phaseValid; feedbackQueueStart = record->queueStart; if (unlikely( @@ -445,13 +449,13 @@ static void frameTimingFinishRender(const LG_RendererFrameTiming * timing, } LG_UNLOCK(l_frameTiming.lock); - if (g_state.jitRender && feedbackOwner && feedbackFrameSerial && - feedbackGeneration && feedbackEpoch && feedbackQueueStart && - prepareStart >= feedbackQueueStart) + if (g_state.jitRender && feedbackValid && feedbackFrameSerial && + feedbackGeneration && feedbackEpoch && feedbackDeadline && + feedbackQueueStart && prepareStart >= feedbackQueueStart) { frameScheduler_feedback( feedbackFrameSerial, feedbackGeneration, feedbackEpoch, - prepareStart - feedbackQueueStart); + feedbackDeadline, prepareStart - feedbackQueueStart); } } @@ -1143,9 +1147,9 @@ int main_frameThread(void * unused) atomic_store_explicit(&l_testFrameSerial, frame.serial, memory_order_release); #endif - frameTimingQueue(frameToken, frame.serial, frame.scheduleGeneration, - frame.scheduleEpoch, frame.scheduleOwner, g_state.frameImportTime, - g_state.frameImportWaitTime, dispatchStart, queueStart); + frameTimingQueue(frameToken, frame.serial, &timing, + g_state.frameImportTime, g_state.frameImportWaitTime, + dispatchStart, queueStart); if (g_state.jitRender) { diff --git a/client/transports/LGMP/lgmp.c b/client/transports/LGMP/lgmp.c index 03455136..1de5ba5a 100644 --- a/client/transports/LGMP/lgmp.c +++ b/client/transports/LGMP/lgmp.c @@ -721,6 +721,18 @@ static bool lgmp_frameSerialNewer(uint32_t lhs, uint32_t rhs) (uint32_t)(lhs - rhs) < UINT32_C(0x80000000); } +static bool lgmp_frameMessageHasMatchingDeadline( + const struct LGMPFrameMessage * message) +{ + if (!message->owner) + return false; + + const uint32_t epoch = (uint32_t)(message->message.udata >> 32); + const uint32_t deadlineSerial = (uint32_t)message->message.udata; + return deadlineSerial && epoch == message->frame->scheduleEpoch && + deadlineSerial == message->frame->scheduleDeadlineSerial; +} + static void lgmp_selectNewestFrameMessage( struct LGMPFrameMessage * candidate, struct LGMPFrameMessage ** selected) @@ -732,7 +744,9 @@ static void lgmp_selectNewestFrameMessage( lgmp_frameSerialNewer(candidate->frame->frameSerial, (*selected)->frame->frameSerial) || (candidate->frame->frameSerial == (*selected)->frame->frameSerial && - candidate->owner && !(*selected)->owner)) + ((candidate->owner && !(*selected)->owner) || + (lgmp_frameMessageHasMatchingDeadline(candidate) && + !lgmp_frameMessageHasMatchingDeadline(*selected))))) *selected = candidate; } @@ -917,9 +931,9 @@ static LG_TransportStatus lgmp_nextFrameLocked(LG_Transport * this, if (selected->owner) { const uint64_t scheduleToken = selected->message.udata; - result->scheduleGeneration = (uint32_t)(scheduleToken >> 32); - result->scheduleEpoch = (uint32_t)scheduleToken; - result->scheduleOwner = true; + result->scheduleEpoch = (uint32_t)(scheduleToken >> 32); + result->scheduleDeadlineSerial = (uint32_t)scheduleToken; + result->scheduleOwner = true; } if (frame->flags & FRAME_FLAG_BLOCK_SCREENSAVER) result->flags |= LG_TRANSPORT_FRAME_BLOCK_SCREENSAVER; @@ -1057,11 +1071,20 @@ static void lgmp_getFrameTiming(LG_Transport * this, return; } - timing->valid = true; - timing->captureTime = lease->frame->captureTime; - timing->postProcessTime = lease->frame->postProcessTime; - timing->copyTime = lease->frame->copyTime; - timing->readyTime = lease->frame->readyTime; + timing->valid = true; + timing->scheduleGeneration = lease->frame->scheduleGeneration; + timing->scheduleEpoch = lease->frame->scheduleEpoch; + timing->scheduleDeadlineSerial = lease->frame->scheduleDeadlineSerial; + timing->phaseValid = frame->scheduleOwner && + timing->scheduleGeneration && timing->scheduleEpoch && + timing->scheduleDeadlineSerial && + timing->scheduleEpoch == frame->scheduleEpoch && + timing->scheduleDeadlineSerial == frame->scheduleDeadlineSerial && + (lease->frame->timingFlags & KVMFR_FRAME_TIMING_PHASE_VALID); + timing->captureTime = lease->frame->captureTime; + timing->postProcessTime = lease->frame->postProcessTime; + timing->copyTime = lease->frame->copyTime; + timing->readyTime = lease->frame->readyTime; LG_UNLOCK(this->frameLock); } @@ -1244,17 +1267,19 @@ static LG_TransportStatus lgmp_sendControl(LG_Transport * this, case LG_TRANSPORT_CONTROL_FRAME_SCHEDULE: { const KVMFRFrameSchedule message = { - .msg.type = KVMFR_MESSAGE_FRAME_SCHEDULE, - .clientID = this->clientID, - .generation = control->frameSchedule.generation, - .flags = control->frameSchedule.flags, - .period = control->frameSchedule.period, - .targetSlack = control->frameSchedule.targetSlack, - .phaseError = control->frameSchedule.phaseError, - .feedbackFrameSerial = control->frameSchedule.feedbackFrameSerial, - .feedbackScheduleEpoch = + .msg.type = KVMFR_MESSAGE_FRAME_SCHEDULE, + .clientID = this->clientID, + .generation = control->frameSchedule.generation, + .flags = control->frameSchedule.flags, + .period = control->frameSchedule.period, + .targetSlack = control->frameSchedule.targetSlack, + .phaseError = control->frameSchedule.phaseError, + .feedbackFrameSerial = control->frameSchedule.feedbackFrameSerial, + .feedbackScheduleEpoch = control->frameSchedule.feedbackScheduleEpoch, - .lease = control->frameSchedule.lease, + .feedbackDeadlineSerial = + control->frameSchedule.feedbackDeadlineSerial, + .lease = control->frameSchedule.lease, }; memcpy(buffer, &message, sizeof(message)); size = sizeof(message); diff --git a/common/include/common/KVMFR.h b/common/include/common/KVMFR.h index 8e3ad0a5..ca4f02ae 100644 --- a/common/include/common/KVMFR.h +++ b/common/include/common/KVMFR.h @@ -30,7 +30,7 @@ #include "LGMPConfig.h" #define KVMFR_MAGIC "KVMFR---" -#define KVMFR_VERSION 27 +#define KVMFR_VERSION 28 // Fallback used by producers that cannot report the source display's SDR // white level. IDD frames override this with IDDCX_METADATA2::SdrWhiteLevel. @@ -166,6 +166,13 @@ enum typedef uint32_t KVMFRFrameFlags; +enum +{ + KVMFR_FRAME_TIMING_PHASE_VALID = 0x1 +}; + +typedef uint32_t KVMFRFrameTimingFlags; + typedef struct KVMFRFrame { /* @@ -211,9 +218,16 @@ typedef struct KVMFRFrame uint32_t timingSerial; uint32_t timingValid; + // Producer cadence identity. Phase feedback is permitted only when the + // matching frame has KVMFR_FRAME_TIMING_PHASE_VALID set. + KVMFRFrameTimingFlags timingFlags; + uint32_t scheduleGeneration; + uint32_t scheduleEpoch; + uint32_t scheduleDeadlineSerial; + // Keep the conditional HDR block and damage rectangles on separate cache // lines from the producer timing fields. - uint8_t timingReserved[24]; + uint8_t timingReserved[8]; // HDR static metadata (valid when FRAME_FLAG_HDR_METADATA is set) // Display color primaries in 0.00002 units (SMPTE ST 2086 format) @@ -237,6 +251,10 @@ KVMFRFrame; #if defined(__cplusplus) static_assert(offsetof(KVMFRFrame, captureTime) == 64, "KVMFRFrame hot fields must fit in one cache line"); +static_assert(offsetof(KVMFRFrame, timingFlags) == 104, + "KVMFRFrame timing flags layout changed"); +static_assert(offsetof(KVMFRFrame, scheduleDeadlineSerial) == 116, + "KVMFRFrame schedule identity layout changed"); static_assert(offsetof(KVMFRFrame, hdrDisplayPrimary) == 128, "KVMFRFrame HDR metadata must be cache-line aligned"); static_assert(offsetof(KVMFRFrame, damageRects) == 192, @@ -244,6 +262,10 @@ static_assert(offsetof(KVMFRFrame, damageRects) == 192, #elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L _Static_assert(offsetof(KVMFRFrame, captureTime) == 64, "KVMFRFrame hot fields must fit in one cache line"); +_Static_assert(offsetof(KVMFRFrame, timingFlags) == 104, + "KVMFRFrame timing flags layout changed"); +_Static_assert(offsetof(KVMFRFrame, scheduleDeadlineSerial) == 116, + "KVMFRFrame schedule identity layout changed"); _Static_assert(offsetof(KVMFRFrame, hdrDisplayPrimary) == 128, "KVMFRFrame HDR metadata must be cache-line aligned"); _Static_assert(offsetof(KVMFRFrame, damageRects) == 192, @@ -297,15 +319,25 @@ typedef struct KVMFRFrameSchedule int64_t phaseError; // ready-to-render phase error (ns) uint32_t feedbackFrameSerial; uint32_t feedbackScheduleEpoch; + // Identifies the exact producer deadline paired with feedbackFrameSerial. + uint32_t feedbackDeadlineSerial; uint32_t lease; // lease duration (ms) - uint8_t reserved[12]; + uint8_t reserved[8]; } KVMFRFrameSchedule; #if defined(__cplusplus) +static_assert(offsetof(KVMFRFrameSchedule, feedbackDeadlineSerial) == 48, + "KVMFR frame schedule feedback identity layout changed"); +static_assert(offsetof(KVMFRFrameSchedule, lease) == 52, + "KVMFR frame schedule lease layout changed"); static_assert(sizeof(KVMFRFrameSchedule) == 64, "KVMFR frame schedule must fit in one LGMP control message"); #elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +_Static_assert(offsetof(KVMFRFrameSchedule, feedbackDeadlineSerial) == 48, + "KVMFR frame schedule feedback identity layout changed"); +_Static_assert(offsetof(KVMFRFrameSchedule, lease) == 52, + "KVMFR frame schedule lease layout changed"); _Static_assert(sizeof(KVMFRFrameSchedule) == 64, "KVMFR frame schedule must fit in one LGMP control message"); #endif diff --git a/host/src/app.c b/host/src/app.c index a14bf953..359db0e6 100644 --- a/host/src/app.c +++ b/host/src/app.c @@ -398,11 +398,16 @@ static bool sendFrame(CaptureResult result, bool * restart) // fi->offset is initialized at startup fi->flags = flags; fi->sdrWhiteLevel = sdrWhiteLevel; - fi->captureTime = frame.captureTime; - fi->postProcessTime = 0; - fi->copyTime = 0; - fi->readyTime = 0; - fi->timingSerial = 0; + + fi->captureTime = frame.captureTime; + fi->postProcessTime = 0; + fi->copyTime = 0; + fi->readyTime = 0; + fi->timingSerial = 0; + fi->timingFlags = 0; + fi->scheduleGeneration = 0; + fi->scheduleEpoch = 0; + fi->scheduleDeadlineSerial = 0; __atomic_store_n(&fi->timingValid, 0, __ATOMIC_RELAXED); if (frame.hdrMetadata) { diff --git a/idd/LGIdd/CFrameBufferResource.h b/idd/LGIdd/CFrameBufferResource.h index bf5678d5..7b6437e0 100644 --- a/idd/LGIdd/CFrameBufferResource.h +++ b/idd/LGIdd/CFrameBufferResource.h @@ -26,6 +26,8 @@ #include #include +#include "CFrameScheduler.h" + class CSwapChainProcessor; using namespace Microsoft::WRL; @@ -33,19 +35,22 @@ using namespace Microsoft::WRL; class CFrameBufferResource { private: - unsigned m_frameIndex = 0; - uint8_t * m_base = nullptr; - size_t m_size = 0; - size_t m_frameSize = 0; - uint64_t m_captureTime = 0; - uint64_t m_postProcessStart = 0; - uint64_t m_copyStart = 0; - unsigned m_timingEffectIndex = 0; - uint64_t m_timingToken = 0; - bool m_fullCopy = false; - unsigned m_candidateIndex = 0; - ComPtr m_res; - void * m_map = nullptr; + unsigned m_frameIndex = 0; + uint8_t * m_base = nullptr; + size_t m_size = 0; + size_t m_frameSize = 0; + uint64_t m_captureTime = 0; + uint64_t m_postProcessStart = 0; + uint64_t m_copyStart = 0; + + CFrameScheduler::Schedule m_schedule = {}; + + unsigned m_timingEffectIndex = 0; + uint64_t m_timingToken = 0; + bool m_fullCopy = false; + unsigned m_candidateIndex = 0; + ComPtr m_res; + void * m_map = nullptr; public: bool Init(CSwapChainProcessor * swapChain, unsigned frameIndex, uint8_t * base, size_t size); @@ -65,6 +70,14 @@ class CFrameBufferResource uint64_t GetCaptureTime () const { return m_captureTime; } uint64_t GetPostProcessStart() const { return m_postProcessStart; } uint64_t GetCopyStart () const { return m_copyStart; } + void SetSchedule(const CFrameScheduler::Schedule& schedule) + { + m_schedule = schedule; + } + const CFrameScheduler::Schedule& GetSchedule() const + { + return m_schedule; + } void SetPostProcessSample( unsigned effectIndex, uint64_t token, bool fullCopy) diff --git a/idd/LGIdd/CFrameScheduler.cpp b/idd/LGIdd/CFrameScheduler.cpp index 32fbe406..15d5839c 100644 --- a/idd/LGIdd/CFrameScheduler.cpp +++ b/idd/LGIdd/CFrameScheduler.cpp @@ -22,6 +22,8 @@ #include "CDebug.h" +#include + static const uint64_t MIN_SOURCE_PERIOD_NS = 100000ULL; static const uint64_t MIN_SCHEDULE_PERIOD_NS = 2000000ULL; static const uint64_t MAX_PERIOD_NS = 1000000000ULL; @@ -82,7 +84,21 @@ CFrameScheduler::Client * CFrameScheduler::FindClient(uint32_t clientID) return nullptr; } -bool CFrameScheduler::ElectOwner(uint64_t now) +CFrameScheduler::Publication * CFrameScheduler::FindPublication( + const Schedule& schedule, uint32_t frameSerial) +{ + for (Publication& publication : m_publications) + if (publication.generation == schedule.generation && + publication.epoch == schedule.epoch && + publication.deadlineSerial == schedule.deadlineSerial && + publication.frameSerial == frameSerial && + publication.deadline == schedule.deadline) + return &publication; + + return nullptr; +} + +bool CFrameScheduler::ElectOwner(uint64_t now, uint32_t resetClientID) { Client * fastest = nullptr; Client * incumbent = FindClient(m_schedule.clientID); @@ -122,7 +138,8 @@ bool CFrameScheduler::ElectOwner(uint64_t now) const uint32_t oldEpoch = m_schedule.epoch; const uint64_t oldPeriod = m_schedule.period; const uint64_t oldSlack = m_schedule.targetSlack; - if (incumbent && incumbent->active && + if (incumbent && incumbent->clientID != resetClientID && + incumbent->active && incumbent->generation == oldGeneration) incumbent->nextDelivery = m_nextDeadline; @@ -142,7 +159,11 @@ bool CFrameScheduler::ElectOwner(uint64_t now) } const bool ownerChanged = oldClientID != m_schedule.clientID; - if (ownerChanged || oldGeneration != m_schedule.generation) + const bool ownerReset = resetClientID && + resetClientID == m_schedule.clientID; + const bool identityChanged = ownerChanged || ownerReset || + oldGeneration != m_schedule.generation; + if (identityChanged) { if (m_scheduling) { @@ -153,6 +174,8 @@ bool CFrameScheduler::ElectOwner(uint64_t now) m_nextDeadline = m_scheduling ? fastest->nextDelivery : 0; if (m_scheduling && !m_nextDeadline) m_nextDeadline = now + m_schedule.period; + m_deadlineSerial = m_scheduling ? 1 : 0; + m_pendingCorrection = 0; if (m_scheduling) { ++m_forceRequestTicket; @@ -165,9 +188,10 @@ bool CFrameScheduler::ElectOwner(uint64_t now) } if (fastest) - fastest->lastFeedbackFrameSerial = 0; + fastest->lastFeedbackDeadlineSerial = 0; - m_lastPublishedFrameSerial = 0; + memset(m_publications, 0, sizeof(m_publications)); + m_publicationIndex = 0; m_lastPhaseError = 0; m_lastLog = now; m_lastLogAcquired = m_acquiredFrames; @@ -182,7 +206,7 @@ bool CFrameScheduler::ElectOwner(uint64_t now) DEBUG_INFO("Frame timing owner released; using push delivery"); } - return ownerChanged || oldGeneration != m_schedule.generation || + return identityChanged || oldPeriod != m_schedule.period || oldSlack != m_schedule.targetSlack || clientExpired; } @@ -199,12 +223,15 @@ void CFrameScheduler::Reset() m_forceAckTicket = m_forceRequestTicket; m_republishAckTicket = m_republishRequestTicket; - m_lastArrival = 0; - m_guestPeriod = 0; - m_workEstimate = 0; - m_nextDeadline = 0; + m_lastArrival = 0; + m_guestPeriod = 0; + m_workEstimate = 0; + m_nextDeadline = 0; + m_deadlineSerial = 0; + m_pendingCorrection = 0; - m_lastPublishedFrameSerial = 0; + memset(m_publications, 0, sizeof(m_publications)); + m_publicationIndex = 0; m_lastPhaseError = 0; m_acquiredFrames = 0; @@ -351,16 +378,18 @@ bool CFrameScheduler::UpdateSchedule(uint32_t sourceClientID, return false; } + const bool explicitReset = + (schedule.flags & KVMFR_FRAME_SCHEDULE_RESET) != 0; const bool reset = client->generation != schedule.generation || - (schedule.flags & KVMFR_FRAME_SCHEDULE_RESET); + explicitReset; bool wake = reset || !client->active || client->period != schedule.period || client->targetSlack != schedule.targetSlack; if (reset) { - client->lastFeedbackFrameSerial = 0; - client->immediate = false; - client->nextDelivery = now + schedule.period; + client->lastFeedbackDeadlineSerial = 0; + client->immediate = false; + client->nextDelivery = now + schedule.period; } client->generation = schedule.generation; client->period = schedule.period; @@ -372,7 +401,7 @@ bool CFrameScheduler::UpdateSchedule(uint32_t sourceClientID, client->immediate = true; wake = true; } - wake |= ElectOwner(now); + wake |= ElectOwner(now, explicitReset ? schedule.clientID : 0); if (m_scheduling && client->clientID == m_schedule.clientID && client->generation == m_schedule.generation && (schedule.flags & KVMFR_FRAME_SCHEDULE_IMMEDIATE)) @@ -395,14 +424,30 @@ bool CFrameScheduler::ApplyFeedback(Client& client, schedule.generation != m_schedule.generation || schedule.feedbackScheduleEpoch != m_schedule.epoch || !schedule.feedbackFrameSerial || - (client.lastFeedbackFrameSerial && - static_cast(schedule.feedbackFrameSerial - - client.lastFeedbackFrameSerial) <= 0) || - !m_lastPublishedFrameSerial || - static_cast(schedule.feedbackFrameSerial - - m_lastPublishedFrameSerial) > 0) + !schedule.feedbackDeadlineSerial || + (client.lastFeedbackDeadlineSerial && + static_cast(schedule.feedbackDeadlineSerial - + client.lastFeedbackDeadlineSerial) <= 0)) return false; + Publication * publication = nullptr; + for (Publication& candidate : m_publications) + if (candidate.generation == schedule.generation && + candidate.epoch == schedule.feedbackScheduleEpoch && + candidate.deadlineSerial == schedule.feedbackDeadlineSerial && + candidate.frameSerial == schedule.feedbackFrameSerial) + { + publication = &candidate; + break; + } + + if (!publication || !publication->committed || + !publication->completed || !publication->phaseValid || + publication->accepted) + return false; + + publication->accepted = true; + int64_t correction = schedule.phaseError / 4; const int64_t limit = static_cast(m_schedule.period / 4); if (correction > limit) @@ -410,18 +455,50 @@ bool CFrameScheduler::ApplyFeedback(Client& client, else if (correction < -limit) correction = -limit; - if (correction >= 0) - m_nextDeadline += static_cast(correction); + m_pendingCorrection += correction; + m_lastPhaseError = schedule.phaseError; + client.lastFeedbackDeadlineSerial = schedule.feedbackDeadlineSerial; + return false; +} + +void CFrameScheduler::AdvanceCurrentDeadline() +{ + m_nextDeadline += m_schedule.period; + if (m_pendingCorrection >= 0) + m_nextDeadline += static_cast(m_pendingCorrection); else { - const uint64_t advance = static_cast(-correction); - m_nextDeadline = m_nextDeadline > advance ? - m_nextDeadline - advance : 0; + const uint64_t correction = + static_cast(-m_pendingCorrection); + m_nextDeadline = m_nextDeadline > correction ? + m_nextDeadline - correction : 0; } - m_lastPhaseError = schedule.phaseError; - client.lastFeedbackFrameSerial = schedule.feedbackFrameSerial; - client.nextDelivery = m_nextDeadline; - return correction != 0; + m_pendingCorrection = 0; + AdvanceDeadlineSerial(1); +} + +void CFrameScheduler::AdvanceDeadlineSerial(uint64_t count) +{ + if (!count || !m_deadlineSerial) + return; + + const uint64_t position = + static_cast(m_deadlineSerial - 1) + count; + uint64_t epochAdvances = position / UINT32_MAX; + m_deadlineSerial = static_cast(position % UINT32_MAX) + 1; + if (!epochAdvances) + return; + + for (; epochAdvances; --epochAdvances) + if (!++m_epoch) + ++m_epoch; + m_schedule.epoch = m_epoch; + + Client * client = FindClient(m_schedule.clientID); + if (client) + client->lastFeedbackDeadlineSerial = 0; + memset(m_publications, 0, sizeof(m_publications)); + m_publicationIndex = 0; } void CFrameScheduler::AdvanceDeadline(uint64_t now) @@ -435,6 +512,7 @@ void CFrameScheduler::AdvanceDeadline(uint64_t now) const uint64_t periods = (target - m_nextDeadline) / m_schedule.period + 1; m_nextDeadline += periods * m_schedule.period; + AdvanceDeadlineSerial(periods); } void CFrameScheduler::AdvanceDelivery(Client& client, uint64_t now) @@ -520,6 +598,9 @@ bool CFrameScheduler::GetPublishTarget(uint64_t now, uint64_t& target, schedule.republishTicket = m_republishRequestTicket; republish = schedule.republishTicket != m_republishAckTicket; + schedule.deadline = m_nextDeadline; + schedule.deadlineSerial = m_deadlineSerial; + const uint64_t lead = PublicationLead(m_schedule.targetSlack, m_workEstimate); const uint64_t periodicTarget = m_nextDeadline > lead ? @@ -528,22 +609,45 @@ bool CFrameScheduler::GetPublishTarget(uint64_t now, uint64_t& target, if (schedule.forceTicket != m_forceAckTicket) { periodic = periodicTarget <= now; + if (periodic) + target = periodicTarget; + schedule.deliveryDeadlineSerial = periodic ? m_deadlineSerial : 0; + schedule.phaseEligible = periodic; ReleaseSRWLockExclusive(&m_lock); return true; } - periodic = true; - if (periodicTarget <= now) - { - ReleaseSRWLockExclusive(&m_lock); - return true; - } - - target = periodicTarget; + periodic = true; + schedule.deliveryDeadlineSerial = m_deadlineSerial; + schedule.phaseEligible = true; + target = periodicTarget; ReleaseSRWLockExclusive(&m_lock); return true; } +void CFrameScheduler::FrameMissed(const Schedule& schedule, uint64_t now, + bool periodic) +{ + if (!periodic) + return; + + AcquireSRWLockExclusive(&m_lock); + if (m_scheduling && schedule.clientID == m_schedule.clientID && + schedule.generation == m_schedule.generation && + schedule.epoch == m_schedule.epoch && + schedule.deadlineSerial == m_deadlineSerial && + schedule.deadline == m_nextDeadline) + { + AdvanceCurrentDeadline(); + AdvanceDeadline(now); + + Client * client = FindClient(m_schedule.clientID); + if (client) + client->nextDelivery = m_nextDeadline; + } + ReleaseSRWLockExclusive(&m_lock); +} + void CFrameScheduler::FrameSuperseded() { AcquireSRWLockExclusive(&m_lock); @@ -551,14 +655,52 @@ void CFrameScheduler::FrameSuperseded() ReleaseSRWLockExclusive(&m_lock); } +bool CFrameScheduler::TryFrameSubmitted(const Schedule& schedule, + uint32_t frameSerial) +{ + if (!schedule.phaseEligible || !schedule.deliveryDeadlineSerial || + schedule.deliveryDeadlineSerial != schedule.deadlineSerial) + return false; + + if (!TryAcquireSRWLockExclusive(&m_lock)) + return false; + + bool registered = false; + if (m_scheduling && schedule.clientID == m_schedule.clientID && + schedule.generation == m_schedule.generation && + schedule.epoch == m_schedule.epoch && + schedule.deadlineSerial == m_deadlineSerial && + schedule.deadline == m_nextDeadline) + { + Publication& publication = + m_publications[m_publicationIndex++ % PUBLICATION_HISTORY_SIZE]; + publication = {}; + publication.generation = schedule.generation; + publication.epoch = schedule.epoch; + publication.deadlineSerial = schedule.deadlineSerial; + publication.frameSerial = frameSerial; + publication.deadline = schedule.deadline; + publication.committed = true; + registered = true; + } + ReleaseSRWLockExclusive(&m_lock); + return registered; +} + void CFrameScheduler::FramePublished(const Schedule& schedule, uint32_t frameSerial, uint64_t now, bool periodic) { AcquireSRWLockExclusive(&m_lock); if (m_scheduling && schedule.clientID == m_schedule.clientID && schedule.generation == m_schedule.generation && - schedule.epoch == m_schedule.epoch) + schedule.epoch == m_schedule.epoch && + schedule.deadlineSerial == m_deadlineSerial && + schedule.deadline == m_nextDeadline) { + Publication * publication = FindPublication(schedule, frameSerial); + if (publication) + publication->committed = true; + if (schedule.forceTicket > m_forceAckTicket) m_forceAckTicket = schedule.forceTicket; if (schedule.republishTicket > m_republishAckTicket) @@ -572,11 +714,10 @@ void CFrameScheduler::FramePublished(const Schedule& schedule, client->lastDeliveredFrameSerial = frameSerial; client->deliveredFrameValid = true; } - m_lastPublishedFrameSerial = frameSerial; ++m_publishedFrames; if (periodic) { - m_nextDeadline += m_schedule.period; + AdvanceCurrentDeadline(); AdvanceDeadline(now); } if (client) @@ -585,6 +726,34 @@ void CFrameScheduler::FramePublished(const Schedule& schedule, ReleaseSRWLockExclusive(&m_lock); } +bool CFrameScheduler::TryFrameCompleted(const Schedule& schedule, + uint32_t frameSerial, uint64_t completedAt) +{ + if (!schedule.phaseEligible || !schedule.deliveryDeadlineSerial || + schedule.deliveryDeadlineSerial != schedule.deadlineSerial || + !schedule.deadline) + return false; + + if (!TryAcquireSRWLockExclusive(&m_lock)) + return false; + + bool phaseValid = false; + if (m_scheduling && schedule.clientID == m_schedule.clientID && + schedule.generation == m_schedule.generation && + schedule.epoch == m_schedule.epoch) + { + Publication * publication = FindPublication(schedule, frameSerial); + if (publication && publication->committed) + { + publication->completed = true; + publication->phaseValid = completedAt <= publication->deadline; + phaseValid = publication->phaseValid; + } + } + ReleaseSRWLockExclusive(&m_lock); + return phaseValid; +} + void CFrameScheduler::FrameRepublished(const Schedule& schedule, uint32_t frameSerial) { @@ -605,7 +774,6 @@ void CFrameScheduler::FrameRepublished(const Schedule& schedule, client->deliveredFrameValid = true; client->nextDelivery = m_nextDeadline; } - m_lastPublishedFrameSerial = frameSerial; ++m_publishedFrames; } ReleaseSRWLockExclusive(&m_lock); @@ -765,12 +933,14 @@ void CFrameScheduler::LogStatistics(uint64_t now) ReleaseSRWLockExclusive(&m_lock); } -void CFrameScheduler::RecordFrameTiming(uint64_t duration) +void CFrameScheduler::TryRecordFrameTiming(uint64_t duration) { if (!duration) return; - AcquireSRWLockExclusive(&m_lock); + if (!TryAcquireSRWLockExclusive(&m_lock)) + return; + if (!m_workEstimate || duration > m_workEstimate) m_workEstimate = duration; else diff --git a/idd/LGIdd/CFrameScheduler.h b/idd/LGIdd/CFrameScheduler.h index 826b9919..22458089 100644 --- a/idd/LGIdd/CFrameScheduler.h +++ b/idd/LGIdd/CFrameScheduler.h @@ -39,8 +39,12 @@ public: uint32_t epoch; uint64_t period; uint64_t targetSlack; + uint64_t deadline; uint64_t forceTicket; uint64_t republishTicket; + uint32_t deadlineSerial; + uint32_t deliveryDeadlineSerial; + bool phaseEligible; }; private: @@ -52,7 +56,7 @@ private: uint64_t targetSlack; uint64_t expiry; uint64_t nextDelivery; - uint32_t lastFeedbackFrameSerial; + uint32_t lastFeedbackDeadlineSerial; uint32_t lastDeliveredFrameSerial; bool subscribed; bool ownerCapable; @@ -62,6 +66,21 @@ private: bool deliveredFrameValid; }; + struct Publication + { + uint32_t generation; + uint32_t epoch; + uint32_t deadlineSerial; + uint32_t frameSerial; + uint64_t deadline; + bool committed; + bool completed; + bool phaseValid; + bool accepted; + }; + + static const unsigned PUBLICATION_HISTORY_SIZE = 128; + mutable SRWLOCK m_lock = SRWLOCK_INIT; HANDLE m_wakeEvent = nullptr; Client m_clients[LGMP_MAX_CLIENTS] = {}; @@ -75,12 +94,15 @@ private: uint64_t m_republishRequestTicket = 0; uint64_t m_republishAckTicket = 0; - uint64_t m_lastArrival = 0; - uint64_t m_guestPeriod = 0; - uint64_t m_workEstimate = 0; - uint64_t m_nextDeadline = 0; + uint64_t m_lastArrival = 0; + uint64_t m_guestPeriod = 0; + uint64_t m_workEstimate = 0; + uint64_t m_nextDeadline = 0; + uint32_t m_deadlineSerial = 0; + int64_t m_pendingCorrection = 0; - uint32_t m_lastPublishedFrameSerial = 0; + Publication m_publications[PUBLICATION_HISTORY_SIZE] = {}; + unsigned m_publicationIndex = 0; int64_t m_lastPhaseError = 0; uint64_t m_acquiredFrames = 0; @@ -92,8 +114,12 @@ private: uint64_t m_lastLogPublished = 0; Client * FindClient(uint32_t clientID); - bool ElectOwner(uint64_t now); + Publication * FindPublication(const Schedule& schedule, + uint32_t frameSerial); + bool ElectOwner(uint64_t now, uint32_t resetClientID = 0); bool ApplyFeedback(Client& client, const KVMFRFrameSchedule& schedule); + void AdvanceCurrentDeadline(); + void AdvanceDeadlineSerial(uint64_t count); void AdvanceDeadline(uint64_t now); static void AdvanceDelivery(Client& client, uint64_t now); void WakePublisher() const; @@ -115,10 +141,14 @@ public: void ForceFrame(); bool GetPublishTarget(uint64_t now, uint64_t& target, Schedule& schedule, bool& periodic, bool& republish); + void FrameMissed(const Schedule& schedule, uint64_t now, bool periodic); void FrameSuperseded(); + bool TryFrameSubmitted(const Schedule& schedule, uint32_t frameSerial); void FramePublished(const Schedule& schedule, uint32_t frameSerial, uint64_t now, bool periodic); void FrameRepublished(const Schedule& schedule, uint32_t frameSerial); + bool TryFrameCompleted(const Schedule& schedule, uint32_t frameSerial, + uint64_t completedAt); unsigned GetSecondaryRecipients(const uint32_t * clientIDs, unsigned count, uint32_t frameSerial, uint64_t now, uint32_t * recipients) const; @@ -128,6 +158,6 @@ public: void FrameDelivered(const uint32_t * clientIDs, unsigned count, uint32_t frameSerial, uint64_t now); void NotifyPublisher() const { WakePublisher(); } - void RecordFrameTiming(uint64_t duration); + void TryRecordFrameTiming(uint64_t duration); void LogStatistics(uint64_t now); }; diff --git a/idd/LGIdd/CIndirectDeviceContext.cpp b/idd/LGIdd/CIndirectDeviceContext.cpp index 54e16151..107d9bb5 100644 --- a/idd/LGIdd/CIndirectDeviceContext.cpp +++ b/idd/LGIdd/CIndirectDeviceContext.cpp @@ -47,8 +47,8 @@ static const struct LGMPQueueConfig POINTER_QUEUE_CONFIG = static uint64_t FrameScheduleToken( const CFrameScheduler::Schedule& schedule) { - return static_cast(schedule.generation) << 32 | - schedule.epoch; + return static_cast(schedule.epoch) << 32 | + schedule.deliveryDeadlineSerial; } static bool FrameScheduleMatches( @@ -1884,11 +1884,16 @@ CIndirectDeviceContext::PreparedFrameBuffer CIndirectDeviceContext::PrepareFrame // fi->offset is initialized at startup fi->flags = flags; fi->sdrWhiteLevel = dstFormat.sdrWhiteLevel; - fi->captureTime = 0; - fi->postProcessTime = 0; - fi->copyTime = 0; - fi->readyTime = 0; - fi->timingSerial = 0; + + fi->captureTime = 0; + fi->postProcessTime = 0; + fi->copyTime = 0; + fi->readyTime = 0; + fi->timingSerial = 0; + fi->timingFlags = 0; + fi->scheduleGeneration = 0; + fi->scheduleEpoch = 0; + fi->scheduleDeadlineSerial = 0; InterlockedExchange((volatile LONG *)&fi->timingValid, 0); fi->rotation = FRAME_ROT_0; fi->type = dstFormat.format; @@ -1953,6 +1958,12 @@ bool CIndirectDeviceContext::PublishFrameBuffer(unsigned frameIndex, return false; } + KVMFRFrame * frame = m_frame[frameIndex]; + frame->timingFlags = 0; + frame->scheduleGeneration = schedule.generation; + frame->scheduleEpoch = schedule.epoch; + frame->scheduleDeadlineSerial = schedule.deliveryDeadlineSerial; + LGMP_STATUS status = LGMP_OK; bool published = false; if (schedule.clientID) @@ -2036,7 +2047,10 @@ bool CIndirectDeviceContext::RepublishFrameBuffer( return false; } - const uint64_t scheduleToken = FrameScheduleToken(schedule); + CFrameScheduler::Schedule deliverySchedule = schedule; + deliverySchedule.deliveryDeadlineSerial = 0; + deliverySchedule.phaseEligible = false; + const uint64_t scheduleToken = FrameScheduleToken(deliverySchedule); const uint32_t frameSerial = m_frame[frameIndex]->frameSerial; if (HasMatchingOwnerDelivery(schedule.clientID, static_cast(frameIndex), scheduleToken)) @@ -2057,7 +2071,7 @@ bool CIndirectDeviceContext::RepublishFrameBuffer( if (ownerQueueIndex < 0) { const bool published = PostSharedOwnerFrame( - static_cast(frameIndex), schedule); + static_cast(frameIndex), deliverySchedule); ReleaseSRWLockExclusive(&m_framePublishLock); if (published) m_frameScheduler.FrameRepublished(schedule, frameSerial); @@ -2106,6 +2120,16 @@ void CIndirectDeviceContext::CommitFrameBuffer(unsigned frameIndex, CFrameScheduler::Nanotime(), periodic); } +bool CIndirectDeviceContext::TryFrameSubmitted(unsigned frameIndex, + const CFrameScheduler::Schedule& schedule) +{ + if (frameIndex >= LGMP_Q_FRAME_BUFFER_LEN) + return false; + + return m_frameScheduler.TryFrameSubmitted( + schedule, m_frame[frameIndex]->frameSerial); +} + void CIndirectDeviceContext::ObserveFrame(uint64_t now) { m_frameScheduler.ObserveFrame(now); @@ -2124,14 +2148,20 @@ bool CIndirectDeviceContext::GetPublishTarget(uint64_t now, now, target, schedule, periodic, republish); } +void CIndirectDeviceContext::FrameMissed( + const CFrameScheduler::Schedule& schedule, uint64_t now, bool periodic) +{ + m_frameScheduler.FrameMissed(schedule, now, periodic); +} + void CIndirectDeviceContext::FrameSuperseded() { m_frameScheduler.FrameSuperseded(); } -void CIndirectDeviceContext::RecordFrameTiming(uint64_t duration) +void CIndirectDeviceContext::TryRecordFrameTiming(uint64_t duration) { - m_frameScheduler.RecordFrameTiming(duration); + m_frameScheduler.TryRecordFrameTiming(duration); } void CIndirectDeviceContext::AbortFrameBuffer(unsigned frameIndex) @@ -2183,17 +2213,23 @@ void CIndirectDeviceContext::CompleteFrameBuffer( void CIndirectDeviceContext::SetFrameTiming(unsigned frameIndex, uint64_t captureTime, uint64_t postProcessTime, uint64_t copyTime, - uint64_t readyTime) + uint64_t readyTime, const CFrameScheduler::Schedule& schedule, + uint64_t completedAt) { if (frameIndex >= LGMP_Q_FRAME_BUFFER_LEN) return; - KVMFRFrame * frame = m_frame[frameIndex]; - frame->captureTime = captureTime; - frame->postProcessTime = postProcessTime; - frame->copyTime = copyTime; - frame->readyTime = readyTime; - frame->timingSerial = frame->frameSerial; + KVMFRFrame * frame = m_frame[frameIndex]; + const bool phaseValid = m_frameScheduler.TryFrameCompleted( + schedule, frame->frameSerial, completedAt); + + frame->captureTime = captureTime; + frame->postProcessTime = postProcessTime; + frame->copyTime = copyTime; + frame->readyTime = readyTime; + frame->timingFlags = phaseValid ? + KVMFR_FRAME_TIMING_PHASE_VALID : 0; + frame->timingSerial = frame->frameSerial; InterlockedExchange((volatile LONG *)&frame->timingValid, 1); } diff --git a/idd/LGIdd/CIndirectDeviceContext.h b/idd/LGIdd/CIndirectDeviceContext.h index 2eaa377a..74cd64ad 100644 --- a/idd/LGIdd/CIndirectDeviceContext.h +++ b/idd/LGIdd/CIndirectDeviceContext.h @@ -282,13 +282,16 @@ public: bool PublishFrameBuffer(unsigned frameIndex, const CFrameScheduler::Schedule& schedule); bool RepublishFrameBuffer(const CFrameScheduler::Schedule& schedule); + bool TryFrameSubmitted(unsigned frameIndex, + const CFrameScheduler::Schedule& schedule); void CommitFrameBuffer(unsigned frameIndex, const CFrameScheduler::Schedule& schedule, bool periodic); void AbortFrameBuffer(unsigned frameIndex); void FailFrameBuffer(unsigned frameIndex); void CompleteFrameBuffer(unsigned frameIndex, bool succeeded); void SetFrameTiming(unsigned frameIndex, uint64_t captureTime, - uint64_t postProcessTime, uint64_t copyTime, uint64_t readyTime); + uint64_t postProcessTime, uint64_t copyTime, uint64_t readyTime, + const CFrameScheduler::Schedule& schedule, uint64_t completedAt); void WriteFrameBuffer(unsigned frameIndex, void* src, size_t offset, size_t len, bool setWritePos) const; void FinalizeFrameBuffer(unsigned frameIndex) const; @@ -296,12 +299,14 @@ public: void ForceFrame(); bool GetPublishTarget(uint64_t now, uint64_t& target, CFrameScheduler::Schedule& schedule, bool& periodic, bool& republish); + void FrameMissed(const CFrameScheduler::Schedule& schedule, + uint64_t now, bool periodic); void FrameSuperseded(); HANDLE GetFrameScheduleEvent() const { return m_frameScheduler.GetWakeEvent(); } - void RecordFrameTiming(uint64_t duration); + void TryRecordFrameTiming(uint64_t duration); void SendCursor(const IDARG_OUT_QUERY_HWCURSOR & info, const BYTE * data, UINT sdrWhiteLevel); diff --git a/idd/LGIdd/CSwapChainProcessor.cpp b/idd/LGIdd/CSwapChainProcessor.cpp index 58f24a1d..33037b68 100644 --- a/idd/LGIdd/CSwapChainProcessor.cpp +++ b/idd/LGIdd/CSwapChainProcessor.cpp @@ -327,6 +327,16 @@ void CSwapChainProcessor::PublisherThread() } const uint64_t replayNow = CFrameScheduler::Nanotime(); + uint64_t cadenceTarget = 0; + if (schedule.deliveryDeadlineSerial && periodic && + schedule.deadline <= replayNow) + { + m_devContext->FrameMissed(schedule, replayNow, periodic); + continue; + } + if (schedule.deliveryDeadlineSerial && periodic) + cadenceTarget = schedule.deadline; + uint64_t replayTarget; if (m_devContext->GetSharedFrameTarget(replayNow, replayTarget)) { @@ -335,18 +345,28 @@ void CSwapChainProcessor::PublisherThread() { if (m_devContext->ReplaySharedFrame(replayNow, retry)) continue; - if (!retry) + if (retry) + replayTarget = replayNow + PUBLISH_RETRY_NS; + else { - if (m_publishTimer.Get()) - CancelWaitableTimer(m_publishTimer.Get()); - if (WaitForMultipleObjects( - ARRAYSIZE(idleHandles), idleHandles, FALSE, INFINITE) == - WAIT_OBJECT_0) - break; - continue; + if (cadenceTarget) + replayTarget = cadenceTarget; + else + { + if (m_publishTimer.Get()) + CancelWaitableTimer(m_publishTimer.Get()); + if (WaitForMultipleObjects( + ARRAYSIZE(idleHandles), idleHandles, FALSE, INFINITE) == + WAIT_OBJECT_0) + break; + continue; + } } } + if (cadenceTarget && cadenceTarget < replayTarget) + replayTarget = cadenceTarget; + const uint64_t delay = replayTarget > replayNow ? replayTarget - replayNow : PUBLISH_RETRY_NS; ArmPublishTimer(m_publishTimer.Get(), delay); @@ -357,6 +377,17 @@ void CSwapChainProcessor::PublisherThread() continue; } + if (cadenceTarget) + { + ArmPublishTimer( + m_publishTimer.Get(), cadenceTarget - replayNow); + if (WaitForMultipleObjects( + ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) == + WAIT_OBJECT_0) + break; + continue; + } + if (m_publishTimer.Get()) CancelWaitableTimer(m_publishTimer.Get()); if (WaitForMultipleObjects( @@ -711,10 +742,7 @@ void CSwapChainProcessor::CompletionFunction( const bool gpuTimingValid = slot->GetGPUTimes(gpuCopyStart, gpuCopyEnd); - // Publish readiness before sampling the endpoint. Timing has its own valid - // flag and is published immediately afterwards. - sc->m_devContext->FinalizeFrameBuffer(fbRes->GetFrameIndex()); - const uint64_t readyEnd = CFrameScheduler::Nanotime(); + const uint64_t copyReady = CFrameScheduler::Nanotime(); const uint64_t postProcessStart = fbRes->GetPostProcessStart(); uint64_t postProcessTime = prepareCopyStart - postProcessStart; @@ -726,31 +754,41 @@ void CSwapChainProcessor::CompletionFunction( prepareCopyTime = prepareGPUEnd - prepareGPUStart; } - uint64_t publishCopyTime = readyEnd - publishStart; + uint64_t publishCopyTime = copyReady - publishStart; if (gpuTimingValid && gpuCopyStart >= publishStart && - gpuCopyEnd >= gpuCopyStart && gpuCopyEnd <= readyEnd) + gpuCopyEnd >= gpuCopyStart && gpuCopyEnd <= copyReady) publishCopyTime = gpuCopyEnd - gpuCopyStart + indirectCopyTime; const uint64_t copyTime = prepareCopyTime + publishCopyTime; - const uint64_t elapsed = readyEnd - postProcessStart; - const uint64_t measured = postProcessTime + copyTime; - const uint64_t readyTime = elapsed > measured ? elapsed - measured : 0; + + // Make the framebuffer readable before phase bookkeeping. If the scheduler + // lock is busy, the frame is still delivered and only this phase sample is + // discarded. + sc->m_devContext->FinalizeFrameBuffer(fbRes->GetFrameIndex()); + const uint64_t publishedAt = CFrameScheduler::Nanotime(); + const uint64_t elapsed = publishedAt - postProcessStart; + const uint64_t measured = postProcessTime + copyTime; + const uint64_t readyTime = elapsed > measured ? elapsed - measured : 0; + + sc->m_devContext->SetFrameTiming(fbRes->GetFrameIndex(), + fbRes->GetCaptureTime(), postProcessTime, copyTime, readyTime, + fbRes->GetSchedule(), publishedAt); + sc->m_devContext->TryRecordFrameTiming( + publishedAt - publishStart); // Use matching wall-clock boundaries for both modes. The split excludes the // cadence hold while including the indirect CPU copy only when it occurs. const uint64_t timingToken = fbRes->GetTimingToken(); if (timingToken && timingStart && prepareReady >= timingStart && - readyEnd >= publishStart) + copyReady >= publishStart) { const uint64_t totalTime = - (prepareReady - timingStart) + (readyEnd - publishStart); + (prepareReady - timingStart) + (copyReady - publishStart); sc->m_postProcessors[candidateIndex].RecordTiming( fbRes->GetTimingEffectIndex(), timingToken, fbRes->IsFullCopy(), totalTime); } - sc->m_devContext->RecordFrameTiming(readyEnd - publishStart); - sc->m_devContext->SetFrameTiming(fbRes->GetFrameIndex(), - fbRes->GetCaptureTime(), postProcessTime, copyTime, readyTime); + sc->m_devContext->CompleteFrameBuffer(fbRes->GetFrameIndex(), true); sc->ReleaseCandidate(candidateIndex); } @@ -1270,6 +1308,12 @@ bool CSwapChainProcessor::PublishNewestCandidate( restoreCandidates(); return false; } + CFrameScheduler::Schedule frameSchedule = schedule; + // Phase accounting must never hold up D3D submission. Keep the immutable + // delivery identity and discard only this feedback sample on contention. + if (!m_devContext->TryFrameSubmitted(buffer.frameIndex, schedule)) + frameSchedule.phaseEligible = false; + fbRes->SetSchedule(frameSchedule); // Retire the candidate damage before submission. The completion callback // may run before Execute returns and make this candidate reusable. diff --git a/obs/frame_scheduler.c b/obs/frame_scheduler.c index 122ffb11..0caa2478 100644 --- a/obs/frame_scheduler.c +++ b/obs/frame_scheduler.c @@ -61,13 +61,14 @@ void lgFrameSchedulerSetPeriod(LGFrameScheduler * scheduler, return; ++scheduler->generation; - scheduler->resetPending = true; - scheduler->immediatePending = true; - scheduler->phaseError = 0; - scheduler->feedbackFrameSerial = 0; - scheduler->feedbackScheduleEpoch = 0; - scheduler->feedbackSamples = 0; - scheduler->feedbackDirty = false; + scheduler->resetPending = true; + scheduler->immediatePending = true; + scheduler->phaseError = 0; + scheduler->feedbackFrameSerial = 0; + scheduler->feedbackScheduleEpoch = 0; + scheduler->feedbackDeadlineSerial = 0; + scheduler->feedbackSamples = 0; + scheduler->feedbackDirty = false; } void lgFrameSchedulerRequestImmediate(LGFrameScheduler * scheduler) @@ -78,31 +79,35 @@ void lgFrameSchedulerRequestImmediate(LGFrameScheduler * scheduler) void lgFrameSchedulerObserveFrame(LGFrameScheduler * scheduler, uint32_t frameSerial, uint32_t generation, uint32_t scheduleEpoch, - uint64_t readyTime) + uint32_t deadlineSerial, uint64_t readyTime) { - if (!generation || !scheduleEpoch || + if (!generation || !scheduleEpoch || !deadlineSerial || (scheduler->readyFrameSerial == frameSerial && scheduler->readyGeneration == generation && - scheduler->readyScheduleEpoch == scheduleEpoch)) + scheduler->readyScheduleEpoch == scheduleEpoch && + scheduler->readyDeadlineSerial == deadlineSerial)) return; - scheduler->readyFrameSerial = frameSerial; - scheduler->readyGeneration = generation; - scheduler->readyScheduleEpoch = scheduleEpoch; - scheduler->readyTime = readyTime; + scheduler->readyFrameSerial = frameSerial; + scheduler->readyGeneration = generation; + scheduler->readyScheduleEpoch = scheduleEpoch; + scheduler->readyDeadlineSerial = deadlineSerial; + scheduler->readyTime = readyTime; } void lgFrameSchedulerFeedback(LGFrameScheduler * scheduler, uint32_t frameSerial, uint32_t generation, uint32_t scheduleEpoch, - uint64_t tickTime) + uint32_t deadlineSerial, uint64_t tickTime) { if (!scheduler->active || generation != scheduler->generation || - !scheduleEpoch || + !scheduleEpoch || !deadlineSerial || (frameSerial == scheduler->feedbackFrameSerial && - scheduleEpoch == scheduler->feedbackScheduleEpoch) || + scheduleEpoch == scheduler->feedbackScheduleEpoch && + deadlineSerial == scheduler->feedbackDeadlineSerial) || scheduler->readyFrameSerial != frameSerial || scheduler->readyGeneration != generation || - scheduler->readyScheduleEpoch != scheduleEpoch) + scheduler->readyScheduleEpoch != scheduleEpoch || + scheduler->readyDeadlineSerial != deadlineSerial) return; if (scheduler->feedbackScheduleEpoch && @@ -132,7 +137,8 @@ void lgFrameSchedulerFeedback(LGFrameScheduler * scheduler, ++scheduler->feedbackSamples; scheduler->feedbackFrameSerial = frameSerial; - scheduler->feedbackScheduleEpoch = scheduleEpoch; + scheduler->feedbackScheduleEpoch = scheduleEpoch; + scheduler->feedbackDeadlineSerial = deadlineSerial; scheduler->feedbackDirty = true; } @@ -159,17 +165,20 @@ void lgFrameSchedulerUpdate(LGFrameScheduler * scheduler, scheduler->feedbackFrameSerial; const uint32_t feedbackScheduleEpoch = scheduler->feedbackScheduleEpoch; + const uint32_t feedbackDeadlineSerial = + scheduler->feedbackDeadlineSerial; const KVMFRFrameSchedule message = { - .msg.type = KVMFR_MESSAGE_FRAME_SCHEDULE, - .clientID = scheduler->clientID, - .generation = scheduler->generation, - .flags = flags, - .period = scheduler->period, - .targetSlack = FRAME_SCHEDULER_TARGET_SLACK_NS, - .phaseError = scheduler->phaseError, - .feedbackFrameSerial = feedbackFrameSerial, - .feedbackScheduleEpoch = feedbackScheduleEpoch, - .lease = FRAME_SCHEDULER_LEASE_MS, + .msg.type = KVMFR_MESSAGE_FRAME_SCHEDULE, + .clientID = scheduler->clientID, + .generation = scheduler->generation, + .flags = flags, + .period = scheduler->period, + .targetSlack = FRAME_SCHEDULER_TARGET_SLACK_NS, + .phaseError = scheduler->phaseError, + .feedbackFrameSerial = feedbackFrameSerial, + .feedbackScheduleEpoch = feedbackScheduleEpoch, + .feedbackDeadlineSerial = feedbackDeadlineSerial, + .lease = FRAME_SCHEDULER_LEASE_MS, }; if (lgmpClientSendData(queue, &message, sizeof(message), NULL) != LGMP_OK) @@ -180,6 +189,7 @@ void lgFrameSchedulerUpdate(LGFrameScheduler * scheduler, scheduler->immediatePending = false; scheduler->lastSend = now; if (scheduler->feedbackFrameSerial == feedbackFrameSerial && - scheduler->feedbackScheduleEpoch == feedbackScheduleEpoch) + scheduler->feedbackScheduleEpoch == feedbackScheduleEpoch && + scheduler->feedbackDeadlineSerial == feedbackDeadlineSerial) scheduler->feedbackDirty = false; } diff --git a/obs/frame_scheduler.h b/obs/frame_scheduler.h index 8fa14a36..d7272343 100644 --- a/obs/frame_scheduler.h +++ b/obs/frame_scheduler.h @@ -41,11 +41,13 @@ typedef struct LGFrameScheduler int64_t phaseError; uint32_t feedbackFrameSerial; uint32_t feedbackScheduleEpoch; + uint32_t feedbackDeadlineSerial; unsigned feedbackSamples; uint32_t readyFrameSerial; uint32_t readyGeneration; uint32_t readyScheduleEpoch; + uint32_t readyDeadlineSerial; uint64_t readyTime; } LGFrameScheduler; @@ -57,10 +59,10 @@ void lgFrameSchedulerSetPeriod(LGFrameScheduler * scheduler, void lgFrameSchedulerRequestImmediate(LGFrameScheduler * scheduler); void lgFrameSchedulerObserveFrame(LGFrameScheduler * scheduler, uint32_t frameSerial, uint32_t generation, uint32_t scheduleEpoch, - uint64_t readyTime); + uint32_t deadlineSerial, uint64_t readyTime); void lgFrameSchedulerFeedback(LGFrameScheduler * scheduler, uint32_t frameSerial, uint32_t generation, uint32_t scheduleEpoch, - uint64_t tickTime); + uint32_t deadlineSerial, uint64_t tickTime); void lgFrameSchedulerUpdate(LGFrameScheduler * scheduler, PLGMPClientQueue queue, uint64_t now); diff --git a/obs/lg.c b/obs/lg.c index 78b48f75..8137bad3 100644 --- a/obs/lg.c +++ b/obs/lg.c @@ -43,6 +43,8 @@ #include "cursor.effect.h" #include "frame_scheduler.h" +#define LG_FRAME_TIMING_SPIN_COUNT 4096 + /* scRGB reference white in cd/m² (OBS GS_CS_709_SCRGB is defined as * 1.0 = 80 cd/m²). PQ encodes an absolute 0..10000 cd/m² range, so linear * PQ output is scaled by 10000 / 80 to reach the scRGB convention. */ @@ -180,8 +182,8 @@ typedef struct { LGMPMessage msg; PLGMPClientQueue queue; - uint32_t generation; uint32_t scheduleEpoch; + uint32_t scheduleDeadlineSerial; bool owner; } LGFrameMessage; @@ -210,6 +212,18 @@ static bool lgFrameSerialNewer(uint32_t lhs, uint32_t rhs) return (int32_t)(lhs - rhs) > 0; } +static bool lgFrameMessageHasMatchingDeadline( + bool owner, const LGMPMessage * message, const KVMFRFrame * frame) +{ + if (!owner) + return false; + + const uint32_t epoch = (uint32_t)(message->udata >> 32); + const uint32_t deadlineSerial = (uint32_t)message->udata; + return deadlineSerial && epoch == frame->scheduleEpoch && + deadlineSerial == frame->scheduleDeadlineSerial; +} + static LGMP_STATUS lgFrameProcessNewest(LGPlugin * this, LGFrameMessage * result) { @@ -260,7 +274,10 @@ static LGMP_STATUS lgFrameProcessNewest(LGPlugin * this, const bool replace = lgFrameSerialNewer( frame->frameSerial, selected->frameSerial) || (frame->frameSerial == selected->frameSerial && - owner && !result->owner); + ((owner && !result->owner) || + (lgFrameMessageHasMatchingDeadline(owner, &msg, frame) && + !lgFrameMessageHasMatchingDeadline( + result->owner, &result->msg, selected)))); PLGMPClientQueue discard = queues[i].queue; if (replace) { @@ -277,13 +294,43 @@ static LGMP_STATUS lgFrameProcessNewest(LGPlugin * this, if (result->owner) { - result->generation = (uint32_t)(result->msg.udata >> 32); - result->scheduleEpoch = (uint32_t)result->msg.udata; + result->scheduleEpoch = (uint32_t)(result->msg.udata >> 32); + result->scheduleDeadlineSerial = (uint32_t)result->msg.udata; } return result->queue ? LGMP_OK : LGMP_ERR_QUEUE_EMPTY; } +static bool lgFrameTimingReady(const KVMFRFrame * frame) +{ + return __atomic_load_n(&frame->timingValid, __ATOMIC_ACQUIRE) && + frame->timingSerial == frame->frameSerial; +} + +static bool lgFramePhaseIdentity(const LGFrameMessage * message, + const KVMFRFrame * frame, uint32_t * generation) +{ + if (!message->owner || !message->scheduleEpoch || + !message->scheduleDeadlineSerial) + return false; + + for (unsigned i = 0; + !lgFrameTimingReady(frame) && i < LG_FRAME_TIMING_SPIN_COUNT; + ++i) + { + } + + if (!lgFrameTimingReady(frame) || + !(frame->timingFlags & KVMFR_FRAME_TIMING_PHASE_VALID) || + !frame->scheduleGeneration || + frame->scheduleEpoch != message->scheduleEpoch || + frame->scheduleDeadlineSerial != message->scheduleDeadlineSerial) + return false; + + *generation = frame->scheduleGeneration; + return true; +} + static void lgFrameUnsubscribeOwnerQueues(LGPlugin * this) { for (unsigned int i = 0; i < LGMP_Q_FRAME_LEN; ++i) @@ -666,9 +713,11 @@ static void * frameThread(void * data) fb, (size_t)frame->dataHeight * frame->pitch)) { now = os_gettime_ns(); - lgFrameSchedulerObserveFrame(&this->frameScheduler, - frame->frameSerial, frameMessage.generation, - frameMessage.scheduleEpoch, now); + uint32_t generation; + if (lgFramePhaseIdentity(&frameMessage, frame, &generation)) + lgFrameSchedulerObserveFrame(&this->frameScheduler, + frame->frameSerial, generation, frameMessage.scheduleEpoch, + frameMessage.scheduleDeadlineSerial, now); } } } @@ -1557,12 +1606,16 @@ static void lgVideoTick(void * data, float seconds) } const uint64_t readyTime = os_gettime_ns(); - lgFrameSchedulerObserveFrame(&this->frameScheduler, - frame->frameSerial, frameMessage.generation, - frameMessage.scheduleEpoch, readyTime); - lgFrameSchedulerFeedback(&this->frameScheduler, - frame->frameSerial, frameMessage.generation, - frameMessage.scheduleEpoch, tickTime); + uint32_t generation; + if (lgFramePhaseIdentity(&frameMessage, frame, &generation)) + { + lgFrameSchedulerObserveFrame(&this->frameScheduler, + frame->frameSerial, generation, frameMessage.scheduleEpoch, + frameMessage.scheduleDeadlineSerial, readyTime); + lgFrameSchedulerFeedback(&this->frameScheduler, + frame->frameSerial, generation, frameMessage.scheduleEpoch, + frameMessage.scheduleDeadlineSerial, tickTime); + } } this->frameSerial = frame->frameSerial;