From 0b9b80553064d2503326772766f64a6068f0b3f5 Mon Sep 17 00:00:00 2001 From: Geoffrey McRae Date: Wed, 5 Aug 2026 12:38:07 +1000 Subject: [PATCH] [idd] scheduler: retain frames for publication deadlines Prepare each acquired content frame into one of two private GPU candidates so the IddCx frame can be released without copying it to IVSHMEM immediately. Wake a high-resolution publisher at the client deadline and send the newest completed candidate. Preserve accumulated damage across superseded candidates, include the scheduling hold in end-to-end timing, and keep transport work on the existing physical copy queue. --- idd/LGIdd/CFrameBufferResource.h | 3 + idd/LGIdd/CFrameScheduler.cpp | 124 ++-- idd/LGIdd/CFrameScheduler.h | 20 +- idd/LGIdd/CIndirectDeviceContext.cpp | 51 +- idd/LGIdd/CIndirectDeviceContext.h | 19 +- idd/LGIdd/CSwapChainProcessor.cpp | 954 +++++++++++++++++++++------ idd/LGIdd/CSwapChainProcessor.h | 58 +- 7 files changed, 953 insertions(+), 276 deletions(-) diff --git a/idd/LGIdd/CFrameBufferResource.h b/idd/LGIdd/CFrameBufferResource.h index e0c3ea09..bf5678d5 100644 --- a/idd/LGIdd/CFrameBufferResource.h +++ b/idd/LGIdd/CFrameBufferResource.h @@ -43,6 +43,7 @@ class CFrameBufferResource unsigned m_timingEffectIndex = 0; uint64_t m_timingToken = 0; bool m_fullCopy = false; + unsigned m_candidateIndex = 0; ComPtr m_res; void * m_map = nullptr; @@ -75,6 +76,8 @@ class CFrameBufferResource unsigned GetTimingEffectIndex() const { return m_timingEffectIndex; } uint64_t GetTimingToken () const { return m_timingToken; } bool IsFullCopy () const { return m_fullCopy; } + void SetCandidateIndex(unsigned index) { m_candidateIndex = index; } + unsigned GetCandidateIndex() const { return m_candidateIndex; } ComPtr Get() { return m_res; } }; diff --git a/idd/LGIdd/CFrameScheduler.cpp b/idd/LGIdd/CFrameScheduler.cpp index dbfed5db..82be4b46 100644 --- a/idd/LGIdd/CFrameScheduler.cpp +++ b/idd/LGIdd/CFrameScheduler.cpp @@ -30,6 +30,23 @@ static const uint64_t MIN_SAFETY_NS = 250000ULL; static const uint64_t LOG_INTERVAL_NS = 5000000000ULL; static const uint64_t CADENCE_BREAK = 4; +CFrameScheduler::CFrameScheduler() +{ + m_wakeEvent = CreateEvent(nullptr, FALSE, FALSE, nullptr); +} + +CFrameScheduler::~CFrameScheduler() +{ + if (m_wakeEvent) + CloseHandle(m_wakeEvent); +} + +void CFrameScheduler::WakePublisher() const +{ + if (m_wakeEvent) + SetEvent(m_wakeEvent); +} + uint64_t CFrameScheduler::Nanotime() { static const uint64_t frequency = []() @@ -58,7 +75,7 @@ CFrameScheduler::Client * CFrameScheduler::FindClient(uint32_t clientID) return nullptr; } -void CFrameScheduler::ElectOwner(uint64_t now) +bool CFrameScheduler::ElectOwner(uint64_t now) { Client * fastest = nullptr; Client * incumbent = FindClient(m_schedule.clientID); @@ -91,6 +108,8 @@ void CFrameScheduler::ElectOwner(uint64_t now) const uint32_t oldClientID = m_schedule.clientID; const uint32_t oldGeneration = m_schedule.generation; + const uint64_t oldPeriod = m_schedule.period; + const uint64_t oldSlack = m_schedule.targetSlack; if (!fastest) { m_schedule = {}; @@ -125,6 +144,9 @@ void CFrameScheduler::ElectOwner(uint64_t now) else if (ownerChanged && oldClientID) DEBUG_INFO("Frame timing owner released; using push delivery"); } + + return ownerChanged || oldGeneration != m_schedule.generation || + oldPeriod != m_schedule.period || oldSlack != m_schedule.targetSlack; } void CFrameScheduler::Reset() @@ -138,11 +160,8 @@ void CFrameScheduler::Reset() m_lastArrival = 0; m_guestPeriod = 0; - m_guestJitter = 0; m_workEstimate = 0; m_nextDeadline = 0; - m_arrivalSamples = 0; - m_timingSamples = 0; m_lastPublishedFrameSerial = 0; @@ -155,6 +174,7 @@ void CFrameScheduler::Reset() m_lastLogSkipped = 0; m_lastLogPublished = 0; ReleaseSRWLockExclusive(&m_lock); + WakePublisher(); } void CFrameScheduler::UpdateSubscribers(const uint32_t * clientIDs, @@ -185,8 +205,10 @@ void CFrameScheduler::UpdateSubscribers(const uint32_t * clientIDs, if (client.clientID && !client.subscribed) client = {}; - ElectOwner(now); + const bool changed = ElectOwner(now); ReleaseSRWLockExclusive(&m_lock); + if (changed) + WakePublisher(); } bool CFrameScheduler::UpdateSchedule(const KVMFRFrameSchedule& schedule, @@ -203,6 +225,7 @@ bool CFrameScheduler::UpdateSchedule(const KVMFRFrameSchedule& schedule, AcquireSRWLockExclusive(&m_lock); Client * client = FindClient(schedule.clientID); + bool wake = false; if (schedule.flags & KVMFR_FRAME_SCHEDULE_RELEASE) { @@ -210,9 +233,11 @@ bool CFrameScheduler::UpdateSchedule(const KVMFRFrameSchedule& schedule, { client->active = false; client->expiry = 0; - ElectOwner(now); + wake = ElectOwner(now); } ReleaseSRWLockExclusive(&m_lock); + if (wake) + WakePublisher(); return true; } @@ -242,14 +267,19 @@ bool CFrameScheduler::UpdateSchedule(const KVMFRFrameSchedule& schedule, client->expiry = now + static_cast(schedule.lease) * 1000000; client->active = true; if (schedule.flags & KVMFR_FRAME_SCHEDULE_IMMEDIATE) + { m_forceNext = true; - ElectOwner(now); - ApplyFeedback(*client, schedule); + wake = true; + } + wake |= ElectOwner(now); + wake |= ApplyFeedback(*client, schedule); ReleaseSRWLockExclusive(&m_lock); + if (wake) + WakePublisher(); return true; } -void CFrameScheduler::ApplyFeedback(Client& client, +bool CFrameScheduler::ApplyFeedback(Client& client, const KVMFRFrameSchedule& schedule) { if (!m_scheduling || client.clientID != m_schedule.clientID || @@ -261,7 +291,7 @@ void CFrameScheduler::ApplyFeedback(Client& client, !m_lastPublishedFrameSerial || static_cast(schedule.feedbackFrameSerial - m_lastPublishedFrameSerial) > 0) - return; + return false; int64_t correction = schedule.phaseError / 4; const int64_t limit = static_cast(m_schedule.period / 4); @@ -280,6 +310,7 @@ void CFrameScheduler::ApplyFeedback(Client& client, } m_lastPhaseError = schedule.phaseError; client.lastFeedbackFrameSerial = schedule.feedbackFrameSerial; + return correction != 0; } void CFrameScheduler::AdvanceDeadline(uint64_t now) @@ -311,35 +342,35 @@ void CFrameScheduler::ObserveFrame(uint64_t now) const uint64_t interval = now - m_lastArrival; if (m_guestPeriod && interval > m_guestPeriod * CADENCE_BREAK) { - m_guestPeriod = 0; - m_guestJitter = 0; - m_arrivalSamples = 0; - m_forceNext = true; + m_guestPeriod = 0; + m_forceNext = true; } else if (interval >= MIN_PERIOD_NS && interval <= MAX_PERIOD_NS) { if (!m_guestPeriod) m_guestPeriod = interval; else - { - const uint64_t error = m_guestPeriod > interval ? - m_guestPeriod - interval : interval - m_guestPeriod; m_guestPeriod = (m_guestPeriod * 7 + interval) / 8; - m_guestJitter = (m_guestJitter * 7 + error) / 8; - } - - if (m_arrivalSamples < 32) - ++m_arrivalSamples; } } m_lastArrival = now; ReleaseSRWLockExclusive(&m_lock); } -bool CFrameScheduler::SelectFrame(uint64_t now, bool force, - uint32_t& generation) +void CFrameScheduler::ForceFrame() { + AcquireSRWLockExclusive(&m_lock); + m_forceNext = true; + ReleaseSRWLockExclusive(&m_lock); + WakePublisher(); +} + +bool CFrameScheduler::GetPublishTarget(uint64_t now, uint64_t& target, + uint32_t& generation, bool& periodic) +{ + target = now; generation = 0; + periodic = false; AcquireSRWLockExclusive(&m_lock); if (!m_scheduling) { @@ -348,31 +379,41 @@ bool CFrameScheduler::SelectFrame(uint64_t now, bool force, } generation = m_schedule.generation; - AdvanceDeadline(now); - if (force) - m_forceNext = true; + if (m_forceNext) + { + periodic = m_nextDeadline <= now; + ReleaseSRWLockExclusive(&m_lock); + return true; + } - if (m_forceNext || m_arrivalSamples < 4 || m_timingSamples < 4) + periodic = true; + if (m_nextDeadline <= now) { ReleaseSRWLockExclusive(&m_lock); return true; } const uint64_t safety = - max(MIN_SAFETY_NS, - max(m_guestJitter * 2, m_workEstimate / 8)); - const uint64_t nextArrival = m_lastArrival + m_guestPeriod; - const bool process = nextArrival <= now || - nextArrival + m_workEstimate + safety > m_nextDeadline; - if (!process) - ++m_skippedFrames; + max(MIN_SAFETY_NS, m_workEstimate / 8); + const uint64_t lead = + m_schedule.targetSlack + m_workEstimate + safety; + target = m_nextDeadline > lead ? m_nextDeadline - lead : now; + if (target < now) + target = now; + ReleaseSRWLockExclusive(&m_lock); + return true; +} + +void CFrameScheduler::FrameSuperseded() +{ + AcquireSRWLockExclusive(&m_lock); + ++m_skippedFrames; ReleaseSRWLockExclusive(&m_lock); - return process; } void CFrameScheduler::FramePublished(uint32_t generation, - uint32_t frameSerial, uint64_t now) + uint32_t frameSerial, uint64_t now, bool periodic) { AcquireSRWLockExclusive(&m_lock); if (m_scheduling && generation == m_schedule.generation) @@ -380,8 +421,11 @@ void CFrameScheduler::FramePublished(uint32_t generation, m_forceNext = false; m_lastPublishedFrameSerial = frameSerial; ++m_publishedFrames; - m_nextDeadline += m_schedule.period; - AdvanceDeadline(now); + if (periodic) + { + m_nextDeadline += m_schedule.period; + AdvanceDeadline(now); + } } ReleaseSRWLockExclusive(&m_lock); } @@ -428,7 +472,5 @@ void CFrameScheduler::RecordFrameTiming(uint64_t duration) else m_workEstimate = (m_workEstimate * 31 + duration) / 32; - if (m_timingSamples < 32) - ++m_timingSamples; ReleaseSRWLockExclusive(&m_lock); } diff --git a/idd/LGIdd/CFrameScheduler.h b/idd/LGIdd/CFrameScheduler.h index 899257a2..1f6366c3 100644 --- a/idd/LGIdd/CFrameScheduler.h +++ b/idd/LGIdd/CFrameScheduler.h @@ -54,6 +54,7 @@ private: }; mutable SRWLOCK m_lock = SRWLOCK_INIT; + HANDLE m_wakeEvent = nullptr; Client m_clients[LGMP_MAX_CLIENTS] = {}; Schedule m_schedule = {}; bool m_scheduling = false; @@ -61,11 +62,8 @@ private: uint64_t m_lastArrival = 0; uint64_t m_guestPeriod = 0; - uint64_t m_guestJitter = 0; uint64_t m_workEstimate = 0; uint64_t m_nextDeadline = 0; - unsigned m_arrivalSamples = 0; - unsigned m_timingSamples = 0; uint32_t m_lastPublishedFrameSerial = 0; @@ -79,11 +77,15 @@ private: uint64_t m_lastLogPublished = 0; Client * FindClient(uint32_t clientID); - void ElectOwner(uint64_t now); - void ApplyFeedback(Client& client, const KVMFRFrameSchedule& schedule); + bool ElectOwner(uint64_t now); + bool ApplyFeedback(Client& client, const KVMFRFrameSchedule& schedule); void AdvanceDeadline(uint64_t now); + void WakePublisher() const; public: + CFrameScheduler(); + ~CFrameScheduler(); + static uint64_t Nanotime(); void Reset(); @@ -91,10 +93,14 @@ public: uint64_t now); bool UpdateSchedule(const KVMFRFrameSchedule& schedule, uint64_t now); bool GetSchedule(Schedule& schedule) const; + HANDLE GetWakeEvent() const { return m_wakeEvent; } void ObserveFrame(uint64_t now); - bool SelectFrame(uint64_t now, bool force, uint32_t& generation); + void ForceFrame(); + bool GetPublishTarget(uint64_t now, uint64_t& target, + uint32_t& generation, bool& periodic); + void FrameSuperseded(); void FramePublished(uint32_t generation, uint32_t frameSerial, - uint64_t now); + uint64_t now, bool periodic); void RecordFrameTiming(uint64_t duration); void LogStatistics(uint64_t now); }; diff --git a/idd/LGIdd/CIndirectDeviceContext.cpp b/idd/LGIdd/CIndirectDeviceContext.cpp index eb0853bb..555578b8 100644 --- a/idd/LGIdd/CIndirectDeviceContext.cpp +++ b/idd/LGIdd/CIndirectDeviceContext.cpp @@ -1246,7 +1246,10 @@ void CIndirectDeviceContext::LGMPTimer() } LGMP_STATUS status; - if ((status = lgmpHostProcess(m_lgmp)) != LGMP_OK) + AcquireSRWLockExclusive(&m_lgmpProcessLock); + status = lgmpHostProcess(m_lgmp); + ReleaseSRWLockExclusive(&m_lgmpProcessLock); + if (status != LGMP_OK) { if (status == LGMP_ERR_CORRUPTED) { @@ -1348,6 +1351,19 @@ bool CIndirectDeviceContext::FrameBufferAvailable() const return !m_frameInFlight[frameIndex].load(std::memory_order_acquire); } +void CIndirectDeviceContext::ProcessFrameQueue() +{ + if (!m_lgmp) + return; + + AcquireSRWLockExclusive(&m_lgmpProcessLock); + const LGMP_STATUS status = lgmpHostProcess(m_lgmp); + ReleaseSRWLockExclusive(&m_lgmpProcessLock); + + if (status != LGMP_OK && status != LGMP_ERR_CORRUPTED) + DEBUG_ERROR("lgmpHostProcess Failed: %s", lgmpStatusString(status)); +} + CIndirectDeviceContext::PreparedFrameBuffer CIndirectDeviceContext::PrepareFrameBuffer( unsigned pitch, const D12FrameFormat& srcFormat, const D12FrameFormat& dstFormat, const RECT * dirtyRects, unsigned nbDirtyRects) @@ -1524,11 +1540,6 @@ bool CIndirectDeviceContext::PublishFrameBuffer(unsigned frameIndex, } ReleaseSRWLockExclusive(&m_framePublishLock); - if (status == LGMP_OK) - m_frameScheduler.FramePublished( - scheduleGeneration, m_frame[frameIndex]->frameSerial, - CFrameScheduler::Nanotime()); - if (status != LGMP_OK) { DEBUG_ERROR("Failed to publish frame: %s", lgmpStatusString(status)); @@ -1538,15 +1549,37 @@ bool CIndirectDeviceContext::PublishFrameBuffer(unsigned frameIndex, return true; } +void CIndirectDeviceContext::CommitFrameBuffer(unsigned frameIndex, + uint32_t scheduleGeneration, bool periodic) +{ + if (frameIndex >= LGMP_Q_FRAME_LEN) + return; + + m_frameScheduler.FramePublished( + scheduleGeneration, m_frame[frameIndex]->frameSerial, + CFrameScheduler::Nanotime(), periodic); +} + void CIndirectDeviceContext::ObserveFrame(uint64_t now) { m_frameScheduler.ObserveFrame(now); } -bool CIndirectDeviceContext::SelectFrame(uint64_t now, bool force, - uint32_t& generation) +void CIndirectDeviceContext::ForceFrame() { - return m_frameScheduler.SelectFrame(now, force, generation); + m_frameScheduler.ForceFrame(); +} + +bool CIndirectDeviceContext::GetPublishTarget(uint64_t now, + uint64_t& target, uint32_t& generation, bool& periodic) +{ + return m_frameScheduler.GetPublishTarget( + now, target, generation, periodic); +} + +void CIndirectDeviceContext::FrameSuperseded() +{ + m_frameScheduler.FrameSuperseded(); } void CIndirectDeviceContext::RecordFrameTiming(uint64_t duration) diff --git a/idd/LGIdd/CIndirectDeviceContext.h b/idd/LGIdd/CIndirectDeviceContext.h index 78081816..d5133753 100644 --- a/idd/LGIdd/CIndirectDeviceContext.h +++ b/idd/LGIdd/CIndirectDeviceContext.h @@ -86,9 +86,10 @@ private: CIVSHMEM m_ivshmem; - PLGMPHost m_lgmp = nullptr; - WDFTIMER m_lgmpTimer = nullptr; - PLGMPHostQueue m_frameQueue = nullptr; + PLGMPHost m_lgmp = nullptr; + WDFTIMER m_lgmpTimer = nullptr; + PLGMPHostQueue m_frameQueue = nullptr; + SRWLOCK m_lgmpProcessLock = SRWLOCK_INIT; CFrameScheduler m_frameScheduler; @@ -224,8 +225,11 @@ public: }; bool FrameBufferAvailable() const; + void ProcessFrameQueue(); PreparedFrameBuffer PrepareFrameBuffer(unsigned pitch, const D12FrameFormat& srcFormat, const D12FrameFormat& dstFormat, const RECT * dirtyRects, unsigned nbDirtyRects); bool PublishFrameBuffer(unsigned frameIndex, uint32_t scheduleGeneration); + void CommitFrameBuffer(unsigned frameIndex, uint32_t scheduleGeneration, + bool periodic); void AbortFrameBuffer(unsigned frameIndex); void FailFrameBuffer(unsigned frameIndex); void CompleteFrameBuffer(unsigned frameIndex); @@ -235,7 +239,14 @@ public: void FinalizeFrameBuffer(unsigned frameIndex) const; void ObserveFrame(uint64_t now); - bool SelectFrame(uint64_t now, bool force, uint32_t& generation); + void ForceFrame(); + bool GetPublishTarget(uint64_t now, uint64_t& target, + uint32_t& generation, bool& periodic); + void FrameSuperseded(); + HANDLE GetFrameScheduleEvent() const + { + return m_frameScheduler.GetWakeEvent(); + } void RecordFrameTiming(uint64_t duration); void SendCursor(const IDARG_OUT_QUERY_HWCURSOR & info, const BYTE * data, diff --git a/idd/LGIdd/CSwapChainProcessor.cpp b/idd/LGIdd/CSwapChainProcessor.cpp index 62be9dd5..84261f06 100644 --- a/idd/LGIdd/CSwapChainProcessor.cpp +++ b/idd/LGIdd/CSwapChainProcessor.cpp @@ -25,12 +25,35 @@ #include "CDebug.h" #include "CPipeServer.h" +#ifndef CREATE_WAITABLE_TIMER_HIGH_RESOLUTION + #define CREATE_WAITABLE_TIMER_HIGH_RESOLUTION 0x00000002 +#endif + static const uint32_t HDR_PQ_MIN_LUMINANCE = 50; static const uint32_t HDR_PQ_MAX_LUMINANCE = 10000; +static const uint64_t PUBLISH_RETRY_NS = 1000000ULL; +static const DWORD CANDIDATE_WAIT_MS = 2; static_assert(LGMP_Q_FRAME_LEN == 2, "IDD damage repair assumes two alternating frame buffers"); +class CSRWExclusiveLock +{ +private: + SRWLOCK * m_lock; + +public: + explicit CSRWExclusiveLock(SRWLOCK * lock) : m_lock(lock) + { + AcquireSRWLockExclusive(m_lock); + } + + ~CSRWExclusiveLock() + { + ReleaseSRWLockExclusive(m_lock); + } +}; + static bool FrameMetadataChanged(const D12FrameFormat& previous, const D12FrameFormat& current) { @@ -96,14 +119,25 @@ CSwapChainProcessor::CSwapChainProcessor(CIndirectMonitorContext * monitorContex "Failed to initialize post-processing effects; effects disabled"); } - // Manual-reset: both worker threads wait on this, so it must stay signalled + // Manual-reset: all worker threads wait on this, so it must stay signalled // once set or only one thread would ever observe termination. m_terminateEvent.Attach(CreateEvent(nullptr, TRUE, FALSE, nullptr)); + m_candidateEvent.Attach(CreateEvent(nullptr, FALSE, FALSE, nullptr)); + m_candidateAvailableEvent.Attach( + CreateEvent(nullptr, FALSE, FALSE, nullptr)); + m_publishTimer.Attach(CreateWaitableTimerExW(nullptr, nullptr, + CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS)); + if (!m_publishTimer.Get()) + m_publishTimer.Attach(CreateWaitableTimerExW( + nullptr, nullptr, 0, TIMER_ALL_ACCESS)); m_cursorDataEvent.Attach(CreateEvent(nullptr, FALSE, FALSE, nullptr)); m_shapeBuffer = new BYTE[512 * 512 * 4]; // Start the worker only after every object it can access is initialized. - m_thread[0].Attach(CreateThread(nullptr, 0, _SwapChainThread, this, 0, nullptr)); + m_thread[0].Attach(CreateThread( + nullptr, 0, _SwapChainThread, this, 0, nullptr)); + m_thread[2].Attach(CreateThread( + nullptr, 0, _PublisherThread, this, 0, nullptr)); } CSwapChainProcessor::~CSwapChainProcessor() @@ -113,11 +147,14 @@ CSwapChainProcessor::~CSwapChainProcessor() WaitForSingleObject(m_thread[0].Get(), INFINITE); if (m_thread[1].Get()) WaitForSingleObject(m_thread[1].Get(), INFINITE); + if (m_thread[2].Get()) + WaitForSingleObject(m_thread[2].Get(), INFINITE); // Drain in-flight GPU work / completion callbacks before releasing the // resources they reference. The swap chain was already released in the // worker epilogue, so this does not hold an IddCx frame. m_dx12Device->WaitForIdle(); + ResetCandidates(); for (CPostProcessor& postProcessor : m_postProcessors) postProcessor.Reset(); @@ -132,6 +169,104 @@ DWORD CALLBACK CSwapChainProcessor::_SwapChainThread(LPVOID arg) return 0; } +static bool ArmPublishTimer(HANDLE timer, uint64_t delay) +{ + if (!timer) + return false; + + LARGE_INTEGER due = {}; + due.QuadPart = -static_cast((delay + 99) / 100); + if (!due.QuadPart) + due.QuadPart = -1; + return SetWaitableTimer(timer, &due, 0, nullptr, nullptr, FALSE) != FALSE; +} + +DWORD CALLBACK CSwapChainProcessor::_PublisherThread(LPVOID arg) +{ + reinterpret_cast(arg)->PublisherThread(); + return 0; +} + +bool CSwapChainProcessor::HasReadyCandidate() +{ + bool ready = false; + AcquireSRWLockShared(&m_candidateLock); + for (const FrameCandidate& candidate : m_candidates) + if (candidate.state == CANDIDATE_READY) + { + ready = true; + break; + } + ReleaseSRWLockShared(&m_candidateLock); + return ready; +} + +void CSwapChainProcessor::PublisherThread() +{ + DWORD avTask = 0; + HANDLE avTaskHandle = AvSetMmThreadCharacteristicsW(L"Distribution", &avTask); + + const HANDLE scheduleEvent = m_devContext->GetFrameScheduleEvent(); + HANDLE idleHandles[] = + { + m_terminateEvent.Get(), + m_candidateEvent.Get(), + scheduleEvent, + }; + HANDLE timerHandles[] = + { + m_terminateEvent.Get(), + m_candidateEvent.Get(), + scheduleEvent, + m_publishTimer.Get(), + }; + + for (;;) + { + if (!HasReadyCandidate()) + { + if (m_publishTimer.Get()) + CancelWaitableTimer(m_publishTimer.Get()); + if (WaitForMultipleObjects( + ARRAYSIZE(idleHandles), idleHandles, FALSE, INFINITE) == + WAIT_OBJECT_0) + break; + continue; + } + + const uint64_t now = CFrameScheduler::Nanotime(); + uint64_t target; + uint32_t generation; + bool periodic; + m_devContext->GetPublishTarget(now, target, generation, periodic); + + if (target > now) + { + ArmPublishTimer(m_publishTimer.Get(), target - now); + if (WaitForMultipleObjects( + ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) == + WAIT_OBJECT_0) + break; + continue; + } + + const uint64_t publishStart = CFrameScheduler::Nanotime(); + m_devContext->ProcessFrameQueue(); + if (!m_devContext->FrameBufferAvailable() || + !PublishNewestCandidate( + generation, periodic, publishStart)) + { + ArmPublishTimer(m_publishTimer.Get(), PUBLISH_RETRY_NS); + if (WaitForMultipleObjects( + ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) == + WAIT_OBJECT_0) + break; + } + } + + AvRevertMmThreadCharacteristics(avTaskHandle); +} + void CSwapChainProcessor::SwapChainThread() { DWORD avTask = 0; @@ -328,21 +463,68 @@ bool CSwapChainProcessor::SwapChainThreadCore() return true; } -void CSwapChainProcessor::CompletionFunction( +void CSwapChainProcessor::CandidateCompletionFunction( CD3D12CommandSlot * slot, bool result, void * param1, void * param2) { - auto sc = (CSwapChainProcessor *)param1; - auto fbRes = (CFrameBufferResource *)param2; + auto sc = static_cast(param1); + auto candidate = static_cast(param2); + + uint64_t gpuStart = 0; + uint64_t gpuEnd = 0; + const bool timingValid = result && slot->GetGPUTimes(gpuStart, gpuEnd); + + AcquireSRWLockExclusive(&sc->m_candidateLock); + if (candidate->state == CANDIDATE_PREPARING) + { + candidate->prepareReady = CFrameScheduler::Nanotime(); + candidate->prepareGPUStart = gpuStart; + candidate->prepareGPUEnd = gpuEnd; + candidate->prepareTimingValid = timingValid; + candidate->state = result ? CANDIDATE_READY : CANDIDATE_FREE; + } + ReleaseSRWLockExclusive(&sc->m_candidateLock); if (!result) { - // A submitted frame may already be in LGMP, or publication may race this - // callback. Make the message releasable even though its contents failed. + sc->SetFullPendingDamage(); + sc->m_devContext->ForceFrame(); + } + sc->SignalCandidateState(); +} + +void CSwapChainProcessor::CompletionFunction( + CD3D12CommandSlot * slot, bool result, void * param1, void * param2) +{ + auto sc = static_cast(param1); + auto fbRes = static_cast(param2); + const unsigned candidateIndex = fbRes->GetCandidateIndex(); + + if (!result) + { + // The frame was reserved in LGMP before GPU submission. Make the message + // releasable even though its contents failed. sc->m_devContext->FailFrameBuffer(fbRes->GetFrameIndex()); + sc->SetFullPendingDamage(); + sc->m_devContext->ForceFrame(); + sc->ReleaseCandidate(candidateIndex); return; } - const uint64_t cpuCopyStart = fbRes->GetCopyStart(); + uint64_t prepareCopyStart; + uint64_t prepareReady; + uint64_t prepareGPUStart; + uint64_t prepareGPUEnd; + bool prepareTimingValid; + AcquireSRWLockShared(&sc->m_candidateLock); + const FrameCandidate& candidate = sc->m_candidates[candidateIndex]; + prepareCopyStart = candidate.prepareCopyStart; + prepareReady = candidate.prepareReady; + prepareGPUStart = candidate.prepareGPUStart; + prepareGPUEnd = candidate.prepareGPUEnd; + prepareTimingValid = candidate.prepareTimingValid; + ReleaseSRWLockShared(&sc->m_candidateLock); + + const uint64_t publishStart = fbRes->GetCopyStart(); uint64_t gpuCopyStart = 0; uint64_t gpuCopyEnd = 0; @@ -360,26 +542,34 @@ void CSwapChainProcessor::CompletionFunction( sc->m_devContext->FinalizeFrameBuffer(fbRes->GetFrameIndex()); const uint64_t readyEnd = CFrameScheduler::Nanotime(); - uint64_t postProcessTime = cpuCopyStart - fbRes->GetPostProcessStart(); - uint64_t copyTime = readyEnd - cpuCopyStart; - uint64_t readyTime = 0; - if (gpuTimingValid && - gpuCopyStart >= fbRes->GetPostProcessStart() && - gpuCopyEnd <= readyEnd) + const uint64_t postProcessStart = fbRes->GetPostProcessStart(); + uint64_t postProcessTime = prepareCopyStart - postProcessStart; + uint64_t prepareCopyTime = prepareReady - prepareCopyStart; + if (prepareTimingValid && prepareGPUStart >= postProcessStart && + prepareGPUEnd >= prepareGPUStart && prepareGPUEnd <= prepareReady) { - postProcessTime = gpuCopyStart - fbRes->GetPostProcessStart(); - copyTime = gpuCopyEnd - gpuCopyStart; - readyTime = readyEnd - gpuCopyEnd; + postProcessTime = prepareGPUStart - postProcessStart; + prepareCopyTime = prepareGPUEnd - prepareGPUStart; } - sc->m_postProcessors[fbRes->GetFrameIndex()].RecordTiming( + uint64_t publishCopyTime = readyEnd - publishStart; + if (gpuTimingValid && gpuCopyStart >= publishStart && + gpuCopyEnd >= gpuCopyStart && gpuCopyEnd <= readyEnd) + publishCopyTime = gpuCopyEnd - gpuCopyStart; + + const uint64_t copyTime = prepareCopyTime + publishCopyTime; + const uint64_t elapsed = readyEnd - postProcessStart; + const uint64_t measured = postProcessTime + copyTime; + const uint64_t readyTime = elapsed > measured ? elapsed - measured : 0; + + sc->m_postProcessors[candidateIndex].RecordTiming( fbRes->GetTimingEffectIndex(), fbRes->GetTimingToken(), - fbRes->IsFullCopy(), postProcessTime + copyTime + readyTime); - sc->m_devContext->RecordFrameTiming( - postProcessTime + copyTime + readyTime); + fbRes->IsFullCopy(), postProcessTime + copyTime); + sc->m_devContext->RecordFrameTiming(readyEnd - publishStart); sc->m_devContext->SetFrameTiming(fbRes->GetFrameIndex(), fbRes->GetCaptureTime(), postProcessTime, copyTime, readyTime); sc->m_devContext->CompleteFrameBuffer(fbRes->GetFrameIndex()); + sc->ReleaseCandidate(candidateIndex); } @@ -520,13 +710,18 @@ static FrameType GetFrameType(DXGI_FORMAT format) void CSwapChainProcessor::SetFullPendingDamage() { + AcquireSRWLockExclusive(&m_damageLock); m_hasPendingDamage = true; m_nbPendingDirtyRects = 0; + ++m_damageGeneration; + ReleaseSRWLockExclusive(&m_damageLock); } void CSwapChainProcessor::AccumulateFrameDamage( const RECT * dirtyRects, unsigned nbDirtyRects) { + AcquireSRWLockExclusive(&m_damageLock); + ++m_damageGeneration; if (nbDirtyRects > LG_MAX_DIRTY_RECTS) nbDirtyRects = 0; @@ -537,6 +732,7 @@ void CSwapChainProcessor::AccumulateFrameDamage( if (nbDirtyRects) memcpy(m_pendingDirtyRects, dirtyRects, nbDirtyRects * sizeof(*m_pendingDirtyRects)); + ReleaseSRWLockExclusive(&m_damageLock); return; } @@ -545,18 +741,384 @@ void CSwapChainProcessor::AccumulateFrameDamage( if (m_nbPendingDirtyRects == 0 || nbDirtyRects == 0) { m_nbPendingDirtyRects = 0; + ReleaseSRWLockExclusive(&m_damageLock); return; } if (m_nbPendingDirtyRects + nbDirtyRects > LG_MAX_DIRTY_RECTS) { m_nbPendingDirtyRects = 0; + ReleaseSRWLockExclusive(&m_damageLock); return; } memcpy(m_pendingDirtyRects + m_nbPendingDirtyRects, dirtyRects, nbDirtyRects * sizeof(*m_pendingDirtyRects)); m_nbPendingDirtyRects += nbDirtyRects; + ReleaseSRWLockExclusive(&m_damageLock); +} + +int CSwapChainProcessor::AcquireCandidate() +{ + HANDLE waitHandles[] = + { + m_candidateAvailableEvent.Get(), + m_terminateEvent.Get(), + }; + + for (;;) + { + int selected = -1; + uint64_t oldest = UINT64_MAX; + bool superseded = false; + + AcquireSRWLockExclusive(&m_candidateLock); + for (unsigned i = 0; i < ARRAYSIZE(m_candidates); ++i) + if (m_candidates[i].state == CANDIDATE_FREE) + { + selected = static_cast(i); + break; + } + + unsigned readyCount = 0; + for (const FrameCandidate& candidate : m_candidates) + if (candidate.state == CANDIDATE_READY) + ++readyCount; + + if (selected < 0 && readyCount > 1) + for (unsigned i = 0; i < ARRAYSIZE(m_candidates); ++i) + if (m_candidates[i].state == CANDIDATE_READY && + m_candidates[i].sequence < oldest) + { + selected = static_cast(i); + oldest = m_candidates[i].sequence; + } + + if (selected >= 0) + { + FrameCandidate& candidate = + m_candidates[static_cast(selected)]; + superseded = candidate.state == CANDIDATE_READY; + candidate.state = CANDIDATE_PREPARING; + candidate.sequence = ++m_candidateSequence; + } + ReleaseSRWLockExclusive(&m_candidateLock); + + if (selected >= 0) + { + if (superseded) + m_devContext->FrameSuperseded(); + return selected; + } + + const DWORD result = WaitForMultipleObjects( + ARRAYSIZE(waitHandles), waitHandles, FALSE, CANDIDATE_WAIT_MS); + if (result == WAIT_OBJECT_0 + 1) + return -1; + if (result == WAIT_TIMEOUT) + return -1; + if (result != WAIT_OBJECT_0) + return -1; + } +} + +void CSwapChainProcessor::ReleaseCandidate(unsigned candidateIndex) +{ + if (candidateIndex >= ARRAYSIZE(m_candidates)) + return; + + AcquireSRWLockExclusive(&m_candidateLock); + m_candidates[candidateIndex].state = CANDIDATE_FREE; + ReleaseSRWLockExclusive(&m_candidateLock); + SignalCandidateState(); +} + +static bool ResourceDescMatches( + const D3D12_RESOURCE_DESC& left, const D3D12_RESOURCE_DESC& right) +{ + return + left.Dimension == right.Dimension && + left.Alignment == right.Alignment && + left.Width == right.Width && + left.Height == right.Height && + left.DepthOrArraySize == right.DepthOrArraySize && + left.MipLevels == right.MipLevels && + left.Format == right.Format && + left.SampleDesc.Count == right.SampleDesc.Count && + left.SampleDesc.Quality == right.SampleDesc.Quality && + left.Layout == right.Layout && + left.Flags == right.Flags; +} + +bool CSwapChainProcessor::EnsureCandidateResource( + unsigned candidateIndex, ID3D12Resource * source) +{ + FrameCandidate& candidate = m_candidates[candidateIndex]; + D3D12_RESOURCE_DESC desc = source->GetDesc(); + desc.Alignment = 0; + desc.Flags = static_cast( + static_cast(desc.Flags) & + ~static_cast(D3D12_RESOURCE_FLAG_ALLOW_CROSS_ADAPTER)); + + if (candidate.resource && + ResourceDescMatches(candidate.resource->GetDesc(), desc)) + return true; + + candidate.resource.Reset(); + + D3D12_HEAP_PROPERTIES heapProps = {}; + heapProps.Type = D3D12_HEAP_TYPE_DEFAULT; + heapProps.CPUPageProperty = D3D12_CPU_PAGE_PROPERTY_UNKNOWN; + heapProps.MemoryPoolPreference = D3D12_MEMORY_POOL_UNKNOWN; + heapProps.CreationNodeMask = 1; + heapProps.VisibleNodeMask = 1; + + const HRESULT hr = m_dx12Device->GetDevice()->CreateCommittedResource( + &heapProps, D3D12_HEAP_FLAG_NONE, &desc, D3D12_RESOURCE_STATE_COMMON, + nullptr, IID_PPV_ARGS(&candidate.resource)); + if (FAILED(hr)) + { + DEBUG_ERROR_HR(hr, "Failed to create retained frame candidate"); + return false; + } + + static const WCHAR * names[] = + { + L"Frame Candidate 0", + L"Frame Candidate 1", + }; + candidate.resource->SetName(names[candidateIndex]); + return true; +} + +void CSwapChainProcessor::ResetCandidates() +{ + AcquireSRWLockExclusive(&m_candidateLock); + for (FrameCandidate& candidate : m_candidates) + candidate = {}; + ReleaseSRWLockExclusive(&m_candidateLock); + SignalCandidateState(); +} + +void CSwapChainProcessor::SignalCandidateState() +{ + SetEvent(m_candidateEvent.Get()); + SetEvent(m_candidateAvailableEvent.Get()); +} + +bool CSwapChainProcessor::PublishNewestCandidate( + uint32_t scheduleGeneration, bool periodic, uint64_t publishStart) +{ + int selectedCandidate = -1; + uint64_t newestSequence = 0; + + AcquireSRWLockExclusive(&m_candidateLock); + for (unsigned i = 0; i < ARRAYSIZE(m_candidates); ++i) + if (m_candidates[i].state == CANDIDATE_READY && + (selectedCandidate < 0 || + m_candidates[i].sequence > newestSequence)) + { + selectedCandidate = static_cast(i); + newestSequence = m_candidates[i].sequence; + } + + if (selectedCandidate >= 0) + for (unsigned i = 0; i < ARRAYSIZE(m_candidates); ++i) + if (static_cast(i) == selectedCandidate) + m_candidates[i].state = CANDIDATE_PUBLISHING; + else if (m_candidates[i].state == CANDIDATE_READY) + m_candidates[i].state = CANDIDATE_HELD; + ReleaseSRWLockExclusive(&m_candidateLock); + + if (selectedCandidate < 0) + return false; + const unsigned candidateIndex = + static_cast(selectedCandidate); + + const auto restoreCandidates = [this, candidateIndex]() + { + AcquireSRWLockExclusive(&m_candidateLock); + if (m_candidates[candidateIndex].state == CANDIDATE_PUBLISHING) + m_candidates[candidateIndex].state = CANDIDATE_READY; + for (FrameCandidate& candidate : m_candidates) + if (candidate.state == CANDIDATE_HELD) + candidate.state = CANDIDATE_READY; + ReleaseSRWLockExclusive(&m_candidateLock); + SignalCandidateState(); + }; + + CSRWExclusiveLock pipelineLock(&m_pipelineLock); + AcquireSRWLockShared(&m_candidateLock); + const bool candidateValid = + m_candidates[candidateIndex].state == CANDIDATE_PUBLISHING && + m_candidates[candidateIndex].resource.Get(); + ReleaseSRWLockShared(&m_candidateLock); + if (!candidateValid) + { + restoreCandidates(); + return false; + } + + FrameCandidate& candidate = m_candidates[candidateIndex]; + CPostProcessor& postProcessor = m_postProcessors[candidateIndex]; + + auto buffer = m_devContext->PrepareFrameBuffer( + candidate.pitch, + candidate.srcFormat, + candidate.dstFormat, + candidate.dirtyRects, + candidate.nbDirtyRects); + if (!buffer.mem) + { + restoreCandidates(); + return false; + } + + CFrameBufferResource * fbRes = + m_fbPool.Get(buffer, candidate.frameSize); + if (!fbRes) + { + m_devContext->AbortFrameBuffer(buffer.frameIndex); + restoreCandidates(); + DEBUG_ERROR("Failed to get a CFrameBufferResource from the pool"); + SetFullPendingDamage(); + return false; + } + + CD3D12CommandSlot * copySlot = + m_dx12Device->GetCopySlot(candidateIndex); + if (!copySlot) + { + m_devContext->AbortFrameBuffer(buffer.frameIndex); + restoreCandidates(); + DEBUG_ERROR("Failed to get a copy CommandSlot for publication"); + SetFullPendingDamage(); + return false; + } + + RECT previousDirtyRects[LG_MAX_DIRTY_RECTS] = {}; + unsigned nbPreviousDirtyRects = 0; + AcquireSRWLockShared(&m_damageLock); + nbPreviousDirtyRects = m_nbDirtyRects; + if (nbPreviousDirtyRects) + memcpy(previousDirtyRects, m_dirtyRects, + nbPreviousDirtyRects * sizeof(*previousDirtyRects)); + ReleaseSRWLockShared(&m_damageLock); + + RECT copyDirtyRects[LG_MAX_DIRTY_RECTS * 2] = {}; + unsigned nbCopyDirtyRects = 0; + bool fullCopy = + candidate.nbDirtyRects == 0 || nbPreviousDirtyRects == 0; + + if (!fullCopy) + { + for (const RECT * rect = previousDirtyRects; + rect < previousDirtyRects + nbPreviousDirtyRects && !fullCopy; + ++rect) + { + RECT clipped = *rect; + if (ClipDirtyRect(clipped, + candidate.dstFormat.width, candidate.dstFormat.height) && + !AddCopyDirtyRect(copyDirtyRects, ARRAYSIZE(copyDirtyRects), + &nbCopyDirtyRects, clipped)) + fullCopy = true; + } + + for (const RECT * rect = candidate.dirtyRects; + rect < candidate.dirtyRects + candidate.nbDirtyRects && !fullCopy; + ++rect) + if (!AddCopyDirtyRect(copyDirtyRects, ARRAYSIZE(copyDirtyRects), + &nbCopyDirtyRects, *rect)) + fullCopy = true; + + if (!fullCopy) + fullCopy = IsFullDamage( + copyDirtyRects, nbCopyDirtyRects, + candidate.dstFormat.width, candidate.dstFormat.height) || + CopyAreaCoversFrame( + copyDirtyRects, nbCopyDirtyRects, + candidate.dstFormat.width, candidate.dstFormat.height); + + if (!fullCopy) + fullCopy = postProcessor.ShouldCopyFully( + copyDirtyRects, nbCopyDirtyRects); + } + + fbRes->SetTiming( + candidate.captureTime, candidate.postProcessStart, publishStart); + fbRes->SetCandidateIndex(candidateIndex); + fbRes->SetPostProcessSample( + candidate.timingEffectIndex, candidate.timingToken, fullCopy); + copySlot->SetCompletionCallback(&CompletionFunction, this, fbRes); + + copySlot->BeginTiming(); + postProcessor.CopyFrame( + copySlot->GetGfxList(), fbRes->Get().Get(), candidate.resource.Get(), + copyDirtyRects, nbCopyDirtyRects, fullCopy); + copySlot->EndTiming(); + + // Reserve the LGMP message before submitting the copy. This makes post + // failure recoverable without racing a very fast GPU completion callback. + if (!m_devContext->PublishFrameBuffer( + buffer.frameIndex, scheduleGeneration)) + { + copySlot->Cancel(); + m_devContext->AbortFrameBuffer(buffer.frameIndex); + restoreCandidates(); + return false; + } + + if (!copySlot->Execute()) + { + AcquireSRWLockShared(&m_candidateLock); + const bool callbackPending = + candidate.state == CANDIDATE_PUBLISHING; + ReleaseSRWLockShared(&m_candidateLock); + if (callbackPending && !copySlot->HasSubmittedWork()) + { + m_devContext->FailFrameBuffer(buffer.frameIndex); + SetFullPendingDamage(); + ReleaseCandidate(candidateIndex); + } + m_devContext->ForceFrame(); + + AcquireSRWLockExclusive(&m_candidateLock); + for (FrameCandidate& held : m_candidates) + if (held.state == CANDIDATE_HELD) + held.state = CANDIDATE_READY; + ReleaseSRWLockExclusive(&m_candidateLock); + SignalCandidateState(); + return false; + } + + AcquireSRWLockExclusive(&m_damageLock); + if (candidate.nbDirtyRects) + memcpy(m_dirtyRects, candidate.dirtyRects, + candidate.nbDirtyRects * sizeof(*m_dirtyRects)); + m_nbDirtyRects = candidate.nbDirtyRects; + if (candidate.damageGeneration == m_damageGeneration) + { + m_hasPendingDamage = false; + m_nbPendingDirtyRects = 0; + } + ReleaseSRWLockExclusive(&m_damageLock); + + m_devContext->CommitFrameBuffer( + buffer.frameIndex, scheduleGeneration, periodic); + + unsigned superseded = 0; + AcquireSRWLockExclusive(&m_candidateLock); + for (FrameCandidate& held : m_candidates) + if (held.state == CANDIDATE_HELD) + { + held.state = CANDIDATE_FREE; + ++superseded; + } + ReleaseSRWLockExclusive(&m_candidateLock); + for (unsigned i = 0; i < superseded; ++i) + m_devContext->FrameSuperseded(); + SignalCandidateState(); + return true; } #ifdef HAS_IDDCX_110 @@ -632,17 +1194,6 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer const uint64_t postProcessStart = CFrameScheduler::Nanotime(); const uint64_t captureTime = postProcessStart - captureStart; - m_devContext->ObserveFrame(postProcessStart); - - // Preserve the fast drop path: never hold an IddCx frame while waiting for - // a slow or disconnected client. We have not read its rectangles, so force - // the next published frame to invalidate the entire image. - if (!m_devContext->FrameBufferAvailable()) - { - SetFullPendingDamage(); - return true; - } - ComPtr texture; HRESULT hr = acquiredBuffer.As(&texture); if (FAILED(hr)) @@ -707,13 +1258,11 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer D3D12_RESOURCE_DESC srcDesc = srcRes->GetRes()->GetDesc(); if (!noImageUpdate) + { + m_devContext->ObserveFrame(postProcessStart); AccumulateFrameDamage( srcRes->GetDirtyRects(), srcRes->GetDirtyRectCount()); - - // Never hold an IddCx frame waiting for a slow or disconnected client. Read - // and retain its damage first so the next published frame remains complete. - if (!m_devContext->FrameBufferAvailable()) - return true; + } D12FrameFormat srcFormat = {}; srcFormat.desc = srcDesc; @@ -795,136 +1344,128 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer break; } - m_postProcessors[0].Update(srcFormat); - - const bool frameMetadataChanged = noImageUpdate && - FrameMetadataChanged(m_postProcessors[0].GetOutputFormat(), srcFormat); - - bool needsReconfigure = false; - for (const CPostProcessor& postProcessor : m_postProcessors) - if (postProcessor.NeedsReconfigure(srcFormat)) - { - needsReconfigure = true; - break; - } - - // SetFormat can replace resources still being read by the COPY queue. Format - // changes are rare, so drain both queues before updating either frame chain. - if (needsReconfigure) - { - m_nbDirtyRects = 0; - SetFullPendingDamage(); - m_dx12Device->WaitForIdle(); - } - - // An optional adaptive effect can reject its candidate while configuring - // either slot. Both queues are already idle, so allow one convergence pass - // to return both effect chains to the same mode. + bool frameMetadataChanged = false; + bool needsReconfigure = false; bool postProcessFormatChanged = false; - bool configurationStable = false; - for (unsigned pass = 0; pass < 2 && !configurationStable; ++pass) + bool requiresFullDamage = false; { - for (unsigned i = 0; i < ARRAYSIZE(m_postProcessors); ++i) - { - bool formatChanged = false; - if (!m_postProcessors[i].Configure(srcFormat, &formatChanged)) - { - SetFullPendingDamage(); - return false; - } + CSRWExclusiveLock pipelineLock(&m_pipelineLock); + m_postProcessors[0].Update(srcFormat); - if (i == 0) - postProcessFormatChanged |= formatChanged; - } + frameMetadataChanged = noImageUpdate && + FrameMetadataChanged( + m_postProcessors[0].GetOutputFormat(), srcFormat); - configurationStable = true; for (const CPostProcessor& postProcessor : m_postProcessors) if (postProcessor.NeedsReconfigure(srcFormat)) { - configurationStable = false; + needsReconfigure = true; break; } + + // A format change can replace resources referenced by either retained + // candidate. Stop publication, drain both queues, then invalidate them. + if (needsReconfigure) + { + AcquireSRWLockExclusive(&m_damageLock); + m_nbDirtyRects = 0; + ReleaseSRWLockExclusive(&m_damageLock); + SetFullPendingDamage(); + m_dx12Device->WaitForIdle(); + ResetCandidates(); + } + + bool configurationStable = false; + for (unsigned pass = 0; pass < 2 && !configurationStable; ++pass) + { + for (unsigned i = 0; i < ARRAYSIZE(m_postProcessors); ++i) + { + bool formatChanged = false; + if (!m_postProcessors[i].Configure(srcFormat, &formatChanged)) + { + SetFullPendingDamage(); + return false; + } + + if (i == 0) + postProcessFormatChanged |= formatChanged; + } + + configurationStable = true; + for (const CPostProcessor& postProcessor : m_postProcessors) + if (postProcessor.NeedsReconfigure(srcFormat)) + { + configurationStable = false; + break; + } + } + + if (!configurationStable) + { + DEBUG_ERROR("Post processor configuration did not stabilize"); + SetFullPendingDamage(); + return false; + } + + if (postProcessFormatChanged) + { + AcquireSRWLockExclusive(&m_damageLock); + m_nbDirtyRects = 0; + ReleaseSRWLockExclusive(&m_damageLock); + SetFullPendingDamage(); + } + else if (frameMetadataChanged) + SetFullPendingDamage(); + + requiresFullDamage = m_postProcessors[0].RequiresFullDamage(); + if (requiresFullDamage) + SetFullPendingDamage(); } - if (!configurationStable) + if (needsReconfigure || postProcessFormatChanged || frameMetadataChanged) + m_devContext->ForceFrame(); + + if (noImageUpdate) { - DEBUG_ERROR("Post processor configuration did not stabilize"); - SetFullPendingDamage(); - return false; + AcquireSRWLockShared(&m_damageLock); + const bool hasPendingDamage = m_hasPendingDamage; + ReleaseSRWLockShared(&m_damageLock); + if (!hasPendingDamage) + return true; } - if (postProcessFormatChanged) + const int selectedCandidate = AcquireCandidate(); + if (selectedCandidate < 0) { - m_nbDirtyRects = 0; - SetFullPendingDamage(); + m_devContext->FrameSuperseded(); + return true; } - else if (frameMetadataChanged) - SetFullPendingDamage(); + const unsigned candidateIndex = + static_cast(selectedCandidate); - // Adaptive effects need comparable full-frame samples until they lock. - const bool requiresFullDamage = - m_postProcessors[0].RequiresFullDamage(); - if (requiresFullDamage) - SetFullPendingDamage(); - - if (noImageUpdate && !m_hasPendingDamage) - return true; - - uint32_t scheduleGeneration = 0; - if (!m_devContext->SelectFrame(CFrameScheduler::Nanotime(), - needsReconfigure || postProcessFormatChanged || - frameMetadataChanged || requiresFullDamage, - scheduleGeneration)) - return true; - - const D12FrameFormat& dstFormat = - m_postProcessors[0].GetOutputFormat(); - const unsigned pitch = m_postProcessors[0].GetOutputPitch(); - const size_t frameSize = m_postProcessors[0].GetOutputSize(); + CSRWExclusiveLock pipelineLock(&m_pipelineLock); + CPostProcessor& postProcessor = m_postProcessors[candidateIndex]; + const D12FrameFormat& dstFormat = postProcessor.GetOutputFormat(); RECT currentDirtyRects[LG_MAX_DIRTY_RECTS] = {}; - RECT frameDirtyRects[LG_MAX_DIRTY_RECTS] = {}; - unsigned nbDirtyRects = m_nbPendingDirtyRects; - unsigned frameDirtyRectCount = nbDirtyRects; - if (nbDirtyRects) + unsigned nbDirtyRects = 0; + uint64_t damageGeneration = 0; + AcquireSRWLockShared(&m_damageLock); + if (m_hasPendingDamage) { - memcpy(currentDirtyRects, m_pendingDirtyRects, - nbDirtyRects * sizeof(*currentDirtyRects)); - memcpy(frameDirtyRects, currentDirtyRects, - nbDirtyRects * sizeof(*frameDirtyRects)); + nbDirtyRects = m_nbPendingDirtyRects; + if (nbDirtyRects) + memcpy(currentDirtyRects, m_pendingDirtyRects, + nbDirtyRects * sizeof(*currentDirtyRects)); + damageGeneration = m_damageGeneration; } - m_postProcessors[0].AdjustFrameDamage( - frameDirtyRects, &frameDirtyRectCount); - - auto buffer = m_devContext->PrepareFrameBuffer( - pitch, - srcFormat, - dstFormat, - frameDirtyRects, - frameDirtyRectCount); - - // Queue or framebuffer ownership can change after the early availability - // check. Treat this as a dropped frame rather than an error. - if (!buffer.mem) - return true; - - CFrameBufferResource * fbRes = m_fbPool.Get(buffer, frameSize); - - if (!fbRes) - { - m_devContext->AbortFrameBuffer(buffer.frameIndex); - DEBUG_ERROR("Failed to get a CFrameBufferResource from the pool"); - SetFullPendingDamage(); - return false; - } - - CPostProcessor& postProcessor = m_postProcessors[buffer.frameIndex]; + ReleaseSRWLockShared(&m_damageLock); CD3D12CommandSlot * copySlot = - m_dx12Device->GetCopySlot(buffer.frameIndex); + m_dx12Device->GetCopySlot(candidateIndex); if (!copySlot) { - m_devContext->AbortFrameBuffer(buffer.frameIndex); + ReleaseCandidate(candidateIndex); DEBUG_ERROR("Failed to get a copy CommandSlot"); SetFullPendingDamage(); return false; @@ -934,11 +1475,11 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer CD3D12CommandSlot * computeSlot = nullptr; if (postProcessor.HasActiveEffects()) { - computeSlot = m_dx12Device->GetComputeSlot(buffer.frameIndex); + computeSlot = m_dx12Device->GetComputeSlot(candidateIndex); if (!computeSlot) { copySlot->Cancel(); - m_devContext->AbortFrameBuffer(buffer.frameIndex); + ReleaseCandidate(candidateIndex); DEBUG_ERROR("Failed to get a compute CommandSlot"); SetFullPendingDamage(); return false; @@ -948,7 +1489,7 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer { computeSlot->Cancel(); copySlot->Cancel(); - m_devContext->AbortFrameBuffer(buffer.frameIndex); + ReleaseCandidate(candidateIndex); SetFullPendingDamage(); return false; } @@ -960,7 +1501,7 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer { computeSlot->Cancel(); copySlot->Cancel(); - m_devContext->AbortFrameBuffer(buffer.frameIndex); + ReleaseCandidate(candidateIndex); DEBUG_ERROR("Post processor returned no output resource"); SetFullPendingDamage(); return false; @@ -970,7 +1511,7 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer { copySlot->Cancel(); m_dx12Device->WaitForIdle(); - m_devContext->AbortFrameBuffer(buffer.frameIndex); + ReleaseCandidate(candidateIndex); SetFullPendingDamage(); return false; } @@ -979,7 +1520,7 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer { copySlot->Cancel(); m_dx12Device->WaitForIdle(); - m_devContext->AbortFrameBuffer(buffer.frameIndex); + ReleaseCandidate(candidateIndex); DEBUG_ERROR("Failed to queue compute synchronization"); SetFullPendingDamage(); return false; @@ -988,7 +1529,7 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer else if (!srcRes->Sync(*copySlot)) { copySlot->Cancel(); - m_devContext->AbortFrameBuffer(buffer.frameIndex); + ReleaseCandidate(candidateIndex); DEBUG_ERROR("Failed to queue source synchronization"); SetFullPendingDamage(); return false; @@ -997,62 +1538,59 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer ClipDirtyRects(currentDirtyRects, &nbDirtyRects, dstFormat.width, dstFormat.height); - const uint64_t copyStart = CFrameScheduler::Nanotime(); - fbRes->SetTiming(captureTime, postProcessStart, copyStart); - - copySlot->SetCompletionCallback(&CompletionFunction, this, fbRes); - - /* Each destination is reused every other frame, so repair both the prior - * and current damage. Coalesce them first to avoid copying overlapping - * regions, especially a prior full frame, more than once. */ - RECT copyDirtyRects[LG_MAX_DIRTY_RECTS * 2] = {}; - unsigned nbCopyDirtyRects = 0; - bool fullCopy = - nbDirtyRects == 0 || m_nbDirtyRects == 0; - - if (!fullCopy) + if (!EnsureCandidateResource(candidateIndex, copySrcResource.Get())) { - for (const RECT * rect = m_dirtyRects; - rect < m_dirtyRects + m_nbDirtyRects && !fullCopy; ++rect) - { - RECT clipped = *rect; - if (ClipDirtyRect(clipped, dstFormat.width, dstFormat.height) && - !AddCopyDirtyRect(copyDirtyRects, ARRAYSIZE(copyDirtyRects), - &nbCopyDirtyRects, clipped)) - fullCopy = true; - } - - for (const RECT * rect = currentDirtyRects; - rect < currentDirtyRects + nbDirtyRects && !fullCopy; ++rect) - if (!AddCopyDirtyRect(copyDirtyRects, ARRAYSIZE(copyDirtyRects), - &nbCopyDirtyRects, *rect)) - fullCopy = true; - - if (!fullCopy) - fullCopy = IsFullDamage( - copyDirtyRects, nbCopyDirtyRects, - dstFormat.width, dstFormat.height) || - CopyAreaCoversFrame( - copyDirtyRects, nbCopyDirtyRects, - dstFormat.width, dstFormat.height); - - if (!fullCopy) - fullCopy = postProcessor.ShouldCopyFully( - copyDirtyRects, nbCopyDirtyRects); + copySlot->Cancel(); + if (computeSlot) + m_dx12Device->WaitForIdle(); + ReleaseCandidate(candidateIndex); + SetFullPendingDamage(); + return false; } - unsigned timingEffectIndex = 0; - uint64_t timingToken = 0; - postProcessor.GetTimingToken(&timingEffectIndex, &timingToken); - fbRes->SetPostProcessSample( - timingEffectIndex, timingToken, fullCopy); + FrameCandidate& candidate = m_candidates[candidateIndex]; + candidate.srcFormat = srcFormat; + candidate.dstFormat = dstFormat; + candidate.nbDirtyRects = nbDirtyRects; + candidate.pitch = postProcessor.GetOutputPitch(); + candidate.frameSize = postProcessor.GetOutputSize(); + candidate.damageGeneration = damageGeneration; + candidate.captureTime = captureTime; + candidate.postProcessStart = postProcessStart; + candidate.prepareCopyStart = CFrameScheduler::Nanotime(); + candidate.prepareReady = 0; + candidate.prepareGPUStart = 0; + candidate.prepareGPUEnd = 0; + candidate.prepareTimingValid = false; + if (nbDirtyRects) + memcpy(candidate.dirtyRects, currentDirtyRects, + nbDirtyRects * sizeof(*candidate.dirtyRects)); + postProcessor.GetTimingToken( + &candidate.timingEffectIndex, &candidate.timingToken); - // Source/compute waits are submitted immediately before this command list. - // The timestamp therefore marks the first actual copy operation. + copySlot->SetCompletionCallback( + &CandidateCompletionFunction, this, &candidate); copySlot->BeginTiming(); - postProcessor.CopyFrame( - copySlot->GetGfxList(), fbRes->Get().Get(), copySrcResource.Get(), - copyDirtyRects, nbCopyDirtyRects, fullCopy); + const D3D12_RESOURCE_DESC copySrcDesc = copySrcResource->GetDesc(); + if (copySrcDesc.Dimension == D3D12_RESOURCE_DIMENSION_BUFFER) + copySlot->GetGfxList()->CopyBufferRegion( + candidate.resource.Get(), 0, copySrcResource.Get(), 0, + copySrcDesc.Width); + else + { + D3D12_TEXTURE_COPY_LOCATION srcLocation = {}; + srcLocation.pResource = copySrcResource.Get(); + srcLocation.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX; + srcLocation.SubresourceIndex = 0; + + D3D12_TEXTURE_COPY_LOCATION dstLocation = {}; + dstLocation.pResource = candidate.resource.Get(); + dstLocation.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX; + dstLocation.SubresourceIndex = 0; + + copySlot->GetGfxList()->CopyTextureRegion( + &dstLocation, 0, 0, 0, &srcLocation, nullptr); + } copySlot->EndTiming(); if (!copySlot->Execute()) @@ -1061,25 +1599,13 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer { if (computeSlot) m_dx12Device->WaitForIdle(); - m_devContext->AbortFrameBuffer(buffer.frameIndex); + ReleaseCandidate(candidateIndex); } SetFullPendingDamage(); + m_devContext->ForceFrame(); return false; } - if (!m_devContext->PublishFrameBuffer( - buffer.frameIndex, scheduleGeneration)) - { - SetFullPendingDamage(); - return false; - } - - memcpy(m_dirtyRects, currentDirtyRects, - nbDirtyRects * sizeof(*m_dirtyRects)); - m_nbDirtyRects = nbDirtyRects; - m_hasPendingDamage = false; - m_nbPendingDirtyRects = 0; - return true; } diff --git a/idd/LGIdd/CSwapChainProcessor.h b/idd/LGIdd/CSwapChainProcessor.h index 83135190..e0cc21e0 100644 --- a/idd/LGIdd/CSwapChainProcessor.h +++ b/idd/LGIdd/CSwapChainProcessor.h @@ -55,8 +55,50 @@ private: CFrameBufferPool m_fbPool; CPostProcessor m_postProcessors[LGMP_Q_FRAME_LEN]; - Wrappers::HandleT m_thread[2]; + enum CandidateState + { + CANDIDATE_FREE, + CANDIDATE_PREPARING, + CANDIDATE_READY, + CANDIDATE_PUBLISHING, + CANDIDATE_HELD, + }; + + struct FrameCandidate + { + CandidateState state = CANDIDATE_FREE; + ComPtr resource; + D12FrameFormat srcFormat = {}; + D12FrameFormat dstFormat = {}; + RECT dirtyRects[LG_MAX_DIRTY_RECTS] = {}; + unsigned nbDirtyRects = 0; + unsigned pitch = 0; + size_t frameSize = 0; + uint64_t sequence = 0; + uint64_t damageGeneration = 0; + uint64_t captureTime = 0; + uint64_t postProcessStart = 0; + uint64_t prepareCopyStart = 0; + uint64_t prepareReady = 0; + uint64_t prepareGPUStart = 0; + uint64_t prepareGPUEnd = 0; + unsigned timingEffectIndex = 0; + uint64_t timingToken = 0; + bool prepareTimingValid = false; + }; + + FrameCandidate m_candidates[LGMP_Q_FRAME_LEN]; + SRWLOCK m_candidateLock = SRWLOCK_INIT; + SRWLOCK m_damageLock = SRWLOCK_INIT; + SRWLOCK m_pipelineLock = SRWLOCK_INIT; + uint64_t m_candidateSequence = 0; + uint64_t m_damageGeneration = 0; + + Wrappers::HandleT m_thread[3]; Wrappers::Event m_terminateEvent; + Wrappers::Event m_candidateEvent; + Wrappers::Event m_candidateAvailableEvent; + Wrappers::HandleT m_publishTimer; Wrappers::Event m_cursorDataEvent; BYTE* m_shapeBuffer; @@ -89,12 +131,26 @@ private: void SwapChainThread(); bool SwapChainThreadCore(); + static DWORD CALLBACK _PublisherThread(LPVOID arg); + void PublisherThread(); + bool PublishNewestCandidate( + uint32_t scheduleGeneration, bool periodic, uint64_t publishStart); + bool HasReadyCandidate(); + int AcquireCandidate(); + void ReleaseCandidate(unsigned candidateIndex); + bool EnsureCandidateResource(unsigned candidateIndex, + ID3D12Resource * source); + void ResetCandidates(); + void SignalCandidateState(); + static DWORD CALLBACK _CursorThread(LPVOID arg); bool QueryHWCursor(); void CursorThread(); static void CompletionFunction( CD3D12CommandSlot * slot, bool result, void * param1, void * param2); + static void CandidateCompletionFunction( + CD3D12CommandSlot * slot, bool result, void * param1, void * param2); void AccumulateFrameDamage(const RECT * dirtyRects, unsigned nbDirtyRects); void SetFullPendingDamage(); #ifdef HAS_IDDCX_110