diff --git a/idd/LGIdd/CFrameBufferResource.h b/idd/LGIdd/CFrameBufferResource.h index e0c3ea09..bf5678d5 100644 --- a/idd/LGIdd/CFrameBufferResource.h +++ b/idd/LGIdd/CFrameBufferResource.h @@ -43,6 +43,7 @@ class CFrameBufferResource unsigned m_timingEffectIndex = 0; uint64_t m_timingToken = 0; bool m_fullCopy = false; + unsigned m_candidateIndex = 0; ComPtr m_res; void * m_map = nullptr; @@ -75,6 +76,8 @@ class CFrameBufferResource unsigned GetTimingEffectIndex() const { return m_timingEffectIndex; } uint64_t GetTimingToken () const { return m_timingToken; } bool IsFullCopy () const { return m_fullCopy; } + void SetCandidateIndex(unsigned index) { m_candidateIndex = index; } + unsigned GetCandidateIndex() const { return m_candidateIndex; } ComPtr Get() { return m_res; } }; diff --git a/idd/LGIdd/CFrameScheduler.cpp b/idd/LGIdd/CFrameScheduler.cpp index dbfed5db..82be4b46 100644 --- a/idd/LGIdd/CFrameScheduler.cpp +++ b/idd/LGIdd/CFrameScheduler.cpp @@ -30,6 +30,23 @@ static const uint64_t MIN_SAFETY_NS = 250000ULL; static const uint64_t LOG_INTERVAL_NS = 5000000000ULL; static const uint64_t CADENCE_BREAK = 4; +CFrameScheduler::CFrameScheduler() +{ + m_wakeEvent = CreateEvent(nullptr, FALSE, FALSE, nullptr); +} + +CFrameScheduler::~CFrameScheduler() +{ + if (m_wakeEvent) + CloseHandle(m_wakeEvent); +} + +void CFrameScheduler::WakePublisher() const +{ + if (m_wakeEvent) + SetEvent(m_wakeEvent); +} + uint64_t CFrameScheduler::Nanotime() { static const uint64_t frequency = []() @@ -58,7 +75,7 @@ CFrameScheduler::Client * CFrameScheduler::FindClient(uint32_t clientID) return nullptr; } -void CFrameScheduler::ElectOwner(uint64_t now) +bool CFrameScheduler::ElectOwner(uint64_t now) { Client * fastest = nullptr; Client * incumbent = FindClient(m_schedule.clientID); @@ -91,6 +108,8 @@ void CFrameScheduler::ElectOwner(uint64_t now) const uint32_t oldClientID = m_schedule.clientID; const uint32_t oldGeneration = m_schedule.generation; + const uint64_t oldPeriod = m_schedule.period; + const uint64_t oldSlack = m_schedule.targetSlack; if (!fastest) { m_schedule = {}; @@ -125,6 +144,9 @@ void CFrameScheduler::ElectOwner(uint64_t now) else if (ownerChanged && oldClientID) DEBUG_INFO("Frame timing owner released; using push delivery"); } + + return ownerChanged || oldGeneration != m_schedule.generation || + oldPeriod != m_schedule.period || oldSlack != m_schedule.targetSlack; } void CFrameScheduler::Reset() @@ -138,11 +160,8 @@ void CFrameScheduler::Reset() m_lastArrival = 0; m_guestPeriod = 0; - m_guestJitter = 0; m_workEstimate = 0; m_nextDeadline = 0; - m_arrivalSamples = 0; - m_timingSamples = 0; m_lastPublishedFrameSerial = 0; @@ -155,6 +174,7 @@ void CFrameScheduler::Reset() m_lastLogSkipped = 0; m_lastLogPublished = 0; ReleaseSRWLockExclusive(&m_lock); + WakePublisher(); } void CFrameScheduler::UpdateSubscribers(const uint32_t * clientIDs, @@ -185,8 +205,10 @@ void CFrameScheduler::UpdateSubscribers(const uint32_t * clientIDs, if (client.clientID && !client.subscribed) client = {}; - ElectOwner(now); + const bool changed = ElectOwner(now); ReleaseSRWLockExclusive(&m_lock); + if (changed) + WakePublisher(); } bool CFrameScheduler::UpdateSchedule(const KVMFRFrameSchedule& schedule, @@ -203,6 +225,7 @@ bool CFrameScheduler::UpdateSchedule(const KVMFRFrameSchedule& schedule, AcquireSRWLockExclusive(&m_lock); Client * client = FindClient(schedule.clientID); + bool wake = false; if (schedule.flags & KVMFR_FRAME_SCHEDULE_RELEASE) { @@ -210,9 +233,11 @@ bool CFrameScheduler::UpdateSchedule(const KVMFRFrameSchedule& schedule, { client->active = false; client->expiry = 0; - ElectOwner(now); + wake = ElectOwner(now); } ReleaseSRWLockExclusive(&m_lock); + if (wake) + WakePublisher(); return true; } @@ -242,14 +267,19 @@ bool CFrameScheduler::UpdateSchedule(const KVMFRFrameSchedule& schedule, client->expiry = now + static_cast(schedule.lease) * 1000000; client->active = true; if (schedule.flags & KVMFR_FRAME_SCHEDULE_IMMEDIATE) + { m_forceNext = true; - ElectOwner(now); - ApplyFeedback(*client, schedule); + wake = true; + } + wake |= ElectOwner(now); + wake |= ApplyFeedback(*client, schedule); ReleaseSRWLockExclusive(&m_lock); + if (wake) + WakePublisher(); return true; } -void CFrameScheduler::ApplyFeedback(Client& client, +bool CFrameScheduler::ApplyFeedback(Client& client, const KVMFRFrameSchedule& schedule) { if (!m_scheduling || client.clientID != m_schedule.clientID || @@ -261,7 +291,7 @@ void CFrameScheduler::ApplyFeedback(Client& client, !m_lastPublishedFrameSerial || static_cast(schedule.feedbackFrameSerial - m_lastPublishedFrameSerial) > 0) - return; + return false; int64_t correction = schedule.phaseError / 4; const int64_t limit = static_cast(m_schedule.period / 4); @@ -280,6 +310,7 @@ void CFrameScheduler::ApplyFeedback(Client& client, } m_lastPhaseError = schedule.phaseError; client.lastFeedbackFrameSerial = schedule.feedbackFrameSerial; + return correction != 0; } void CFrameScheduler::AdvanceDeadline(uint64_t now) @@ -311,35 +342,35 @@ void CFrameScheduler::ObserveFrame(uint64_t now) const uint64_t interval = now - m_lastArrival; if (m_guestPeriod && interval > m_guestPeriod * CADENCE_BREAK) { - m_guestPeriod = 0; - m_guestJitter = 0; - m_arrivalSamples = 0; - m_forceNext = true; + m_guestPeriod = 0; + m_forceNext = true; } else if (interval >= MIN_PERIOD_NS && interval <= MAX_PERIOD_NS) { if (!m_guestPeriod) m_guestPeriod = interval; else - { - const uint64_t error = m_guestPeriod > interval ? - m_guestPeriod - interval : interval - m_guestPeriod; m_guestPeriod = (m_guestPeriod * 7 + interval) / 8; - m_guestJitter = (m_guestJitter * 7 + error) / 8; - } - - if (m_arrivalSamples < 32) - ++m_arrivalSamples; } } m_lastArrival = now; ReleaseSRWLockExclusive(&m_lock); } -bool CFrameScheduler::SelectFrame(uint64_t now, bool force, - uint32_t& generation) +void CFrameScheduler::ForceFrame() { + AcquireSRWLockExclusive(&m_lock); + m_forceNext = true; + ReleaseSRWLockExclusive(&m_lock); + WakePublisher(); +} + +bool CFrameScheduler::GetPublishTarget(uint64_t now, uint64_t& target, + uint32_t& generation, bool& periodic) +{ + target = now; generation = 0; + periodic = false; AcquireSRWLockExclusive(&m_lock); if (!m_scheduling) { @@ -348,31 +379,41 @@ bool CFrameScheduler::SelectFrame(uint64_t now, bool force, } generation = m_schedule.generation; - AdvanceDeadline(now); - if (force) - m_forceNext = true; + if (m_forceNext) + { + periodic = m_nextDeadline <= now; + ReleaseSRWLockExclusive(&m_lock); + return true; + } - if (m_forceNext || m_arrivalSamples < 4 || m_timingSamples < 4) + periodic = true; + if (m_nextDeadline <= now) { ReleaseSRWLockExclusive(&m_lock); return true; } const uint64_t safety = - max(MIN_SAFETY_NS, - max(m_guestJitter * 2, m_workEstimate / 8)); - const uint64_t nextArrival = m_lastArrival + m_guestPeriod; - const bool process = nextArrival <= now || - nextArrival + m_workEstimate + safety > m_nextDeadline; - if (!process) - ++m_skippedFrames; + max(MIN_SAFETY_NS, m_workEstimate / 8); + const uint64_t lead = + m_schedule.targetSlack + m_workEstimate + safety; + target = m_nextDeadline > lead ? m_nextDeadline - lead : now; + if (target < now) + target = now; + ReleaseSRWLockExclusive(&m_lock); + return true; +} + +void CFrameScheduler::FrameSuperseded() +{ + AcquireSRWLockExclusive(&m_lock); + ++m_skippedFrames; ReleaseSRWLockExclusive(&m_lock); - return process; } void CFrameScheduler::FramePublished(uint32_t generation, - uint32_t frameSerial, uint64_t now) + uint32_t frameSerial, uint64_t now, bool periodic) { AcquireSRWLockExclusive(&m_lock); if (m_scheduling && generation == m_schedule.generation) @@ -380,8 +421,11 @@ void CFrameScheduler::FramePublished(uint32_t generation, m_forceNext = false; m_lastPublishedFrameSerial = frameSerial; ++m_publishedFrames; - m_nextDeadline += m_schedule.period; - AdvanceDeadline(now); + if (periodic) + { + m_nextDeadline += m_schedule.period; + AdvanceDeadline(now); + } } ReleaseSRWLockExclusive(&m_lock); } @@ -428,7 +472,5 @@ void CFrameScheduler::RecordFrameTiming(uint64_t duration) else m_workEstimate = (m_workEstimate * 31 + duration) / 32; - if (m_timingSamples < 32) - ++m_timingSamples; ReleaseSRWLockExclusive(&m_lock); } diff --git a/idd/LGIdd/CFrameScheduler.h b/idd/LGIdd/CFrameScheduler.h index 899257a2..1f6366c3 100644 --- a/idd/LGIdd/CFrameScheduler.h +++ b/idd/LGIdd/CFrameScheduler.h @@ -54,6 +54,7 @@ private: }; mutable SRWLOCK m_lock = SRWLOCK_INIT; + HANDLE m_wakeEvent = nullptr; Client m_clients[LGMP_MAX_CLIENTS] = {}; Schedule m_schedule = {}; bool m_scheduling = false; @@ -61,11 +62,8 @@ private: uint64_t m_lastArrival = 0; uint64_t m_guestPeriod = 0; - uint64_t m_guestJitter = 0; uint64_t m_workEstimate = 0; uint64_t m_nextDeadline = 0; - unsigned m_arrivalSamples = 0; - unsigned m_timingSamples = 0; uint32_t m_lastPublishedFrameSerial = 0; @@ -79,11 +77,15 @@ private: uint64_t m_lastLogPublished = 0; Client * FindClient(uint32_t clientID); - void ElectOwner(uint64_t now); - void ApplyFeedback(Client& client, const KVMFRFrameSchedule& schedule); + bool ElectOwner(uint64_t now); + bool ApplyFeedback(Client& client, const KVMFRFrameSchedule& schedule); void AdvanceDeadline(uint64_t now); + void WakePublisher() const; public: + CFrameScheduler(); + ~CFrameScheduler(); + static uint64_t Nanotime(); void Reset(); @@ -91,10 +93,14 @@ public: uint64_t now); bool UpdateSchedule(const KVMFRFrameSchedule& schedule, uint64_t now); bool GetSchedule(Schedule& schedule) const; + HANDLE GetWakeEvent() const { return m_wakeEvent; } void ObserveFrame(uint64_t now); - bool SelectFrame(uint64_t now, bool force, uint32_t& generation); + void ForceFrame(); + bool GetPublishTarget(uint64_t now, uint64_t& target, + uint32_t& generation, bool& periodic); + void FrameSuperseded(); void FramePublished(uint32_t generation, uint32_t frameSerial, - uint64_t now); + uint64_t now, bool periodic); void RecordFrameTiming(uint64_t duration); void LogStatistics(uint64_t now); }; diff --git a/idd/LGIdd/CIndirectDeviceContext.cpp b/idd/LGIdd/CIndirectDeviceContext.cpp index eb0853bb..555578b8 100644 --- a/idd/LGIdd/CIndirectDeviceContext.cpp +++ b/idd/LGIdd/CIndirectDeviceContext.cpp @@ -1246,7 +1246,10 @@ void CIndirectDeviceContext::LGMPTimer() } LGMP_STATUS status; - if ((status = lgmpHostProcess(m_lgmp)) != LGMP_OK) + AcquireSRWLockExclusive(&m_lgmpProcessLock); + status = lgmpHostProcess(m_lgmp); + ReleaseSRWLockExclusive(&m_lgmpProcessLock); + if (status != LGMP_OK) { if (status == LGMP_ERR_CORRUPTED) { @@ -1348,6 +1351,19 @@ bool CIndirectDeviceContext::FrameBufferAvailable() const return !m_frameInFlight[frameIndex].load(std::memory_order_acquire); } +void CIndirectDeviceContext::ProcessFrameQueue() +{ + if (!m_lgmp) + return; + + AcquireSRWLockExclusive(&m_lgmpProcessLock); + const LGMP_STATUS status = lgmpHostProcess(m_lgmp); + ReleaseSRWLockExclusive(&m_lgmpProcessLock); + + if (status != LGMP_OK && status != LGMP_ERR_CORRUPTED) + DEBUG_ERROR("lgmpHostProcess Failed: %s", lgmpStatusString(status)); +} + CIndirectDeviceContext::PreparedFrameBuffer CIndirectDeviceContext::PrepareFrameBuffer( unsigned pitch, const D12FrameFormat& srcFormat, const D12FrameFormat& dstFormat, const RECT * dirtyRects, unsigned nbDirtyRects) @@ -1524,11 +1540,6 @@ bool CIndirectDeviceContext::PublishFrameBuffer(unsigned frameIndex, } ReleaseSRWLockExclusive(&m_framePublishLock); - if (status == LGMP_OK) - m_frameScheduler.FramePublished( - scheduleGeneration, m_frame[frameIndex]->frameSerial, - CFrameScheduler::Nanotime()); - if (status != LGMP_OK) { DEBUG_ERROR("Failed to publish frame: %s", lgmpStatusString(status)); @@ -1538,15 +1549,37 @@ bool CIndirectDeviceContext::PublishFrameBuffer(unsigned frameIndex, return true; } +void CIndirectDeviceContext::CommitFrameBuffer(unsigned frameIndex, + uint32_t scheduleGeneration, bool periodic) +{ + if (frameIndex >= LGMP_Q_FRAME_LEN) + return; + + m_frameScheduler.FramePublished( + scheduleGeneration, m_frame[frameIndex]->frameSerial, + CFrameScheduler::Nanotime(), periodic); +} + void CIndirectDeviceContext::ObserveFrame(uint64_t now) { m_frameScheduler.ObserveFrame(now); } -bool CIndirectDeviceContext::SelectFrame(uint64_t now, bool force, - uint32_t& generation) +void CIndirectDeviceContext::ForceFrame() { - return m_frameScheduler.SelectFrame(now, force, generation); + m_frameScheduler.ForceFrame(); +} + +bool CIndirectDeviceContext::GetPublishTarget(uint64_t now, + uint64_t& target, uint32_t& generation, bool& periodic) +{ + return m_frameScheduler.GetPublishTarget( + now, target, generation, periodic); +} + +void CIndirectDeviceContext::FrameSuperseded() +{ + m_frameScheduler.FrameSuperseded(); } void CIndirectDeviceContext::RecordFrameTiming(uint64_t duration) diff --git a/idd/LGIdd/CIndirectDeviceContext.h b/idd/LGIdd/CIndirectDeviceContext.h index 78081816..d5133753 100644 --- a/idd/LGIdd/CIndirectDeviceContext.h +++ b/idd/LGIdd/CIndirectDeviceContext.h @@ -86,9 +86,10 @@ private: CIVSHMEM m_ivshmem; - PLGMPHost m_lgmp = nullptr; - WDFTIMER m_lgmpTimer = nullptr; - PLGMPHostQueue m_frameQueue = nullptr; + PLGMPHost m_lgmp = nullptr; + WDFTIMER m_lgmpTimer = nullptr; + PLGMPHostQueue m_frameQueue = nullptr; + SRWLOCK m_lgmpProcessLock = SRWLOCK_INIT; CFrameScheduler m_frameScheduler; @@ -224,8 +225,11 @@ public: }; bool FrameBufferAvailable() const; + void ProcessFrameQueue(); PreparedFrameBuffer PrepareFrameBuffer(unsigned pitch, const D12FrameFormat& srcFormat, const D12FrameFormat& dstFormat, const RECT * dirtyRects, unsigned nbDirtyRects); bool PublishFrameBuffer(unsigned frameIndex, uint32_t scheduleGeneration); + void CommitFrameBuffer(unsigned frameIndex, uint32_t scheduleGeneration, + bool periodic); void AbortFrameBuffer(unsigned frameIndex); void FailFrameBuffer(unsigned frameIndex); void CompleteFrameBuffer(unsigned frameIndex); @@ -235,7 +239,14 @@ public: void FinalizeFrameBuffer(unsigned frameIndex) const; void ObserveFrame(uint64_t now); - bool SelectFrame(uint64_t now, bool force, uint32_t& generation); + void ForceFrame(); + bool GetPublishTarget(uint64_t now, uint64_t& target, + uint32_t& generation, bool& periodic); + void FrameSuperseded(); + HANDLE GetFrameScheduleEvent() const + { + return m_frameScheduler.GetWakeEvent(); + } void RecordFrameTiming(uint64_t duration); void SendCursor(const IDARG_OUT_QUERY_HWCURSOR & info, const BYTE * data, diff --git a/idd/LGIdd/CSwapChainProcessor.cpp b/idd/LGIdd/CSwapChainProcessor.cpp index 62be9dd5..84261f06 100644 --- a/idd/LGIdd/CSwapChainProcessor.cpp +++ b/idd/LGIdd/CSwapChainProcessor.cpp @@ -25,12 +25,35 @@ #include "CDebug.h" #include "CPipeServer.h" +#ifndef CREATE_WAITABLE_TIMER_HIGH_RESOLUTION + #define CREATE_WAITABLE_TIMER_HIGH_RESOLUTION 0x00000002 +#endif + static const uint32_t HDR_PQ_MIN_LUMINANCE = 50; static const uint32_t HDR_PQ_MAX_LUMINANCE = 10000; +static const uint64_t PUBLISH_RETRY_NS = 1000000ULL; +static const DWORD CANDIDATE_WAIT_MS = 2; static_assert(LGMP_Q_FRAME_LEN == 2, "IDD damage repair assumes two alternating frame buffers"); +class CSRWExclusiveLock +{ +private: + SRWLOCK * m_lock; + +public: + explicit CSRWExclusiveLock(SRWLOCK * lock) : m_lock(lock) + { + AcquireSRWLockExclusive(m_lock); + } + + ~CSRWExclusiveLock() + { + ReleaseSRWLockExclusive(m_lock); + } +}; + static bool FrameMetadataChanged(const D12FrameFormat& previous, const D12FrameFormat& current) { @@ -96,14 +119,25 @@ CSwapChainProcessor::CSwapChainProcessor(CIndirectMonitorContext * monitorContex "Failed to initialize post-processing effects; effects disabled"); } - // Manual-reset: both worker threads wait on this, so it must stay signalled + // Manual-reset: all worker threads wait on this, so it must stay signalled // once set or only one thread would ever observe termination. m_terminateEvent.Attach(CreateEvent(nullptr, TRUE, FALSE, nullptr)); + m_candidateEvent.Attach(CreateEvent(nullptr, FALSE, FALSE, nullptr)); + m_candidateAvailableEvent.Attach( + CreateEvent(nullptr, FALSE, FALSE, nullptr)); + m_publishTimer.Attach(CreateWaitableTimerExW(nullptr, nullptr, + CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS)); + if (!m_publishTimer.Get()) + m_publishTimer.Attach(CreateWaitableTimerExW( + nullptr, nullptr, 0, TIMER_ALL_ACCESS)); m_cursorDataEvent.Attach(CreateEvent(nullptr, FALSE, FALSE, nullptr)); m_shapeBuffer = new BYTE[512 * 512 * 4]; // Start the worker only after every object it can access is initialized. - m_thread[0].Attach(CreateThread(nullptr, 0, _SwapChainThread, this, 0, nullptr)); + m_thread[0].Attach(CreateThread( + nullptr, 0, _SwapChainThread, this, 0, nullptr)); + m_thread[2].Attach(CreateThread( + nullptr, 0, _PublisherThread, this, 0, nullptr)); } CSwapChainProcessor::~CSwapChainProcessor() @@ -113,11 +147,14 @@ CSwapChainProcessor::~CSwapChainProcessor() WaitForSingleObject(m_thread[0].Get(), INFINITE); if (m_thread[1].Get()) WaitForSingleObject(m_thread[1].Get(), INFINITE); + if (m_thread[2].Get()) + WaitForSingleObject(m_thread[2].Get(), INFINITE); // Drain in-flight GPU work / completion callbacks before releasing the // resources they reference. The swap chain was already released in the // worker epilogue, so this does not hold an IddCx frame. m_dx12Device->WaitForIdle(); + ResetCandidates(); for (CPostProcessor& postProcessor : m_postProcessors) postProcessor.Reset(); @@ -132,6 +169,104 @@ DWORD CALLBACK CSwapChainProcessor::_SwapChainThread(LPVOID arg) return 0; } +static bool ArmPublishTimer(HANDLE timer, uint64_t delay) +{ + if (!timer) + return false; + + LARGE_INTEGER due = {}; + due.QuadPart = -static_cast((delay + 99) / 100); + if (!due.QuadPart) + due.QuadPart = -1; + return SetWaitableTimer(timer, &due, 0, nullptr, nullptr, FALSE) != FALSE; +} + +DWORD CALLBACK CSwapChainProcessor::_PublisherThread(LPVOID arg) +{ + reinterpret_cast(arg)->PublisherThread(); + return 0; +} + +bool CSwapChainProcessor::HasReadyCandidate() +{ + bool ready = false; + AcquireSRWLockShared(&m_candidateLock); + for (const FrameCandidate& candidate : m_candidates) + if (candidate.state == CANDIDATE_READY) + { + ready = true; + break; + } + ReleaseSRWLockShared(&m_candidateLock); + return ready; +} + +void CSwapChainProcessor::PublisherThread() +{ + DWORD avTask = 0; + HANDLE avTaskHandle = AvSetMmThreadCharacteristicsW(L"Distribution", &avTask); + + const HANDLE scheduleEvent = m_devContext->GetFrameScheduleEvent(); + HANDLE idleHandles[] = + { + m_terminateEvent.Get(), + m_candidateEvent.Get(), + scheduleEvent, + }; + HANDLE timerHandles[] = + { + m_terminateEvent.Get(), + m_candidateEvent.Get(), + scheduleEvent, + m_publishTimer.Get(), + }; + + for (;;) + { + if (!HasReadyCandidate()) + { + if (m_publishTimer.Get()) + CancelWaitableTimer(m_publishTimer.Get()); + if (WaitForMultipleObjects( + ARRAYSIZE(idleHandles), idleHandles, FALSE, INFINITE) == + WAIT_OBJECT_0) + break; + continue; + } + + const uint64_t now = CFrameScheduler::Nanotime(); + uint64_t target; + uint32_t generation; + bool periodic; + m_devContext->GetPublishTarget(now, target, generation, periodic); + + if (target > now) + { + ArmPublishTimer(m_publishTimer.Get(), target - now); + if (WaitForMultipleObjects( + ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) == + WAIT_OBJECT_0) + break; + continue; + } + + const uint64_t publishStart = CFrameScheduler::Nanotime(); + m_devContext->ProcessFrameQueue(); + if (!m_devContext->FrameBufferAvailable() || + !PublishNewestCandidate( + generation, periodic, publishStart)) + { + ArmPublishTimer(m_publishTimer.Get(), PUBLISH_RETRY_NS); + if (WaitForMultipleObjects( + ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) == + WAIT_OBJECT_0) + break; + } + } + + AvRevertMmThreadCharacteristics(avTaskHandle); +} + void CSwapChainProcessor::SwapChainThread() { DWORD avTask = 0; @@ -328,21 +463,68 @@ bool CSwapChainProcessor::SwapChainThreadCore() return true; } -void CSwapChainProcessor::CompletionFunction( +void CSwapChainProcessor::CandidateCompletionFunction( CD3D12CommandSlot * slot, bool result, void * param1, void * param2) { - auto sc = (CSwapChainProcessor *)param1; - auto fbRes = (CFrameBufferResource *)param2; + auto sc = static_cast(param1); + auto candidate = static_cast(param2); + + uint64_t gpuStart = 0; + uint64_t gpuEnd = 0; + const bool timingValid = result && slot->GetGPUTimes(gpuStart, gpuEnd); + + AcquireSRWLockExclusive(&sc->m_candidateLock); + if (candidate->state == CANDIDATE_PREPARING) + { + candidate->prepareReady = CFrameScheduler::Nanotime(); + candidate->prepareGPUStart = gpuStart; + candidate->prepareGPUEnd = gpuEnd; + candidate->prepareTimingValid = timingValid; + candidate->state = result ? CANDIDATE_READY : CANDIDATE_FREE; + } + ReleaseSRWLockExclusive(&sc->m_candidateLock); if (!result) { - // A submitted frame may already be in LGMP, or publication may race this - // callback. Make the message releasable even though its contents failed. + sc->SetFullPendingDamage(); + sc->m_devContext->ForceFrame(); + } + sc->SignalCandidateState(); +} + +void CSwapChainProcessor::CompletionFunction( + CD3D12CommandSlot * slot, bool result, void * param1, void * param2) +{ + auto sc = static_cast(param1); + auto fbRes = static_cast(param2); + const unsigned candidateIndex = fbRes->GetCandidateIndex(); + + if (!result) + { + // The frame was reserved in LGMP before GPU submission. Make the message + // releasable even though its contents failed. sc->m_devContext->FailFrameBuffer(fbRes->GetFrameIndex()); + sc->SetFullPendingDamage(); + sc->m_devContext->ForceFrame(); + sc->ReleaseCandidate(candidateIndex); return; } - const uint64_t cpuCopyStart = fbRes->GetCopyStart(); + uint64_t prepareCopyStart; + uint64_t prepareReady; + uint64_t prepareGPUStart; + uint64_t prepareGPUEnd; + bool prepareTimingValid; + AcquireSRWLockShared(&sc->m_candidateLock); + const FrameCandidate& candidate = sc->m_candidates[candidateIndex]; + prepareCopyStart = candidate.prepareCopyStart; + prepareReady = candidate.prepareReady; + prepareGPUStart = candidate.prepareGPUStart; + prepareGPUEnd = candidate.prepareGPUEnd; + prepareTimingValid = candidate.prepareTimingValid; + ReleaseSRWLockShared(&sc->m_candidateLock); + + const uint64_t publishStart = fbRes->GetCopyStart(); uint64_t gpuCopyStart = 0; uint64_t gpuCopyEnd = 0; @@ -360,26 +542,34 @@ void CSwapChainProcessor::CompletionFunction( sc->m_devContext->FinalizeFrameBuffer(fbRes->GetFrameIndex()); const uint64_t readyEnd = CFrameScheduler::Nanotime(); - uint64_t postProcessTime = cpuCopyStart - fbRes->GetPostProcessStart(); - uint64_t copyTime = readyEnd - cpuCopyStart; - uint64_t readyTime = 0; - if (gpuTimingValid && - gpuCopyStart >= fbRes->GetPostProcessStart() && - gpuCopyEnd <= readyEnd) + const uint64_t postProcessStart = fbRes->GetPostProcessStart(); + uint64_t postProcessTime = prepareCopyStart - postProcessStart; + uint64_t prepareCopyTime = prepareReady - prepareCopyStart; + if (prepareTimingValid && prepareGPUStart >= postProcessStart && + prepareGPUEnd >= prepareGPUStart && prepareGPUEnd <= prepareReady) { - postProcessTime = gpuCopyStart - fbRes->GetPostProcessStart(); - copyTime = gpuCopyEnd - gpuCopyStart; - readyTime = readyEnd - gpuCopyEnd; + postProcessTime = prepareGPUStart - postProcessStart; + prepareCopyTime = prepareGPUEnd - prepareGPUStart; } - sc->m_postProcessors[fbRes->GetFrameIndex()].RecordTiming( + uint64_t publishCopyTime = readyEnd - publishStart; + if (gpuTimingValid && gpuCopyStart >= publishStart && + gpuCopyEnd >= gpuCopyStart && gpuCopyEnd <= readyEnd) + publishCopyTime = gpuCopyEnd - gpuCopyStart; + + const uint64_t copyTime = prepareCopyTime + publishCopyTime; + const uint64_t elapsed = readyEnd - postProcessStart; + const uint64_t measured = postProcessTime + copyTime; + const uint64_t readyTime = elapsed > measured ? elapsed - measured : 0; + + sc->m_postProcessors[candidateIndex].RecordTiming( fbRes->GetTimingEffectIndex(), fbRes->GetTimingToken(), - fbRes->IsFullCopy(), postProcessTime + copyTime + readyTime); - sc->m_devContext->RecordFrameTiming( - postProcessTime + copyTime + readyTime); + fbRes->IsFullCopy(), postProcessTime + copyTime); + sc->m_devContext->RecordFrameTiming(readyEnd - publishStart); sc->m_devContext->SetFrameTiming(fbRes->GetFrameIndex(), fbRes->GetCaptureTime(), postProcessTime, copyTime, readyTime); sc->m_devContext->CompleteFrameBuffer(fbRes->GetFrameIndex()); + sc->ReleaseCandidate(candidateIndex); } @@ -520,13 +710,18 @@ static FrameType GetFrameType(DXGI_FORMAT format) void CSwapChainProcessor::SetFullPendingDamage() { + AcquireSRWLockExclusive(&m_damageLock); m_hasPendingDamage = true; m_nbPendingDirtyRects = 0; + ++m_damageGeneration; + ReleaseSRWLockExclusive(&m_damageLock); } void CSwapChainProcessor::AccumulateFrameDamage( const RECT * dirtyRects, unsigned nbDirtyRects) { + AcquireSRWLockExclusive(&m_damageLock); + ++m_damageGeneration; if (nbDirtyRects > LG_MAX_DIRTY_RECTS) nbDirtyRects = 0; @@ -537,6 +732,7 @@ void CSwapChainProcessor::AccumulateFrameDamage( if (nbDirtyRects) memcpy(m_pendingDirtyRects, dirtyRects, nbDirtyRects * sizeof(*m_pendingDirtyRects)); + ReleaseSRWLockExclusive(&m_damageLock); return; } @@ -545,18 +741,384 @@ void CSwapChainProcessor::AccumulateFrameDamage( if (m_nbPendingDirtyRects == 0 || nbDirtyRects == 0) { m_nbPendingDirtyRects = 0; + ReleaseSRWLockExclusive(&m_damageLock); return; } if (m_nbPendingDirtyRects + nbDirtyRects > LG_MAX_DIRTY_RECTS) { m_nbPendingDirtyRects = 0; + ReleaseSRWLockExclusive(&m_damageLock); return; } memcpy(m_pendingDirtyRects + m_nbPendingDirtyRects, dirtyRects, nbDirtyRects * sizeof(*m_pendingDirtyRects)); m_nbPendingDirtyRects += nbDirtyRects; + ReleaseSRWLockExclusive(&m_damageLock); +} + +int CSwapChainProcessor::AcquireCandidate() +{ + HANDLE waitHandles[] = + { + m_candidateAvailableEvent.Get(), + m_terminateEvent.Get(), + }; + + for (;;) + { + int selected = -1; + uint64_t oldest = UINT64_MAX; + bool superseded = false; + + AcquireSRWLockExclusive(&m_candidateLock); + for (unsigned i = 0; i < ARRAYSIZE(m_candidates); ++i) + if (m_candidates[i].state == CANDIDATE_FREE) + { + selected = static_cast(i); + break; + } + + unsigned readyCount = 0; + for (const FrameCandidate& candidate : m_candidates) + if (candidate.state == CANDIDATE_READY) + ++readyCount; + + if (selected < 0 && readyCount > 1) + for (unsigned i = 0; i < ARRAYSIZE(m_candidates); ++i) + if (m_candidates[i].state == CANDIDATE_READY && + m_candidates[i].sequence < oldest) + { + selected = static_cast(i); + oldest = m_candidates[i].sequence; + } + + if (selected >= 0) + { + FrameCandidate& candidate = + m_candidates[static_cast(selected)]; + superseded = candidate.state == CANDIDATE_READY; + candidate.state = CANDIDATE_PREPARING; + candidate.sequence = ++m_candidateSequence; + } + ReleaseSRWLockExclusive(&m_candidateLock); + + if (selected >= 0) + { + if (superseded) + m_devContext->FrameSuperseded(); + return selected; + } + + const DWORD result = WaitForMultipleObjects( + ARRAYSIZE(waitHandles), waitHandles, FALSE, CANDIDATE_WAIT_MS); + if (result == WAIT_OBJECT_0 + 1) + return -1; + if (result == WAIT_TIMEOUT) + return -1; + if (result != WAIT_OBJECT_0) + return -1; + } +} + +void CSwapChainProcessor::ReleaseCandidate(unsigned candidateIndex) +{ + if (candidateIndex >= ARRAYSIZE(m_candidates)) + return; + + AcquireSRWLockExclusive(&m_candidateLock); + m_candidates[candidateIndex].state = CANDIDATE_FREE; + ReleaseSRWLockExclusive(&m_candidateLock); + SignalCandidateState(); +} + +static bool ResourceDescMatches( + const D3D12_RESOURCE_DESC& left, const D3D12_RESOURCE_DESC& right) +{ + return + left.Dimension == right.Dimension && + left.Alignment == right.Alignment && + left.Width == right.Width && + left.Height == right.Height && + left.DepthOrArraySize == right.DepthOrArraySize && + left.MipLevels == right.MipLevels && + left.Format == right.Format && + left.SampleDesc.Count == right.SampleDesc.Count && + left.SampleDesc.Quality == right.SampleDesc.Quality && + left.Layout == right.Layout && + left.Flags == right.Flags; +} + +bool CSwapChainProcessor::EnsureCandidateResource( + unsigned candidateIndex, ID3D12Resource * source) +{ + FrameCandidate& candidate = m_candidates[candidateIndex]; + D3D12_RESOURCE_DESC desc = source->GetDesc(); + desc.Alignment = 0; + desc.Flags = static_cast( + static_cast(desc.Flags) & + ~static_cast(D3D12_RESOURCE_FLAG_ALLOW_CROSS_ADAPTER)); + + if (candidate.resource && + ResourceDescMatches(candidate.resource->GetDesc(), desc)) + return true; + + candidate.resource.Reset(); + + D3D12_HEAP_PROPERTIES heapProps = {}; + heapProps.Type = D3D12_HEAP_TYPE_DEFAULT; + heapProps.CPUPageProperty = D3D12_CPU_PAGE_PROPERTY_UNKNOWN; + heapProps.MemoryPoolPreference = D3D12_MEMORY_POOL_UNKNOWN; + heapProps.CreationNodeMask = 1; + heapProps.VisibleNodeMask = 1; + + const HRESULT hr = m_dx12Device->GetDevice()->CreateCommittedResource( + &heapProps, D3D12_HEAP_FLAG_NONE, &desc, D3D12_RESOURCE_STATE_COMMON, + nullptr, IID_PPV_ARGS(&candidate.resource)); + if (FAILED(hr)) + { + DEBUG_ERROR_HR(hr, "Failed to create retained frame candidate"); + return false; + } + + static const WCHAR * names[] = + { + L"Frame Candidate 0", + L"Frame Candidate 1", + }; + candidate.resource->SetName(names[candidateIndex]); + return true; +} + +void CSwapChainProcessor::ResetCandidates() +{ + AcquireSRWLockExclusive(&m_candidateLock); + for (FrameCandidate& candidate : m_candidates) + candidate = {}; + ReleaseSRWLockExclusive(&m_candidateLock); + SignalCandidateState(); +} + +void CSwapChainProcessor::SignalCandidateState() +{ + SetEvent(m_candidateEvent.Get()); + SetEvent(m_candidateAvailableEvent.Get()); +} + +bool CSwapChainProcessor::PublishNewestCandidate( + uint32_t scheduleGeneration, bool periodic, uint64_t publishStart) +{ + int selectedCandidate = -1; + uint64_t newestSequence = 0; + + AcquireSRWLockExclusive(&m_candidateLock); + for (unsigned i = 0; i < ARRAYSIZE(m_candidates); ++i) + if (m_candidates[i].state == CANDIDATE_READY && + (selectedCandidate < 0 || + m_candidates[i].sequence > newestSequence)) + { + selectedCandidate = static_cast(i); + newestSequence = m_candidates[i].sequence; + } + + if (selectedCandidate >= 0) + for (unsigned i = 0; i < ARRAYSIZE(m_candidates); ++i) + if (static_cast(i) == selectedCandidate) + m_candidates[i].state = CANDIDATE_PUBLISHING; + else if (m_candidates[i].state == CANDIDATE_READY) + m_candidates[i].state = CANDIDATE_HELD; + ReleaseSRWLockExclusive(&m_candidateLock); + + if (selectedCandidate < 0) + return false; + const unsigned candidateIndex = + static_cast(selectedCandidate); + + const auto restoreCandidates = [this, candidateIndex]() + { + AcquireSRWLockExclusive(&m_candidateLock); + if (m_candidates[candidateIndex].state == CANDIDATE_PUBLISHING) + m_candidates[candidateIndex].state = CANDIDATE_READY; + for (FrameCandidate& candidate : m_candidates) + if (candidate.state == CANDIDATE_HELD) + candidate.state = CANDIDATE_READY; + ReleaseSRWLockExclusive(&m_candidateLock); + SignalCandidateState(); + }; + + CSRWExclusiveLock pipelineLock(&m_pipelineLock); + AcquireSRWLockShared(&m_candidateLock); + const bool candidateValid = + m_candidates[candidateIndex].state == CANDIDATE_PUBLISHING && + m_candidates[candidateIndex].resource.Get(); + ReleaseSRWLockShared(&m_candidateLock); + if (!candidateValid) + { + restoreCandidates(); + return false; + } + + FrameCandidate& candidate = m_candidates[candidateIndex]; + CPostProcessor& postProcessor = m_postProcessors[candidateIndex]; + + auto buffer = m_devContext->PrepareFrameBuffer( + candidate.pitch, + candidate.srcFormat, + candidate.dstFormat, + candidate.dirtyRects, + candidate.nbDirtyRects); + if (!buffer.mem) + { + restoreCandidates(); + return false; + } + + CFrameBufferResource * fbRes = + m_fbPool.Get(buffer, candidate.frameSize); + if (!fbRes) + { + m_devContext->AbortFrameBuffer(buffer.frameIndex); + restoreCandidates(); + DEBUG_ERROR("Failed to get a CFrameBufferResource from the pool"); + SetFullPendingDamage(); + return false; + } + + CD3D12CommandSlot * copySlot = + m_dx12Device->GetCopySlot(candidateIndex); + if (!copySlot) + { + m_devContext->AbortFrameBuffer(buffer.frameIndex); + restoreCandidates(); + DEBUG_ERROR("Failed to get a copy CommandSlot for publication"); + SetFullPendingDamage(); + return false; + } + + RECT previousDirtyRects[LG_MAX_DIRTY_RECTS] = {}; + unsigned nbPreviousDirtyRects = 0; + AcquireSRWLockShared(&m_damageLock); + nbPreviousDirtyRects = m_nbDirtyRects; + if (nbPreviousDirtyRects) + memcpy(previousDirtyRects, m_dirtyRects, + nbPreviousDirtyRects * sizeof(*previousDirtyRects)); + ReleaseSRWLockShared(&m_damageLock); + + RECT copyDirtyRects[LG_MAX_DIRTY_RECTS * 2] = {}; + unsigned nbCopyDirtyRects = 0; + bool fullCopy = + candidate.nbDirtyRects == 0 || nbPreviousDirtyRects == 0; + + if (!fullCopy) + { + for (const RECT * rect = previousDirtyRects; + rect < previousDirtyRects + nbPreviousDirtyRects && !fullCopy; + ++rect) + { + RECT clipped = *rect; + if (ClipDirtyRect(clipped, + candidate.dstFormat.width, candidate.dstFormat.height) && + !AddCopyDirtyRect(copyDirtyRects, ARRAYSIZE(copyDirtyRects), + &nbCopyDirtyRects, clipped)) + fullCopy = true; + } + + for (const RECT * rect = candidate.dirtyRects; + rect < candidate.dirtyRects + candidate.nbDirtyRects && !fullCopy; + ++rect) + if (!AddCopyDirtyRect(copyDirtyRects, ARRAYSIZE(copyDirtyRects), + &nbCopyDirtyRects, *rect)) + fullCopy = true; + + if (!fullCopy) + fullCopy = IsFullDamage( + copyDirtyRects, nbCopyDirtyRects, + candidate.dstFormat.width, candidate.dstFormat.height) || + CopyAreaCoversFrame( + copyDirtyRects, nbCopyDirtyRects, + candidate.dstFormat.width, candidate.dstFormat.height); + + if (!fullCopy) + fullCopy = postProcessor.ShouldCopyFully( + copyDirtyRects, nbCopyDirtyRects); + } + + fbRes->SetTiming( + candidate.captureTime, candidate.postProcessStart, publishStart); + fbRes->SetCandidateIndex(candidateIndex); + fbRes->SetPostProcessSample( + candidate.timingEffectIndex, candidate.timingToken, fullCopy); + copySlot->SetCompletionCallback(&CompletionFunction, this, fbRes); + + copySlot->BeginTiming(); + postProcessor.CopyFrame( + copySlot->GetGfxList(), fbRes->Get().Get(), candidate.resource.Get(), + copyDirtyRects, nbCopyDirtyRects, fullCopy); + copySlot->EndTiming(); + + // Reserve the LGMP message before submitting the copy. This makes post + // failure recoverable without racing a very fast GPU completion callback. + if (!m_devContext->PublishFrameBuffer( + buffer.frameIndex, scheduleGeneration)) + { + copySlot->Cancel(); + m_devContext->AbortFrameBuffer(buffer.frameIndex); + restoreCandidates(); + return false; + } + + if (!copySlot->Execute()) + { + AcquireSRWLockShared(&m_candidateLock); + const bool callbackPending = + candidate.state == CANDIDATE_PUBLISHING; + ReleaseSRWLockShared(&m_candidateLock); + if (callbackPending && !copySlot->HasSubmittedWork()) + { + m_devContext->FailFrameBuffer(buffer.frameIndex); + SetFullPendingDamage(); + ReleaseCandidate(candidateIndex); + } + m_devContext->ForceFrame(); + + AcquireSRWLockExclusive(&m_candidateLock); + for (FrameCandidate& held : m_candidates) + if (held.state == CANDIDATE_HELD) + held.state = CANDIDATE_READY; + ReleaseSRWLockExclusive(&m_candidateLock); + SignalCandidateState(); + return false; + } + + AcquireSRWLockExclusive(&m_damageLock); + if (candidate.nbDirtyRects) + memcpy(m_dirtyRects, candidate.dirtyRects, + candidate.nbDirtyRects * sizeof(*m_dirtyRects)); + m_nbDirtyRects = candidate.nbDirtyRects; + if (candidate.damageGeneration == m_damageGeneration) + { + m_hasPendingDamage = false; + m_nbPendingDirtyRects = 0; + } + ReleaseSRWLockExclusive(&m_damageLock); + + m_devContext->CommitFrameBuffer( + buffer.frameIndex, scheduleGeneration, periodic); + + unsigned superseded = 0; + AcquireSRWLockExclusive(&m_candidateLock); + for (FrameCandidate& held : m_candidates) + if (held.state == CANDIDATE_HELD) + { + held.state = CANDIDATE_FREE; + ++superseded; + } + ReleaseSRWLockExclusive(&m_candidateLock); + for (unsigned i = 0; i < superseded; ++i) + m_devContext->FrameSuperseded(); + SignalCandidateState(); + return true; } #ifdef HAS_IDDCX_110 @@ -632,17 +1194,6 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer const uint64_t postProcessStart = CFrameScheduler::Nanotime(); const uint64_t captureTime = postProcessStart - captureStart; - m_devContext->ObserveFrame(postProcessStart); - - // Preserve the fast drop path: never hold an IddCx frame while waiting for - // a slow or disconnected client. We have not read its rectangles, so force - // the next published frame to invalidate the entire image. - if (!m_devContext->FrameBufferAvailable()) - { - SetFullPendingDamage(); - return true; - } - ComPtr texture; HRESULT hr = acquiredBuffer.As(&texture); if (FAILED(hr)) @@ -707,13 +1258,11 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer D3D12_RESOURCE_DESC srcDesc = srcRes->GetRes()->GetDesc(); if (!noImageUpdate) + { + m_devContext->ObserveFrame(postProcessStart); AccumulateFrameDamage( srcRes->GetDirtyRects(), srcRes->GetDirtyRectCount()); - - // Never hold an IddCx frame waiting for a slow or disconnected client. Read - // and retain its damage first so the next published frame remains complete. - if (!m_devContext->FrameBufferAvailable()) - return true; + } D12FrameFormat srcFormat = {}; srcFormat.desc = srcDesc; @@ -795,136 +1344,128 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer break; } - m_postProcessors[0].Update(srcFormat); - - const bool frameMetadataChanged = noImageUpdate && - FrameMetadataChanged(m_postProcessors[0].GetOutputFormat(), srcFormat); - - bool needsReconfigure = false; - for (const CPostProcessor& postProcessor : m_postProcessors) - if (postProcessor.NeedsReconfigure(srcFormat)) - { - needsReconfigure = true; - break; - } - - // SetFormat can replace resources still being read by the COPY queue. Format - // changes are rare, so drain both queues before updating either frame chain. - if (needsReconfigure) - { - m_nbDirtyRects = 0; - SetFullPendingDamage(); - m_dx12Device->WaitForIdle(); - } - - // An optional adaptive effect can reject its candidate while configuring - // either slot. Both queues are already idle, so allow one convergence pass - // to return both effect chains to the same mode. + bool frameMetadataChanged = false; + bool needsReconfigure = false; bool postProcessFormatChanged = false; - bool configurationStable = false; - for (unsigned pass = 0; pass < 2 && !configurationStable; ++pass) + bool requiresFullDamage = false; { - for (unsigned i = 0; i < ARRAYSIZE(m_postProcessors); ++i) - { - bool formatChanged = false; - if (!m_postProcessors[i].Configure(srcFormat, &formatChanged)) - { - SetFullPendingDamage(); - return false; - } + CSRWExclusiveLock pipelineLock(&m_pipelineLock); + m_postProcessors[0].Update(srcFormat); - if (i == 0) - postProcessFormatChanged |= formatChanged; - } + frameMetadataChanged = noImageUpdate && + FrameMetadataChanged( + m_postProcessors[0].GetOutputFormat(), srcFormat); - configurationStable = true; for (const CPostProcessor& postProcessor : m_postProcessors) if (postProcessor.NeedsReconfigure(srcFormat)) { - configurationStable = false; + needsReconfigure = true; break; } + + // A format change can replace resources referenced by either retained + // candidate. Stop publication, drain both queues, then invalidate them. + if (needsReconfigure) + { + AcquireSRWLockExclusive(&m_damageLock); + m_nbDirtyRects = 0; + ReleaseSRWLockExclusive(&m_damageLock); + SetFullPendingDamage(); + m_dx12Device->WaitForIdle(); + ResetCandidates(); + } + + bool configurationStable = false; + for (unsigned pass = 0; pass < 2 && !configurationStable; ++pass) + { + for (unsigned i = 0; i < ARRAYSIZE(m_postProcessors); ++i) + { + bool formatChanged = false; + if (!m_postProcessors[i].Configure(srcFormat, &formatChanged)) + { + SetFullPendingDamage(); + return false; + } + + if (i == 0) + postProcessFormatChanged |= formatChanged; + } + + configurationStable = true; + for (const CPostProcessor& postProcessor : m_postProcessors) + if (postProcessor.NeedsReconfigure(srcFormat)) + { + configurationStable = false; + break; + } + } + + if (!configurationStable) + { + DEBUG_ERROR("Post processor configuration did not stabilize"); + SetFullPendingDamage(); + return false; + } + + if (postProcessFormatChanged) + { + AcquireSRWLockExclusive(&m_damageLock); + m_nbDirtyRects = 0; + ReleaseSRWLockExclusive(&m_damageLock); + SetFullPendingDamage(); + } + else if (frameMetadataChanged) + SetFullPendingDamage(); + + requiresFullDamage = m_postProcessors[0].RequiresFullDamage(); + if (requiresFullDamage) + SetFullPendingDamage(); } - if (!configurationStable) + if (needsReconfigure || postProcessFormatChanged || frameMetadataChanged) + m_devContext->ForceFrame(); + + if (noImageUpdate) { - DEBUG_ERROR("Post processor configuration did not stabilize"); - SetFullPendingDamage(); - return false; + AcquireSRWLockShared(&m_damageLock); + const bool hasPendingDamage = m_hasPendingDamage; + ReleaseSRWLockShared(&m_damageLock); + if (!hasPendingDamage) + return true; } - if (postProcessFormatChanged) + const int selectedCandidate = AcquireCandidate(); + if (selectedCandidate < 0) { - m_nbDirtyRects = 0; - SetFullPendingDamage(); + m_devContext->FrameSuperseded(); + return true; } - else if (frameMetadataChanged) - SetFullPendingDamage(); + const unsigned candidateIndex = + static_cast(selectedCandidate); - // Adaptive effects need comparable full-frame samples until they lock. - const bool requiresFullDamage = - m_postProcessors[0].RequiresFullDamage(); - if (requiresFullDamage) - SetFullPendingDamage(); - - if (noImageUpdate && !m_hasPendingDamage) - return true; - - uint32_t scheduleGeneration = 0; - if (!m_devContext->SelectFrame(CFrameScheduler::Nanotime(), - needsReconfigure || postProcessFormatChanged || - frameMetadataChanged || requiresFullDamage, - scheduleGeneration)) - return true; - - const D12FrameFormat& dstFormat = - m_postProcessors[0].GetOutputFormat(); - const unsigned pitch = m_postProcessors[0].GetOutputPitch(); - const size_t frameSize = m_postProcessors[0].GetOutputSize(); + CSRWExclusiveLock pipelineLock(&m_pipelineLock); + CPostProcessor& postProcessor = m_postProcessors[candidateIndex]; + const D12FrameFormat& dstFormat = postProcessor.GetOutputFormat(); RECT currentDirtyRects[LG_MAX_DIRTY_RECTS] = {}; - RECT frameDirtyRects[LG_MAX_DIRTY_RECTS] = {}; - unsigned nbDirtyRects = m_nbPendingDirtyRects; - unsigned frameDirtyRectCount = nbDirtyRects; - if (nbDirtyRects) + unsigned nbDirtyRects = 0; + uint64_t damageGeneration = 0; + AcquireSRWLockShared(&m_damageLock); + if (m_hasPendingDamage) { - memcpy(currentDirtyRects, m_pendingDirtyRects, - nbDirtyRects * sizeof(*currentDirtyRects)); - memcpy(frameDirtyRects, currentDirtyRects, - nbDirtyRects * sizeof(*frameDirtyRects)); + nbDirtyRects = m_nbPendingDirtyRects; + if (nbDirtyRects) + memcpy(currentDirtyRects, m_pendingDirtyRects, + nbDirtyRects * sizeof(*currentDirtyRects)); + damageGeneration = m_damageGeneration; } - m_postProcessors[0].AdjustFrameDamage( - frameDirtyRects, &frameDirtyRectCount); - - auto buffer = m_devContext->PrepareFrameBuffer( - pitch, - srcFormat, - dstFormat, - frameDirtyRects, - frameDirtyRectCount); - - // Queue or framebuffer ownership can change after the early availability - // check. Treat this as a dropped frame rather than an error. - if (!buffer.mem) - return true; - - CFrameBufferResource * fbRes = m_fbPool.Get(buffer, frameSize); - - if (!fbRes) - { - m_devContext->AbortFrameBuffer(buffer.frameIndex); - DEBUG_ERROR("Failed to get a CFrameBufferResource from the pool"); - SetFullPendingDamage(); - return false; - } - - CPostProcessor& postProcessor = m_postProcessors[buffer.frameIndex]; + ReleaseSRWLockShared(&m_damageLock); CD3D12CommandSlot * copySlot = - m_dx12Device->GetCopySlot(buffer.frameIndex); + m_dx12Device->GetCopySlot(candidateIndex); if (!copySlot) { - m_devContext->AbortFrameBuffer(buffer.frameIndex); + ReleaseCandidate(candidateIndex); DEBUG_ERROR("Failed to get a copy CommandSlot"); SetFullPendingDamage(); return false; @@ -934,11 +1475,11 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer CD3D12CommandSlot * computeSlot = nullptr; if (postProcessor.HasActiveEffects()) { - computeSlot = m_dx12Device->GetComputeSlot(buffer.frameIndex); + computeSlot = m_dx12Device->GetComputeSlot(candidateIndex); if (!computeSlot) { copySlot->Cancel(); - m_devContext->AbortFrameBuffer(buffer.frameIndex); + ReleaseCandidate(candidateIndex); DEBUG_ERROR("Failed to get a compute CommandSlot"); SetFullPendingDamage(); return false; @@ -948,7 +1489,7 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer { computeSlot->Cancel(); copySlot->Cancel(); - m_devContext->AbortFrameBuffer(buffer.frameIndex); + ReleaseCandidate(candidateIndex); SetFullPendingDamage(); return false; } @@ -960,7 +1501,7 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer { computeSlot->Cancel(); copySlot->Cancel(); - m_devContext->AbortFrameBuffer(buffer.frameIndex); + ReleaseCandidate(candidateIndex); DEBUG_ERROR("Post processor returned no output resource"); SetFullPendingDamage(); return false; @@ -970,7 +1511,7 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer { copySlot->Cancel(); m_dx12Device->WaitForIdle(); - m_devContext->AbortFrameBuffer(buffer.frameIndex); + ReleaseCandidate(candidateIndex); SetFullPendingDamage(); return false; } @@ -979,7 +1520,7 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer { copySlot->Cancel(); m_dx12Device->WaitForIdle(); - m_devContext->AbortFrameBuffer(buffer.frameIndex); + ReleaseCandidate(candidateIndex); DEBUG_ERROR("Failed to queue compute synchronization"); SetFullPendingDamage(); return false; @@ -988,7 +1529,7 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer else if (!srcRes->Sync(*copySlot)) { copySlot->Cancel(); - m_devContext->AbortFrameBuffer(buffer.frameIndex); + ReleaseCandidate(candidateIndex); DEBUG_ERROR("Failed to queue source synchronization"); SetFullPendingDamage(); return false; @@ -997,62 +1538,59 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer ClipDirtyRects(currentDirtyRects, &nbDirtyRects, dstFormat.width, dstFormat.height); - const uint64_t copyStart = CFrameScheduler::Nanotime(); - fbRes->SetTiming(captureTime, postProcessStart, copyStart); - - copySlot->SetCompletionCallback(&CompletionFunction, this, fbRes); - - /* Each destination is reused every other frame, so repair both the prior - * and current damage. Coalesce them first to avoid copying overlapping - * regions, especially a prior full frame, more than once. */ - RECT copyDirtyRects[LG_MAX_DIRTY_RECTS * 2] = {}; - unsigned nbCopyDirtyRects = 0; - bool fullCopy = - nbDirtyRects == 0 || m_nbDirtyRects == 0; - - if (!fullCopy) + if (!EnsureCandidateResource(candidateIndex, copySrcResource.Get())) { - for (const RECT * rect = m_dirtyRects; - rect < m_dirtyRects + m_nbDirtyRects && !fullCopy; ++rect) - { - RECT clipped = *rect; - if (ClipDirtyRect(clipped, dstFormat.width, dstFormat.height) && - !AddCopyDirtyRect(copyDirtyRects, ARRAYSIZE(copyDirtyRects), - &nbCopyDirtyRects, clipped)) - fullCopy = true; - } - - for (const RECT * rect = currentDirtyRects; - rect < currentDirtyRects + nbDirtyRects && !fullCopy; ++rect) - if (!AddCopyDirtyRect(copyDirtyRects, ARRAYSIZE(copyDirtyRects), - &nbCopyDirtyRects, *rect)) - fullCopy = true; - - if (!fullCopy) - fullCopy = IsFullDamage( - copyDirtyRects, nbCopyDirtyRects, - dstFormat.width, dstFormat.height) || - CopyAreaCoversFrame( - copyDirtyRects, nbCopyDirtyRects, - dstFormat.width, dstFormat.height); - - if (!fullCopy) - fullCopy = postProcessor.ShouldCopyFully( - copyDirtyRects, nbCopyDirtyRects); + copySlot->Cancel(); + if (computeSlot) + m_dx12Device->WaitForIdle(); + ReleaseCandidate(candidateIndex); + SetFullPendingDamage(); + return false; } - unsigned timingEffectIndex = 0; - uint64_t timingToken = 0; - postProcessor.GetTimingToken(&timingEffectIndex, &timingToken); - fbRes->SetPostProcessSample( - timingEffectIndex, timingToken, fullCopy); + FrameCandidate& candidate = m_candidates[candidateIndex]; + candidate.srcFormat = srcFormat; + candidate.dstFormat = dstFormat; + candidate.nbDirtyRects = nbDirtyRects; + candidate.pitch = postProcessor.GetOutputPitch(); + candidate.frameSize = postProcessor.GetOutputSize(); + candidate.damageGeneration = damageGeneration; + candidate.captureTime = captureTime; + candidate.postProcessStart = postProcessStart; + candidate.prepareCopyStart = CFrameScheduler::Nanotime(); + candidate.prepareReady = 0; + candidate.prepareGPUStart = 0; + candidate.prepareGPUEnd = 0; + candidate.prepareTimingValid = false; + if (nbDirtyRects) + memcpy(candidate.dirtyRects, currentDirtyRects, + nbDirtyRects * sizeof(*candidate.dirtyRects)); + postProcessor.GetTimingToken( + &candidate.timingEffectIndex, &candidate.timingToken); - // Source/compute waits are submitted immediately before this command list. - // The timestamp therefore marks the first actual copy operation. + copySlot->SetCompletionCallback( + &CandidateCompletionFunction, this, &candidate); copySlot->BeginTiming(); - postProcessor.CopyFrame( - copySlot->GetGfxList(), fbRes->Get().Get(), copySrcResource.Get(), - copyDirtyRects, nbCopyDirtyRects, fullCopy); + const D3D12_RESOURCE_DESC copySrcDesc = copySrcResource->GetDesc(); + if (copySrcDesc.Dimension == D3D12_RESOURCE_DIMENSION_BUFFER) + copySlot->GetGfxList()->CopyBufferRegion( + candidate.resource.Get(), 0, copySrcResource.Get(), 0, + copySrcDesc.Width); + else + { + D3D12_TEXTURE_COPY_LOCATION srcLocation = {}; + srcLocation.pResource = copySrcResource.Get(); + srcLocation.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX; + srcLocation.SubresourceIndex = 0; + + D3D12_TEXTURE_COPY_LOCATION dstLocation = {}; + dstLocation.pResource = candidate.resource.Get(); + dstLocation.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX; + dstLocation.SubresourceIndex = 0; + + copySlot->GetGfxList()->CopyTextureRegion( + &dstLocation, 0, 0, 0, &srcLocation, nullptr); + } copySlot->EndTiming(); if (!copySlot->Execute()) @@ -1061,25 +1599,13 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer { if (computeSlot) m_dx12Device->WaitForIdle(); - m_devContext->AbortFrameBuffer(buffer.frameIndex); + ReleaseCandidate(candidateIndex); } SetFullPendingDamage(); + m_devContext->ForceFrame(); return false; } - if (!m_devContext->PublishFrameBuffer( - buffer.frameIndex, scheduleGeneration)) - { - SetFullPendingDamage(); - return false; - } - - memcpy(m_dirtyRects, currentDirtyRects, - nbDirtyRects * sizeof(*m_dirtyRects)); - m_nbDirtyRects = nbDirtyRects; - m_hasPendingDamage = false; - m_nbPendingDirtyRects = 0; - return true; } diff --git a/idd/LGIdd/CSwapChainProcessor.h b/idd/LGIdd/CSwapChainProcessor.h index 83135190..e0cc21e0 100644 --- a/idd/LGIdd/CSwapChainProcessor.h +++ b/idd/LGIdd/CSwapChainProcessor.h @@ -55,8 +55,50 @@ private: CFrameBufferPool m_fbPool; CPostProcessor m_postProcessors[LGMP_Q_FRAME_LEN]; - Wrappers::HandleT m_thread[2]; + enum CandidateState + { + CANDIDATE_FREE, + CANDIDATE_PREPARING, + CANDIDATE_READY, + CANDIDATE_PUBLISHING, + CANDIDATE_HELD, + }; + + struct FrameCandidate + { + CandidateState state = CANDIDATE_FREE; + ComPtr resource; + D12FrameFormat srcFormat = {}; + D12FrameFormat dstFormat = {}; + RECT dirtyRects[LG_MAX_DIRTY_RECTS] = {}; + unsigned nbDirtyRects = 0; + unsigned pitch = 0; + size_t frameSize = 0; + uint64_t sequence = 0; + uint64_t damageGeneration = 0; + uint64_t captureTime = 0; + uint64_t postProcessStart = 0; + uint64_t prepareCopyStart = 0; + uint64_t prepareReady = 0; + uint64_t prepareGPUStart = 0; + uint64_t prepareGPUEnd = 0; + unsigned timingEffectIndex = 0; + uint64_t timingToken = 0; + bool prepareTimingValid = false; + }; + + FrameCandidate m_candidates[LGMP_Q_FRAME_LEN]; + SRWLOCK m_candidateLock = SRWLOCK_INIT; + SRWLOCK m_damageLock = SRWLOCK_INIT; + SRWLOCK m_pipelineLock = SRWLOCK_INIT; + uint64_t m_candidateSequence = 0; + uint64_t m_damageGeneration = 0; + + Wrappers::HandleT m_thread[3]; Wrappers::Event m_terminateEvent; + Wrappers::Event m_candidateEvent; + Wrappers::Event m_candidateAvailableEvent; + Wrappers::HandleT m_publishTimer; Wrappers::Event m_cursorDataEvent; BYTE* m_shapeBuffer; @@ -89,12 +131,26 @@ private: void SwapChainThread(); bool SwapChainThreadCore(); + static DWORD CALLBACK _PublisherThread(LPVOID arg); + void PublisherThread(); + bool PublishNewestCandidate( + uint32_t scheduleGeneration, bool periodic, uint64_t publishStart); + bool HasReadyCandidate(); + int AcquireCandidate(); + void ReleaseCandidate(unsigned candidateIndex); + bool EnsureCandidateResource(unsigned candidateIndex, + ID3D12Resource * source); + void ResetCandidates(); + void SignalCandidateState(); + static DWORD CALLBACK _CursorThread(LPVOID arg); bool QueryHWCursor(); void CursorThread(); static void CompletionFunction( CD3D12CommandSlot * slot, bool result, void * param1, void * param2); + static void CandidateCompletionFunction( + CD3D12CommandSlot * slot, bool result, void * param1, void * param2); void AccumulateFrameDamage(const RECT * dirtyRects, unsigned nbDirtyRects); void SetFullPendingDamage(); #ifdef HAS_IDDCX_110