diff --git a/idd/LGIdd/capture/CFrameBufferPool.cpp b/idd/LGIdd/capture/CFrameBufferPool.cpp index d63d62f7..214117cd 100644 --- a/idd/LGIdd/capture/CFrameBufferPool.cpp +++ b/idd/LGIdd/capture/CFrameBufferPool.cpp @@ -19,15 +19,12 @@ */ #include "capture/CFrameBufferPool.h" -#include "transport/IFrameTransport.h" - #include void CFrameBufferPool::Init( - IFrameTransport * transport, CD3D12Device * dx12) + IFrameTransport *, CD3D12Device * dx12) { - m_transport = transport; - m_dx12 = dx12; + m_dx12 = dx12; } void CFrameBufferPool::Reset() @@ -44,7 +41,7 @@ CFrameBufferResource * CFrameBufferPool::Get( CFrameBufferResource * fbr = &m_buffers[buffer.resourceSlot]; if (!fbr->Init(m_dx12, buffer.token, buffer.mem, - buffer.heapOffset, minSize, m_transport->GetMaxFrameSize())) + buffer.heapOffset, minSize, buffer.capacity, buffer.direct)) return nullptr; return fbr; diff --git a/idd/LGIdd/capture/CFrameBufferPool.h b/idd/LGIdd/capture/CFrameBufferPool.h index d481e7eb..7bfcd26b 100644 --- a/idd/LGIdd/capture/CFrameBufferPool.h +++ b/idd/LGIdd/capture/CFrameBufferPool.h @@ -30,13 +30,12 @@ class IFrameTransport; class CFrameBufferPool { private: - IFrameTransport * m_transport = nullptr; - CD3D12Device * m_dx12 = nullptr; + CD3D12Device * m_dx12 = nullptr; - CFrameBufferResource m_buffers[CAPTURE_FRAME_BUFFERS]; + CFrameBufferResource m_buffers[FRAME_BUFFER_RESOURCES]; public: - void Init(IFrameTransport * transport, CD3D12Device * dx12); + void Init(IFrameTransport *, CD3D12Device * dx12); void Reset(); CFrameBufferResource * Get( diff --git a/idd/LGIdd/capture/CFrameBufferResource.cpp b/idd/LGIdd/capture/CFrameBufferResource.cpp index 912cd9fd..8bbe3c0e 100644 --- a/idd/LGIdd/capture/CFrameBufferResource.cpp +++ b/idd/LGIdd/capture/CFrameBufferResource.cpp @@ -26,7 +26,7 @@ bool CFrameBufferResource::Init(CD3D12Device * dx12, const FrameToken& token, uint8_t * base, uint64_t heapOffset, size_t size, - size_t maxFrameSize) + size_t maxFrameSize, bool direct) { if (size > maxFrameSize) { @@ -35,7 +35,7 @@ bool CFrameBufferResource::Init(CD3D12Device * dx12, return false; } - const bool indirect = dx12->IsIndirectCopy(); + const bool indirect = !direct || dx12->IsIndirectCopy(); D3D12_RESOURCE_DESC desc = {}; desc.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER; @@ -55,7 +55,9 @@ bool CFrameBufferResource::Init(CD3D12Device * dx12, } // Nothing to do if the resource already represents this allocation. - if (m_base == base && m_size >= size) + if (m_base == base && m_size >= size && + m_capacity == maxFrameSize && m_heapOffset == heapOffset && + m_direct == direct) { m_token = token; m_frameSize = size; @@ -136,10 +138,13 @@ bool CFrameBufferResource::Init(CD3D12Device * dx12, m_res->SetName(resName); - m_token = token; - m_base = base; - m_size = size; - m_frameSize = size; + m_token = token; + m_base = base; + m_size = size; + m_capacity = maxFrameSize; + m_heapOffset = heapOffset; + m_frameSize = size; + m_direct = direct; return true; } @@ -154,8 +159,11 @@ void CFrameBufferResource::Reset() m_token = {}; m_base = nullptr; m_size = 0; + m_capacity = 0; + m_heapOffset = 0; m_frameSize = 0; m_fullCopy = false; + m_direct = false; m_nbCopyDirtyRects = 0; m_copyPitch = 0; m_copyBytesPerPixel = 0; diff --git a/idd/LGIdd/capture/CFrameBufferResource.h b/idd/LGIdd/capture/CFrameBufferResource.h index 4106e7af..b9a587cc 100644 --- a/idd/LGIdd/capture/CFrameBufferResource.h +++ b/idd/LGIdd/capture/CFrameBufferResource.h @@ -41,6 +41,8 @@ class CFrameBufferResource FrameToken m_token = {}; uint8_t * m_base = nullptr; size_t m_size = 0; + size_t m_capacity = 0; + uint64_t m_heapOffset = 0; size_t m_frameSize = 0; uint64_t m_captureTime = 0; uint64_t m_postProcessStart = 0; @@ -51,6 +53,7 @@ class CFrameBufferResource unsigned m_timingEffectIndex = 0; uint64_t m_timingToken = 0; bool m_fullCopy = false; + bool m_direct = false; RECT m_copyDirtyRects[LG_MAX_DIRTY_RECTS * 2] = {}; unsigned m_nbCopyDirtyRects = 0; unsigned m_copyPitch = 0; @@ -62,7 +65,7 @@ class CFrameBufferResource public: bool Init(CD3D12Device * dx12, const FrameToken& token, uint8_t * base, - uint64_t heapOffset, size_t size, size_t maxFrameSize); + uint64_t heapOffset, size_t size, size_t maxFrameSize, bool direct); void Reset(); const FrameToken& GetToken() const { return m_token; } diff --git a/idd/LGIdd/capture/CFrameProcessor.h b/idd/LGIdd/capture/CFrameProcessor.h index 5c1f3385..cbc7cd8c 100644 --- a/idd/LGIdd/capture/CFrameProcessor.h +++ b/idd/LGIdd/capture/CFrameProcessor.h @@ -34,6 +34,15 @@ using namespace Microsoft::WRL; class IFrameTransport; +struct FramePlan; + +struct FrameCopyBatch +{ + PreparedFrameBatch prepared; + CFrameBufferResource * resources[FRAME_MAX_SINKS] = {}; + uint32_t accepted = 0; + unsigned candidateIndex = 0; +}; struct FrameSubmission { @@ -84,8 +93,8 @@ public: virtual bool IsValid() const; virtual bool Submit(const FrameSubmission& submission) = 0; virtual bool HasReadyFrame() const = 0; - virtual bool Publish(const CFrameScheduler::Schedule& schedule, - bool periodic, uint64_t publishStart) = 0; + virtual bool Publish( + const FramePlan& plan, uint64_t publishStart) = 0; virtual bool UsesCadence() const = 0; virtual void Reset(); virtual void Invalidate(); diff --git a/idd/LGIdd/capture/CFrameScheduler.cpp b/idd/LGIdd/capture/CFrameScheduler.cpp index 596a65bc..91ba9579 100644 --- a/idd/LGIdd/capture/CFrameScheduler.cpp +++ b/idd/LGIdd/capture/CFrameScheduler.cpp @@ -54,6 +54,15 @@ void CFrameScheduler::WakePublisher() const { if (m_wakeEvent) SetEvent(m_wakeEvent); + CSRWSharedLock lock(m_wakeLock); + if (m_sharedWakeEvent) + SetEvent(m_sharedWakeEvent); +} + +void CFrameScheduler::SetSharedWakeEvent(HANDLE event) +{ + CSRWExclusiveLock lock(m_wakeLock); + m_sharedWakeEvent = event; } uint64_t CFrameScheduler::Nanotime() diff --git a/idd/LGIdd/capture/CFrameScheduler.h b/idd/LGIdd/capture/CFrameScheduler.h index 0630c887..eed835ea 100644 --- a/idd/LGIdd/capture/CFrameScheduler.h +++ b/idd/LGIdd/capture/CFrameScheduler.h @@ -103,7 +103,9 @@ private: static const unsigned WORK_TIMING_HISTORY_SIZE = 32; mutable CSRWLock m_lock; + mutable CSRWLock m_wakeLock; HANDLE m_wakeEvent = nullptr; + HANDLE m_sharedWakeEvent = nullptr; Client m_clients[MAX_CLIENTS] = {}; Schedule m_schedule = {}; bool m_scheduling = false; @@ -162,6 +164,7 @@ public: const FrameScheduleUpdate& schedule, uint64_t now); bool GetSchedule(Schedule& schedule) const; HANDLE GetWakeEvent() const { return m_wakeEvent; } + void SetSharedWakeEvent(HANDLE event); void ObserveFrame(uint64_t now); void ForceFrame(); bool GetPublishTarget(uint64_t now, uint64_t& target, diff --git a/idd/LGIdd/capture/CHardwareFrameProcessor.cpp b/idd/LGIdd/capture/CHardwareFrameProcessor.cpp index 766bb2e6..da30244f 100644 --- a/idd/LGIdd/capture/CHardwareFrameProcessor.cpp +++ b/idd/LGIdd/capture/CHardwareFrameProcessor.cpp @@ -73,9 +73,10 @@ public: CHardwareFrameProcessor::CHardwareFrameProcessor( IFrameTransport * transport, std::shared_ptr dx12, CPostProcessor postProcessors[CAPTURE_PIPELINE_SLOTS], - CSRWLock * pipelineLock, HANDLE terminateEvent) : + CSRWLock * pipelineLock, HANDLE terminateEvent, bool useCadence) : CFrameProcessor(transport, std::move(dx12), postProcessors, - pipelineLock, terminateEvent) + pipelineLock, terminateEvent), + m_useCadence(useCadence) { m_candidateAvailableEvent.Attach( CreateEvent(nullptr, FALSE, FALSE, nullptr)); @@ -144,15 +145,19 @@ void CHardwareFrameProcessor::ResetPipeline() bool CHardwareFrameProcessor::HasReadyFrame() const { - bool ready = false; - CSRWSharedLock lock(m_candidateLock); - for (const FrameCandidate& candidate : m_candidates) - if (candidate.state == CANDIDATE_READY) + bool ready = false; + bool retained = false; + { + CSRWSharedLock lock(m_candidateLock); + for (const FrameCandidate& candidate : m_candidates) { - ready = true; - break; + if (candidate.state == CANDIDATE_READY) + ready = true; + else if (candidate.state == CANDIDATE_RETAINED) + retained = true; } - return ready; + } + return ready || (retained && m_transport->NeedsFrame()); } int CHardwareFrameProcessor::AcquireCandidate( @@ -163,12 +168,18 @@ int CHardwareFrameProcessor::AcquireCandidate( bool superseded = false; bool idle = true; bool publishing = false; + bool retained = false; { CSRWExclusiveLock lock(m_candidateLock); for (unsigned i = 0; i < ARRAYSIZE(m_candidates); ++i) { - if (m_candidates[i].state != CANDIDATE_FREE) + if (m_candidates[i].state == CANDIDATE_RETAINED) + { + retained = true; + continue; + } + else if (m_candidates[i].state != CANDIDATE_FREE) { idle = false; if (m_candidates[i].state == CANDIDATE_PUBLISHING) @@ -187,7 +198,7 @@ int CHardwareFrameProcessor::AcquireCandidate( ++readyCount; if (allowSupersede && !exclusiveSample && selected < 0 && - readyCount > (publishing ? 0U : 1U)) + readyCount > ((publishing || retained) ? 0U : 1U)) for (unsigned i = 0; i < ARRAYSIZE(m_candidates); ++i) if (m_candidates[i].state == CANDIDATE_READY && m_candidates[i].sequence < oldest) @@ -202,7 +213,7 @@ int CHardwareFrameProcessor::AcquireCandidate( m_candidates[static_cast(selected)]; superseded = candidate.state == CANDIDATE_READY; candidate.state = CANDIDATE_PREPARING; - candidate.sequence = ++m_candidateSequence; + candidate.sequence = m_transport->NextContentSerial(); } } @@ -223,6 +234,44 @@ void CHardwareFrameProcessor::ReleaseCandidate(unsigned candidateIndex) SignalCandidateState(); } +void CHardwareFrameProcessor::RetainCandidate(unsigned candidateIndex) +{ + if (candidateIndex >= ARRAYSIZE(m_candidates)) + return; + + unsigned superseded = 0; + { + CSRWExclusiveLock lock(m_candidateLock); + FrameCandidate& candidate = m_candidates[candidateIndex]; + if (candidate.state != CANDIDATE_PUBLISHING) + return; + + bool newer = false; + for (unsigned i = 0; i < ARRAYSIZE(m_candidates); ++i) + if (i != candidateIndex) + { + FrameCandidate& current = m_candidates[i]; + const bool complete = current.state == CANDIDATE_READY || + current.state == CANDIDATE_PUBLISHING || + current.state == CANDIDATE_RETAINED; + if (complete && current.sequence > candidate.sequence) + newer = true; + else if (current.sequence < candidate.sequence && + (current.state == CANDIDATE_READY || + current.state == CANDIDATE_RETAINED)) + { + if (current.state == CANDIDATE_READY) + ++superseded; + current.state = CANDIDATE_FREE; + } + } + candidate.state = newer ? CANDIDATE_FREE : CANDIDATE_RETAINED; + } + for (unsigned i = 0; i < superseded; ++i) + m_transport->FrameSuperseded(); + SignalCandidateState(); +} + bool CHardwareFrameProcessor::EnsureCandidateResource( unsigned candidateIndex, size_t frameSize) { @@ -312,7 +361,8 @@ void CHardwareFrameProcessor::CandidateCompletionFunction( uint64_t gpuEnd = 0; const bool timingValid = result && slot->GetGPUTimes(gpuStart, gpuEnd); - bool forceFrame = false; + uint64_t readySequence = 0; + bool forceFrame = false; { CSRWExclusiveLock lock(processor->m_candidateLock); if (candidate->state == CANDIDATE_PREPARING) @@ -321,9 +371,28 @@ void CHardwareFrameProcessor::CandidateCompletionFunction( candidate->prepareGPUStart = gpuStart; candidate->prepareGPUEnd = gpuEnd; candidate->prepareTimingValid = timingValid; - candidate->state = - result ? CANDIDATE_READY : CANDIDATE_FREE; - forceFrame = result && candidate->timingToken != 0; + if (result) + { + bool newer = false; + for (FrameCandidate& current : processor->m_candidates) + if (¤t != candidate) + { + const bool complete = current.state == CANDIDATE_READY || + current.state == CANDIDATE_PUBLISHING || + current.state == CANDIDATE_RETAINED; + if (complete && current.sequence > candidate->sequence) + newer = true; + else if (current.state == CANDIDATE_RETAINED) + current.state = CANDIDATE_FREE; + } + candidate->state = newer ? CANDIDATE_FREE : CANDIDATE_READY; + } + else + candidate->state = CANDIDATE_FREE; + if (candidate->state == CANDIDATE_READY) + readySequence = candidate->sequence; + forceFrame = candidate->state == CANDIDATE_READY && + candidate->timingToken != 0; } } @@ -332,8 +401,12 @@ void CHardwareFrameProcessor::CandidateCompletionFunction( processor->SetFullDamage(); processor->m_transport->ForceFrame(); } - else if (forceFrame) - processor->m_transport->ForceFrame(); + else if (readySequence) + { + processor->m_transport->FrameProductReady(readySequence); + if (forceFrame) + processor->m_transport->ForceFrame(); + } processor->SignalCandidateState(); } @@ -341,15 +414,16 @@ void CHardwareFrameProcessor::CompletionFunction( CD3D12CommandSlot * slot, bool result, void * param1, void * param2) { auto processor = static_cast(param1); - auto fbRes = static_cast(param2); - const unsigned candidateIndex = fbRes->GetCandidateIndex(); + const FrameCopyBatch batch = + *static_cast(param2); + const unsigned candidateIndex = batch.candidateIndex; if (!result) { - processor->m_transport->FailFrameBuffer(fbRes->GetToken()); + processor->m_transport->FailFrameBatch(batch.prepared.token); processor->SetFullDamage(); processor->m_transport->ForceFrame(); - processor->ReleaseCandidate(candidateIndex); + processor->RetainCandidate(candidateIndex); return; } @@ -371,79 +445,114 @@ void CHardwareFrameProcessor::CompletionFunction( prepareTimingValid = candidate.prepareTimingValid; } - const uint64_t publishStart = fbRes->GetCopyStart(); + uint64_t publishStart = 0; + for (unsigned i = 0; i < batch.prepared.count; ++i) + if ((batch.accepted & (1U << i)) && batch.resources[i]) + { + publishStart = batch.resources[i]->GetCopyStart(); + break; + } uint64_t gpuCopyStart = 0; uint64_t gpuCopyEnd = 0; - uint64_t indirectCopyTime = 0; - if (processor->m_dx12->IsIndirectCopy()) - { - const uint64_t indirectCopyStart = CFrameScheduler::Nanotime(); - processor->m_transport->WriteFrameBuffer( - fbRes->GetToken(), fbRes->GetMap(), 0, - fbRes->GetFrameSize(), false); - indirectCopyTime = CFrameScheduler::Nanotime() - indirectCopyStart; - } - const bool gpuTimingValid = slot->GetGPUTimes(gpuCopyStart, gpuCopyEnd); - const uint64_t copyReady = CFrameScheduler::Nanotime(); - - const uint64_t postProcessStart = fbRes->GetPostProcessStart(); - uint64_t postProcessTime = prepareCopyStart - postProcessStart; - uint64_t prepareCopyTime = prepareReady - prepareCopyStart; - if (prepareTimingValid && prepareGPUStart >= postProcessStart && - prepareGPUEnd >= prepareGPUStart && prepareGPUEnd <= prepareReady) + bool timingRecorded = false; + for (unsigned i = 0; i < batch.prepared.count; ++i) { - postProcessTime = prepareGPUStart - postProcessStart; - prepareCopyTime = prepareGPUEnd - prepareGPUStart; + if (!(batch.accepted & (1U << i))) + continue; + CFrameBufferResource * fbRes = batch.resources[i]; + if (!fbRes) + continue; + + uint64_t stagedCopyTime = 0; + if (fbRes->GetMap()) + { + const uint64_t stagedCopyStart = CFrameScheduler::Nanotime(); + if (fbRes->IsFullCopy()) + processor->m_transport->WriteFrameTarget(batch.prepared.token, + i, fbRes->GetMap(), 0, fbRes->GetFrameSize(), false); + else + { + const unsigned pitch = fbRes->GetCopyPitch(); + const unsigned bytesPerPixel = fbRes->GetCopyBytesPerPixel(); + const RECT * dirtyRects = fbRes->GetCopyDirtyRects(); + const unsigned count = fbRes->GetCopyDirtyRectCount(); + for (const RECT * rect = dirtyRects; + rect < dirtyRects + count; ++rect) + { + const size_t rowOffset = + (size_t)rect->top * pitch + + (size_t)rect->left * bytesPerPixel; + const size_t rowBytes = + (size_t)(rect->right - rect->left) * bytesPerPixel; + processor->m_transport->WriteFrameTargetRows( + batch.prepared.token, i, fbRes->GetMap(), rowOffset, + rowBytes, pitch, (unsigned)(rect->bottom - rect->top)); + } + } + stagedCopyTime = CFrameScheduler::Nanotime() - stagedCopyStart; + } + const uint64_t copyReady = CFrameScheduler::Nanotime(); + const uint64_t postProcessStart = fbRes->GetPostProcessStart(); + uint64_t postProcessTime = prepareCopyStart - postProcessStart; + uint64_t prepareCopyTime = prepareReady - prepareCopyStart; + if (prepareTimingValid && prepareGPUStart >= postProcessStart && + prepareGPUEnd >= prepareGPUStart && prepareGPUEnd <= prepareReady) + { + postProcessTime = prepareGPUStart - postProcessStart; + prepareCopyTime = prepareGPUEnd - prepareGPUStart; + } + + uint64_t publishCopyTime = copyReady - publishStart; + if (gpuTimingValid && gpuCopyStart >= publishStart && + gpuCopyEnd >= gpuCopyStart && gpuCopyEnd <= copyReady) + publishCopyTime = gpuCopyEnd - gpuCopyStart + stagedCopyTime; + + const uint64_t copyTime = prepareCopyTime + publishCopyTime; + + processor->m_transport->FinalizeFrameTarget( + batch.prepared.token, i); + const uint64_t publishedAt = CFrameScheduler::Nanotime(); + const uint64_t prepareElapsed = prepareReady >= postProcessStart ? + prepareReady - postProcessStart : 0; + const uint64_t prepareMeasured = postProcessTime + prepareCopyTime; + const uint64_t prepareReadyTime = prepareElapsed > prepareMeasured ? + prepareElapsed - prepareMeasured : 0; + const uint64_t publishElapsed = publishedAt >= publishStart ? + publishedAt - publishStart : 0; + const uint64_t publishReadyTime = publishElapsed > publishCopyTime ? + publishElapsed - publishCopyTime : 0; + const uint64_t readyTime = prepareReadyTime + publishReadyTime; + const uint64_t holdTime = publishStart >= prepareReady ? + publishStart - prepareReady : 0; + + processor->m_transport->SetFrameTargetTiming(batch.prepared.token, i, + fbRes->GetCaptureTime(), postProcessTime, copyTime, readyTime, + holdTime, publishedAt); + processor->m_transport->TryRecordFrameTiming( + batch.prepared.token, i, publishedAt - publishStart); + + const uint64_t timingToken = fbRes->GetTimingToken(); + if (!timingRecorded && timingToken && timingStart && + prepareReady >= timingStart && copyReady >= publishStart) + { + const uint64_t totalTime = + (prepareReady - timingStart) + (copyReady - publishStart); + processor->m_postProcessors[candidateIndex].RecordTiming( + fbRes->GetTimingEffectIndex(), timingToken, + fbRes->IsFullCopy(), totalTime); + timingRecorded = true; + } + + processor->m_transport->CompleteFrameTarget( + batch.prepared.token, i, true); } - - uint64_t publishCopyTime = copyReady - publishStart; - if (gpuTimingValid && gpuCopyStart >= publishStart && - gpuCopyEnd >= gpuCopyStart && gpuCopyEnd <= copyReady) - publishCopyTime = gpuCopyEnd - gpuCopyStart + indirectCopyTime; - - const uint64_t copyTime = prepareCopyTime + publishCopyTime; - - processor->m_transport->FinalizeFrameBuffer(fbRes->GetToken()); - const uint64_t publishedAt = CFrameScheduler::Nanotime(); - const uint64_t prepareElapsed = prepareReady >= postProcessStart ? - prepareReady - postProcessStart : 0; - const uint64_t prepareMeasured = postProcessTime + prepareCopyTime; - const uint64_t prepareReadyTime = prepareElapsed > prepareMeasured ? - prepareElapsed - prepareMeasured : 0; - const uint64_t publishElapsed = publishedAt >= publishStart ? - publishedAt - publishStart : 0; - const uint64_t publishReadyTime = publishElapsed > publishCopyTime ? - publishElapsed - publishCopyTime : 0; - const uint64_t readyTime = prepareReadyTime + publishReadyTime; - const uint64_t holdTime = publishStart >= prepareReady ? - publishStart - prepareReady : 0; - - processor->m_transport->SetFrameTiming(fbRes->GetToken(), - fbRes->GetCaptureTime(), postProcessTime, copyTime, readyTime, holdTime, - fbRes->GetSchedule(), publishedAt); - processor->m_transport->TryRecordFrameTiming( - fbRes->GetToken(), publishedAt - publishStart); - - const uint64_t timingToken = fbRes->GetTimingToken(); - if (timingToken && timingStart && prepareReady >= timingStart && - copyReady >= publishStart) - { - const uint64_t totalTime = - (prepareReady - timingStart) + (copyReady - publishStart); - processor->m_postProcessors[candidateIndex].RecordTiming( - fbRes->GetTimingEffectIndex(), timingToken, - fbRes->IsFullCopy(), totalTime); - } - - processor->m_transport->CompleteFrameBuffer(fbRes->GetToken(), true); - processor->ReleaseCandidate(candidateIndex); + processor->RetainCandidate(candidateIndex); } bool CHardwareFrameProcessor::Publish( - const CFrameScheduler::Schedule& schedule, bool periodic, - uint64_t publishStart) + const FramePlan& plan, uint64_t publishStart) { CPublishPending publishPending( m_copySubmitLock, &m_publishPending, m_copySubmitEvent.Get()); @@ -451,6 +560,7 @@ bool CHardwareFrameProcessor::Publish( int selectedCandidate = -1; uint64_t newestSequence = 0; + CandidateState selectedState = CANDIDATE_FREE; { CSRWExclusiveLock lock(m_candidateLock); @@ -463,9 +573,23 @@ bool CHardwareFrameProcessor::Publish( newestSequence = m_candidates[i].sequence; } + if (selectedCandidate < 0 && m_transport->NeedsFrame()) + for (unsigned i = 0; i < ARRAYSIZE(m_candidates); ++i) + if (m_candidates[i].state == CANDIDATE_RETAINED && + (selectedCandidate < 0 || + m_candidates[i].sequence > newestSequence)) + { + selectedCandidate = static_cast(i); + newestSequence = m_candidates[i].sequence; + } + if (selectedCandidate >= 0) - m_candidates[static_cast(selectedCandidate)].state = - CANDIDATE_PUBLISHING; + { + FrameCandidate& selected = + m_candidates[static_cast(selectedCandidate)]; + selectedState = selected.state; + selected.state = CANDIDATE_PUBLISHING; + } } if (selectedCandidate < 0) @@ -473,12 +597,13 @@ bool CHardwareFrameProcessor::Publish( const unsigned candidateIndex = static_cast(selectedCandidate); - const auto restoreCandidate = [this, candidateIndex]() + const auto restoreCandidate = + [this, candidateIndex, selectedState]() { { CSRWExclusiveLock lock(m_candidateLock); if (m_candidates[candidateIndex].state == CANDIDATE_PUBLISHING) - m_candidates[candidateIndex].state = CANDIDATE_READY; + m_candidates[candidateIndex].state = selectedState; } SignalCandidateState(); }; @@ -500,30 +625,20 @@ bool CHardwareFrameProcessor::Publish( CPostProcessor& postProcessor = m_postProcessors[candidateIndex]; const uint64_t candidateSequence = candidate.sequence; - auto buffer = m_transport->PrepareFrameBuffer( - candidate.pitch, candidate.srcFormat, candidate.dstFormat, - candidate.dirtyRects, candidate.nbDirtyRects, schedule); - if (!buffer.mem) + PreparedFrameBatch prepared = {}; + if (!m_transport->PrepareFrameBatch(plan, candidate.sequence, + candidate.pitch, candidate.frameSize, candidate.srcFormat, + candidate.dstFormat, candidate.dirtyRects, + candidate.nbDirtyRects, true, prepared)) { restoreCandidate(); return false; } - CFrameBufferResource * fbRes = - m_frameBuffers.Get(buffer, candidate.frameSize); - if (!fbRes) - { - m_transport->AbortFrameBuffer(buffer.token); - restoreCandidate(); - DEBUG_ERROR("Failed to get a CFrameBufferResource from the pool"); - SetFullDamage(); - return false; - } - CD3D12CommandSlot * copySlot = m_dx12->GetCopySlot(candidateIndex); if (!copySlot) { - m_transport->AbortFrameBuffer(buffer.token); + m_transport->AbortFrameBatch(prepared.token); restoreCandidate(); DEBUG_ERROR("Failed to get a copy CommandSlot for publication"); SetFullDamage(); @@ -534,42 +649,72 @@ bool CHardwareFrameProcessor::Publish( unsigned nbPreviousDirtyRects = 0; GetPreviousDamage(previousDirtyRects, &nbPreviousDirtyRects); - RECT copyDirtyRects[LG_MAX_DIRTY_RECTS * 2] = {}; - unsigned nbCopyDirtyRects = 0; - const bool fullCopy = CFrameProcessorUtil::BuildCopyDamage( - postProcessor, buffer.fullCopy, - previousDirtyRects, nbPreviousDirtyRects, - candidate.dirtyRects, candidate.nbDirtyRects, - candidate.dstFormat.width, candidate.dstFormat.height, - copyDirtyRects, &nbCopyDirtyRects); + FrameCopyBatch& batch = m_publishBatches[candidateIndex]; + batch = {}; + batch.prepared = prepared; + batch.candidateIndex = candidateIndex; + const unsigned bytesPerPixel = + candidate.dstFormat.format == FRAME_TYPE_RGBA16F ? 8 : 4; + bool resourcesReady = true; + for (unsigned i = 0; i < prepared.count; ++i) + { + CFrameBufferResource * fbRes = + m_frameBuffers.Get(prepared.targets[i], candidate.frameSize); + batch.resources[i] = fbRes; + if (!fbRes) + { + resourcesReady = false; + break; + } - fbRes->SetTiming( - candidate.captureTime, candidate.postProcessStart, publishStart); - fbRes->SetCandidateIndex(candidateIndex); - fbRes->SetPostProcessSample( - candidate.timingEffectIndex, candidate.timingToken, fullCopy); - copySlot->SetCompletionCallback(&CompletionFunction, this, fbRes); - - copySlot->BeginTiming(); - postProcessor.CopyFromCandidate( - copySlot->GetGfxList(), fbRes->Get().Get(), candidate.resource.Get(), - copyDirtyRects, nbCopyDirtyRects, fullCopy); - copySlot->EndTiming(); - - bool deliveredToOwner; - if (!m_transport->PublishFrameBuffer( - buffer.token, schedule, deliveredToOwner)) + RECT copyDirtyRects[LG_MAX_DIRTY_RECTS * 2] = {}; + unsigned nbCopyDirtyRects = 0; + const bool fullCopy = CFrameProcessorUtil::BuildCopyDamage( + postProcessor, prepared.targets[i].fullCopy, + previousDirtyRects, nbPreviousDirtyRects, + candidate.dirtyRects, candidate.nbDirtyRects, + candidate.dstFormat.width, candidate.dstFormat.height, + copyDirtyRects, &nbCopyDirtyRects); + fbRes->SetTiming( + candidate.captureTime, candidate.postProcessStart, publishStart); + fbRes->SetCandidateIndex(candidateIndex); + fbRes->SetPostProcessSample( + candidate.timingEffectIndex, candidate.timingToken, fullCopy); + fbRes->SetCopyDamage(copyDirtyRects, nbCopyDirtyRects, + fullCopy, candidate.pitch, bytesPerPixel); + } + if (!resourcesReady) + { + copySlot->Cancel(); + m_transport->FailFrameBatch(prepared.token); + restoreCandidate(); + DEBUG_ERROR("Failed to get a CFrameBufferResource from the pool"); + SetFullDamage(); + return false; + } + + const uint32_t accepted = + m_transport->PublishFrameBatch(prepared.token); + if (!accepted) { copySlot->Cancel(); - m_transport->AbortFrameBuffer(buffer.token); restoreCandidate(); return false; } - CFrameScheduler::Schedule frameSchedule = schedule; - if (!deliveredToOwner || - !m_transport->TryFrameSubmitted(buffer.token, schedule)) - frameSchedule.phaseEligible = false; - fbRes->SetSchedule(frameSchedule); + batch.accepted = accepted; + + copySlot->SetCompletionCallback(&CompletionFunction, this, &batch); + copySlot->BeginTiming(); + for (unsigned i = 0; i < prepared.count; ++i) + if (accepted & (1U << i)) + { + CFrameBufferResource * fbRes = batch.resources[i]; + postProcessor.CopyFromCandidate(copySlot->GetGfxList(), + fbRes->Get().Get(), candidate.resource.Get(), + fbRes->GetCopyDirtyRects(), fbRes->GetCopyDirtyRectCount(), + fbRes->IsFullCopy()); + } + copySlot->EndTiming(); { CSRWExclusiveLock lock(m_damageLock); @@ -595,23 +740,25 @@ bool CHardwareFrameProcessor::Publish( if (!submitted) { SetFullDamage(); + const bool submittedWork = copySlot->HasSubmittedWork(); bool callbackPending; { CSRWSharedLock lock(m_candidateLock); callbackPending = candidate.state == CANDIDATE_PUBLISHING; } - if (callbackPending && !copySlot->HasSubmittedWork()) + if (submittedWork || !callbackPending) + m_transport->CommitFrameBatch(prepared.token); + else { - m_transport->FailFrameBuffer(buffer.token); - ReleaseCandidate(candidateIndex); + m_transport->FailFrameBatch(prepared.token); + RetainCandidate(candidateIndex); } m_transport->ForceFrame(); SignalCandidateState(); return false; } - m_transport->CommitFrameBuffer( - buffer.token, schedule, periodic, deliveredToOwner); + m_transport->CommitFrameBatch(prepared.token); unsigned superseded = 0; { @@ -780,6 +927,18 @@ bool CHardwareFrameProcessor::Submit(const FrameSubmission& submission) dstFormat.width, dstFormat.height); const size_t frameSize = postProcessor.GetOutputSize(); + const unsigned pitch = postProcessor.GetOutputPitch(); + if (!pitch || !frameSize || frameSize > m_transport->GetMaxFrameSize()) + { + copySlot->Cancel(); + if (computeSlot) + m_dx12->WaitForIdle(); + ReleaseCandidate(candidateIndex); + DEBUG_ERROR("Processed frame does not fit in primary frame memory"); + SetFullDamage(); + return false; + } + if (!EnsureCandidateResource(candidateIndex, frameSize)) { copySlot->Cancel(); @@ -793,7 +952,7 @@ bool CHardwareFrameProcessor::Submit(const FrameSubmission& submission) candidate.srcFormat = submission.sourceFormat; candidate.dstFormat = dstFormat; candidate.nbDirtyRects = nbDirtyRects; - candidate.pitch = postProcessor.GetOutputPitch(); + candidate.pitch = pitch; candidate.frameSize = frameSize; candidate.captureTime = submission.captureTime; candidate.postProcessStart = submission.postProcessStart; diff --git a/idd/LGIdd/capture/CHardwareFrameProcessor.h b/idd/LGIdd/capture/CHardwareFrameProcessor.h index b21708a4..c9ac6f32 100644 --- a/idd/LGIdd/capture/CHardwareFrameProcessor.h +++ b/idd/LGIdd/capture/CHardwareFrameProcessor.h @@ -22,7 +22,7 @@ #include "capture/CFrameProcessor.h" -class CHardwareFrameProcessor final : public CFrameProcessor +class CHardwareFrameProcessor : public CFrameProcessor { private: enum CandidateState @@ -31,6 +31,7 @@ private: CANDIDATE_PREPARING, CANDIDATE_READY, CANDIDATE_PUBLISHING, + CANDIDATE_RETAINED, }; struct FrameCandidate @@ -67,10 +68,11 @@ private: FrameCandidate m_candidates[CAPTURE_PIPELINE_SLOTS]; CandidateDamageTail m_candidateDamageTail[CAPTURE_PIPELINE_SLOTS]; + FrameCopyBatch m_publishBatches[CAPTURE_PIPELINE_SLOTS]; mutable CSRWLock m_candidateLock; CSRWLock m_copySubmitLock; - uint64_t m_candidateSequence = 0; bool m_publishPending = false; + bool m_useCadence = true; Wrappers::Event m_candidateAvailableEvent; Wrappers::Event m_copySubmitEvent; @@ -80,6 +82,7 @@ private: CD3D12CommandSlot * slot, bool result, void * param1, void * param2); int AcquireCandidate(bool exclusiveSample, bool allowSupersede); void ReleaseCandidate(unsigned candidateIndex); + void RetainCandidate(unsigned candidateIndex); bool EnsureCandidateResource(unsigned candidateIndex, size_t frameSize); void ResetCandidates(); void SignalCandidateState(); @@ -92,14 +95,14 @@ public: CHardwareFrameProcessor(IFrameTransport * transport, std::shared_ptr dx12, CPostProcessor postProcessors[CAPTURE_PIPELINE_SLOTS], - CSRWLock * pipelineLock, HANDLE terminateEvent); + CSRWLock * pipelineLock, HANDLE terminateEvent, + bool useCadence = true); bool IsValid() const override; bool Submit(const FrameSubmission& submission) override; bool HasReadyFrame() const override; - bool Publish(const CFrameScheduler::Schedule& schedule, - bool periodic, uint64_t publishStart) override; - bool UsesCadence() const override { return true; } + bool Publish(const FramePlan& plan, uint64_t publishStart) override; + bool UsesCadence() const override { return m_useCadence; } void Reset() override; void ResetPipeline() override; }; diff --git a/idd/LGIdd/capture/CSoftwareFrameProcessor.cpp b/idd/LGIdd/capture/CSoftwareFrameProcessor.cpp index 5bb683a4..2871561d 100644 --- a/idd/LGIdd/capture/CSoftwareFrameProcessor.cpp +++ b/idd/LGIdd/capture/CSoftwareFrameProcessor.cpp @@ -20,13 +20,18 @@ #include "capture/CSoftwareFrameProcessor.h" #include "capture/CFrameProcessorUtil.h" -#include "capture/FramePipeline.h" -#include "CSRWLock.h" #include "transport/IFrameTransport.h" +#include "CSRWLock.h" #include "CDebug.h" +#include #include +using namespace Microsoft::WRL; + +static_assert(CAPTURE_PIPELINE_SLOTS == 2, + "Software frame products assume two pipeline slots"); + CSoftwareFrameProcessor::CSoftwareFrameProcessor( IFrameTransport * transport, std::shared_ptr dx12, CPostProcessor postProcessors[CAPTURE_PIPELINE_SLOTS], @@ -34,268 +39,949 @@ CSoftwareFrameProcessor::CSoftwareFrameProcessor( CFrameProcessor(transport, std::move(dx12), postProcessors, pipelineLock, terminateEvent) { + m_productAvailableEvent.Attach( + CreateEvent(nullptr, FALSE, FALSE, nullptr)); +} + +void CSoftwareFrameProcessor::SignalProductState(bool available) +{ + SetEvent(m_readyEvent.Get()); + if (available) + SetEvent(m_productAvailableEvent.Get()); +} + +int CSoftwareFrameProcessor::AcquireProduct() +{ + CSRWExclusiveLock lock(m_productLock); + for (unsigned i = 0; i < ARRAYSIZE(m_products); ++i) + if (m_products[i].state == PRODUCT_FREE) + { + Product& product = m_products[i]; + product.state = PRODUCT_PREPARING; + product.restoreState = PRODUCT_FREE; + product.completedState = PRODUCT_FREE; + product.sequence = m_transport->NextContentSerial(); + product.sourceReady = false; + product.batchCommitted = false; + product.productNotified = false; + product.executeActive = false; + product.completionDone = false; + product.completionSucceeded = false; + product.batch = {}; + return static_cast(i); + } + return -1; +} + +int CSoftwareFrameProcessor::WaitForProduct() +{ + for (;;) + { + const int selected = AcquireProduct(); + if (selected >= 0) + return selected; + + HANDLE handles[] = + { + m_terminateEvent, + m_productAvailableEvent.Get(), + }; + const DWORD result = WaitForMultipleObjects( + ARRAYSIZE(handles), handles, FALSE, INFINITE); + if (result == WAIT_OBJECT_0) + return -1; + if (result != WAIT_OBJECT_0 + 1) + { + DEBUG_ERROR_HR(HRESULT_FROM_WIN32(GetLastError()), + "Failed while waiting for a software frame product"); + return -2; + } + } +} + +bool CSoftwareFrameProcessor::IsValid() const +{ + return CFrameProcessor::IsValid() && m_productAvailableEvent.Get(); +} + +void CSoftwareFrameProcessor::RestoreProduct( + unsigned productIndex, ProductState state) +{ + if (productIndex >= ARRAYSIZE(m_products)) + return; + + { + CSRWExclusiveLock lock(m_productLock); + Product& product = m_products[productIndex]; + product.state = state; + product.restoreState = PRODUCT_FREE; + product.completedState = PRODUCT_FREE; + product.executeActive = false; + product.batch = {}; + } + SignalProductState(state == PRODUCT_FREE); +} + +void CSoftwareFrameProcessor::MarkSourceReady( + unsigned productIndex, uint64_t sequence) +{ + bool notify = false; + { + CSRWExclusiveLock lock(m_productLock); + Product& product = m_products[productIndex]; + if (product.sequence != sequence) + return; + const bool retained = product.state == PRODUCT_RETAINED || + (product.state == PRODUCT_COMPLETING && + product.completionSucceeded && + product.completedState == PRODUCT_RETAINED); + if (!retained) + return; + product.sourceReady = true; + if (product.batchCommitted && !product.productNotified) + { + product.productNotified = true; + notify = true; + } + } + if (notify) + m_transport->FrameProductReady(sequence); +} + +void CSoftwareFrameProcessor::MarkBatchCommitted( + unsigned productIndex, uint64_t sequence) +{ + bool notify = false; + { + CSRWExclusiveLock lock(m_productLock); + Product& product = m_products[productIndex]; + if (product.sequence != sequence) + return; + const bool retained = product.state == PRODUCT_RETAINED || + (product.state == PRODUCT_COMPLETING && + product.completionSucceeded && + product.completedState == PRODUCT_RETAINED); + product.batchCommitted = true; + if (retained && product.sourceReady && !product.productNotified) + { + product.productNotified = true; + notify = true; + } + } + if (notify) + m_transport->FrameProductReady(sequence); +} + +void CSoftwareFrameProcessor::FinishProduct( + unsigned productIndex, bool publishing, bool result) +{ + bool available; + { + CSRWExclusiveLock lock(m_productLock); + Product& product = m_products[productIndex]; + const ProductState expected = + publishing ? PRODUCT_PUBLISHING : PRODUCT_PREPARING; + if (product.state != expected) + return; + + bool newer = false; + for (unsigned i = 0; i < ARRAYSIZE(m_products); ++i) + { + if (i == productIndex) + continue; + + Product& current = m_products[i]; + const bool completing = current.state == PRODUCT_COMPLETING && + current.completionSucceeded && + current.completedState != PRODUCT_FREE; + const bool complete = current.state == PRODUCT_READY || + current.state == PRODUCT_PUBLISHING || + current.state == PRODUCT_RETAINED || completing; + if (complete && current.sequence > product.sequence) + newer = true; + else if (result && complete && + current.state != PRODUCT_PUBLISHING && + current.sequence < product.sequence) + { + if (current.state == PRODUCT_COMPLETING) + current.completedState = PRODUCT_FREE; + else + { + current.state = PRODUCT_FREE; + SetEvent(m_productAvailableEvent.Get()); + } + } + } + + ProductState completedState; + if (newer) + completedState = PRODUCT_FREE; + else if (!result) + completedState = publishing ? product.restoreState : PRODUCT_FREE; + else + completedState = PRODUCT_RETAINED; + product.completionDone = true; + product.completionSucceeded = result; + if (product.executeActive) + { + product.state = PRODUCT_COMPLETING; + product.completedState = completedState; + } + else + { + product.state = completedState; + product.completedState = PRODUCT_FREE; + } + product.restoreState = PRODUCT_FREE; + + available = m_products[productIndex].state == PRODUCT_FREE; + if (!available) + for (unsigned i = 0; i < ARRAYSIZE(m_products); ++i) + if (i != productIndex && + m_products[i].state == PRODUCT_FREE) + { + available = true; + break; + } + } + SignalProductState(available); +} + +bool CSoftwareFrameProcessor::EnsureProductResource( + unsigned productIndex, size_t frameSize) +{ + Product& product = m_products[productIndex]; + + D3D12_RESOURCE_DESC desc = {}; + desc.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER; + desc.Width = frameSize; + desc.Height = 1; + desc.DepthOrArraySize = 1; + desc.MipLevels = 1; + desc.Format = DXGI_FORMAT_UNKNOWN; + desc.SampleDesc.Count = 1; + desc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR; + + if (product.resource && + CFrameProcessorUtil::ResourceDescMatches( + product.resource->GetDesc(), desc, false)) + return true; + + product.resource.Reset(); + + D3D12_HEAP_PROPERTIES heapProps = {}; + heapProps.Type = D3D12_HEAP_TYPE_DEFAULT; + heapProps.CPUPageProperty = D3D12_CPU_PAGE_PROPERTY_UNKNOWN; + heapProps.MemoryPoolPreference = D3D12_MEMORY_POOL_UNKNOWN; + heapProps.CreationNodeMask = 1; + heapProps.VisibleNodeMask = 1; + + const HRESULT hr = m_dx12->GetDevice()->CreateCommittedResource( + &heapProps, D3D12_HEAP_FLAG_NONE, &desc, + D3D12_RESOURCE_STATE_COMMON, nullptr, + IID_PPV_ARGS(&product.resource)); + if (FAILED(hr)) + { + DEBUG_ERROR_HR(hr, "Failed to create retained software frame"); + return false; + } + + static const WCHAR * names[] = + { + L"Software Frame 0", + L"Software Frame 1", + }; + product.resource->SetName(names[productIndex]); + return true; +} + +bool CSoftwareFrameProcessor::BeginExecute(unsigned productIndex, + uint64_t sequence, ProductState state) +{ + CSRWExclusiveLock lock(m_productLock); + Product& product = m_products[productIndex]; + if (product.sequence != sequence || product.state != state) + return false; + product.executeActive = true; + product.completionDone = false; + product.completionSucceeded = false; + product.completedState = PRODUCT_FREE; + return true; +} + +void CSoftwareFrameProcessor::EndExecute(unsigned productIndex, + uint64_t sequence, bool& completed, bool& succeeded) +{ + bool available = false; + completed = false; + succeeded = false; + { + CSRWExclusiveLock lock(m_productLock); + Product& product = m_products[productIndex]; + if (product.sequence != sequence) + return; + + product.executeActive = false; + completed = product.completionDone; + succeeded = product.completionSucceeded; + if (product.state == PRODUCT_COMPLETING) + { + product.state = product.completedState; + product.completedState = PRODUCT_FREE; + available = product.state == PRODUCT_FREE; + } + } + SignalProductState(available); +} + +bool CSoftwareFrameProcessor::PrepareBatch(Product& product, + unsigned productIndex, const FramePlan& plan, uint64_t copyStart) +{ + FramePlan currentPlan = {}; + { + CSRWSharedLock lock(m_productLock); + if (product.state == PRODUCT_READY || + product.state == PRODUCT_RETAINED) + { + currentPlan = plan; + for (unsigned i = 0; i < currentPlan.count; ++i) + currentPlan.targets[i].commitSchedule.phaseEligible = false; + } + } + const FramePlan& batchPlan = currentPlan.count ? currentPlan : plan; + + PreparedFrameBatch prepared = {}; + if (!m_transport->PrepareFrameBatch(batchPlan, product.sequence, + product.pitch, product.frameSize, product.srcFormat, + product.dstFormat, product.dirtyRects, product.nbDirtyRects, + true, prepared)) + return false; + + FrameCopyBatch& batch = product.batch; + batch = {}; + batch.prepared = prepared; + batch.candidateIndex = productIndex; + + CPostProcessor& postProcessor = m_postProcessors[productIndex]; + const unsigned bytesPerPixel = + product.dstFormat.format == FRAME_TYPE_RGBA16F ? 8 : 4; + for (unsigned i = 0; i < prepared.count; ++i) + { + CFrameBufferResource * fbRes = + m_frameBuffers.Get(prepared.targets[i], product.frameSize); + batch.resources[i] = fbRes; + if (!fbRes) + { + m_transport->AbortFrameBatch(prepared.token); + batch = {}; + DEBUG_ERROR("Failed to get a software frame target resource"); + return false; + } + + RECT copyDirtyRects[LG_MAX_DIRTY_RECTS * 2] = {}; + unsigned nbCopyDirtyRects = 0; + const bool fullCopy = CFrameProcessorUtil::BuildCopyDamage( + postProcessor, prepared.targets[i].fullCopy, + product.previousDirtyRects, product.nbPreviousDirtyRects, + product.dirtyRects, product.nbDirtyRects, + product.dstFormat.width, product.dstFormat.height, + copyDirtyRects, &nbCopyDirtyRects); + fbRes->SetTiming( + product.captureTime, product.postProcessStart, copyStart); + fbRes->SetCandidateIndex(productIndex); + fbRes->SetPostProcessSample(product.timingEffectIndex, + product.timingToken, fullCopy); + fbRes->SetCopyDamage(copyDirtyRects, nbCopyDirtyRects, + fullCopy, product.pitch, bytesPerPixel); + fbRes->ResetCompletion(); + } + + batch.accepted = m_transport->PublishFrameBatch(prepared.token); + if (!batch.accepted) + { + batch = {}; + return false; + } + + return true; +} + +void CSoftwareFrameProcessor::CompleteBatch(CD3D12CommandSlot * slot, + Product& product, unsigned productIndex, bool publishing, bool result) +{ + const FrameCopyBatch batch = product.batch; + if (!batch.accepted) + return; + + for (unsigned i = 0; i < batch.prepared.count; ++i) + if ((batch.accepted & (1U << i)) && batch.resources[i]) + batch.resources[i]->MarkCompletion(); + + if (!result) + { + return; + } + + uint64_t gpuStart = 0; + uint64_t gpuEnd = 0; + const bool gpuTimingValid = slot->GetGPUTimes(gpuStart, gpuEnd); + bool timingRecorded = false; + for (unsigned i = 0; i < batch.prepared.count; ++i) + { + if (!(batch.accepted & (1U << i))) + continue; + CFrameBufferResource * fbRes = batch.resources[i]; + if (!fbRes) + continue; + + uint64_t stagedCopyTime = 0; + if (fbRes->GetMap()) + { + const uint64_t stagedCopyStart = CFrameScheduler::Nanotime(); + if (fbRes->IsFullCopy()) + m_transport->WriteFrameTarget(batch.prepared.token, + i, fbRes->GetMap(), 0, fbRes->GetFrameSize(), false); + else + { + const unsigned pitch = fbRes->GetCopyPitch(); + const unsigned bytesPerPixel = fbRes->GetCopyBytesPerPixel(); + const RECT * dirtyRects = fbRes->GetCopyDirtyRects(); + const unsigned count = fbRes->GetCopyDirtyRectCount(); + for (const RECT * rect = dirtyRects; + rect < dirtyRects + count; ++rect) + { + const size_t rowOffset = + (size_t)rect->top * pitch + + (size_t)rect->left * bytesPerPixel; + const size_t rowBytes = + (size_t)(rect->right - rect->left) * bytesPerPixel; + m_transport->WriteFrameTargetRows(batch.prepared.token, + i, fbRes->GetMap(), rowOffset, rowBytes, pitch, + (unsigned)(rect->bottom - rect->top)); + } + } + stagedCopyTime = CFrameScheduler::Nanotime() - stagedCopyStart; + } + + const uint64_t copyReady = CFrameScheduler::Nanotime(); + m_transport->FinalizeFrameTarget(batch.prepared.token, i); + const uint64_t publishedAt = CFrameScheduler::Nanotime(); + + uint64_t postProcessTime = 0; + uint64_t copyTime = 0; + uint64_t readyTime = 0; + uint64_t holdTime = 0; + if (!publishing) + { + const uint64_t copyStart = fbRes->GetCopyStart(); + postProcessTime = copyStart >= product.postProcessStart ? + copyStart - product.postProcessStart : 0; + copyTime = copyReady >= copyStart ? + copyReady - copyStart : 0; + if (gpuTimingValid && gpuStart >= product.postProcessStart && + gpuEnd >= gpuStart && gpuEnd <= copyReady) + { + postProcessTime = gpuStart - product.postProcessStart; + copyTime = gpuEnd - gpuStart + stagedCopyTime; + } + const uint64_t elapsed = publishedAt >= product.postProcessStart ? + publishedAt - product.postProcessStart : 0; + const uint64_t measured = postProcessTime + copyTime; + readyTime = elapsed > measured ? elapsed - measured : 0; + } + else + { + const uint64_t publishStart = fbRes->GetCopyStart(); + postProcessTime = product.prepareCopyStart >= + product.postProcessStart ? + product.prepareCopyStart - product.postProcessStart : 0; + uint64_t prepareCopyTime = product.prepareReady >= + product.prepareCopyStart ? + product.prepareReady - product.prepareCopyStart : 0; + if (product.prepareTimingValid && + product.prepareGPUStart >= product.postProcessStart && + product.prepareGPUEnd >= product.prepareGPUStart && + product.prepareGPUEnd <= product.prepareReady) + { + postProcessTime = + product.prepareGPUStart - product.postProcessStart; + prepareCopyTime = + product.prepareGPUEnd - product.prepareGPUStart; + } + + uint64_t publishCopyTime = copyReady >= publishStart ? + copyReady - publishStart : 0; + if (gpuTimingValid && gpuStart >= publishStart && + gpuEnd >= gpuStart && gpuEnd <= copyReady) + publishCopyTime = gpuEnd - gpuStart + stagedCopyTime; + copyTime = prepareCopyTime + publishCopyTime; + + const uint64_t prepareElapsed = product.prepareReady >= + product.postProcessStart ? + product.prepareReady - product.postProcessStart : 0; + const uint64_t prepareMeasured = postProcessTime + prepareCopyTime; + const uint64_t prepareWait = prepareElapsed > prepareMeasured ? + prepareElapsed - prepareMeasured : 0; + const uint64_t publishElapsed = publishedAt >= publishStart ? + publishedAt - publishStart : 0; + const uint64_t publishWait = publishElapsed > publishCopyTime ? + publishElapsed - publishCopyTime : 0; + readyTime = prepareWait + publishWait; + holdTime = publishStart >= product.prepareReady ? + publishStart - product.prepareReady : 0; + } + + m_transport->SetFrameTargetTiming(batch.prepared.token, i, + fbRes->GetCaptureTime(), postProcessTime, copyTime, readyTime, + holdTime, publishedAt); + m_transport->TryRecordFrameTiming( + batch.prepared.token, i, publishedAt - fbRes->GetCopyStart()); + + if (!timingRecorded && product.timingToken && product.timingStart) + { + uint64_t totalTime = 0; + if (!publishing && copyReady >= product.timingStart) + totalTime = copyReady - product.timingStart; + else if (publishing && + product.prepareReady >= product.timingStart && + copyReady >= fbRes->GetCopyStart()) + totalTime = (product.prepareReady - product.timingStart) + + (copyReady - fbRes->GetCopyStart()); + if (totalTime) + { + m_postProcessors[productIndex].RecordTiming( + fbRes->GetTimingEffectIndex(), fbRes->GetTimingToken(), + fbRes->IsFullCopy(), totalTime); + timingRecorded = true; + } + } + + m_transport->CompleteFrameTarget(batch.prepared.token, i, true); + } } void CSoftwareFrameProcessor::CompletionFunction( CD3D12CommandSlot * slot, bool result, void * param1, void * param2) { auto processor = static_cast(param1); - auto fbRes = static_cast(param2); - fbRes->MarkCompletion(); + auto product = static_cast(param2); + const unsigned productIndex = + static_cast(product - processor->m_products); + bool publishing; + uint64_t sequence; + { + CSRWExclusiveLock lock(processor->m_productLock); + if (product->state == PRODUCT_PUBLISHING) + publishing = true; + else if (product->state == PRODUCT_PREPARING) + publishing = false; + else + return; + sequence = product->sequence; + if (!publishing) + { + product->prepareReady = CFrameScheduler::Nanotime(); + product->prepareTimingValid = result && slot->GetGPUTimes( + product->prepareGPUStart, product->prepareGPUEnd); + } + } + + processor->CompleteBatch( + slot, *product, productIndex, publishing, result); if (!result) { - processor->m_transport->FailFrameBuffer(fbRes->GetToken()); + if (product->batch.accepted) + processor->m_transport->FailFrameBatch( + product->batch.prepared.token); processor->SetFullDamage(); processor->m_transport->ForceFrame(); - return; } + processor->FinishProduct(productIndex, publishing, result); + if (result && !publishing) + processor->MarkSourceReady(productIndex, sequence); +} - uint64_t indirectCopyTime = 0; - if (processor->m_dx12->IsIndirectCopy()) +bool CSoftwareFrameProcessor::HasReadyFrame() const +{ + uint64_t completeSequence = 0; + uint64_t preparingSequence = 0; + bool ready = false; { - const uint64_t indirectCopyStart = CFrameScheduler::Nanotime(); - if (fbRes->IsFullCopy()) - processor->m_transport->WriteFrameBuffer(fbRes->GetToken(), - fbRes->GetMap(), 0, fbRes->GetFrameSize(), false); - else + CSRWSharedLock lock(m_productLock); + for (const Product& product : m_products) { - const unsigned pitch = fbRes->GetCopyPitch(); - const unsigned bytesPerPixel = fbRes->GetCopyBytesPerPixel(); - const RECT * dirtyRects = fbRes->GetCopyDirtyRects(); - const unsigned count = fbRes->GetCopyDirtyRectCount(); - for (const RECT * rect = dirtyRects; rect < dirtyRects + count; ++rect) - { - const size_t rowOffset = - (size_t)rect->top * pitch + - (size_t)rect->left * bytesPerPixel; - const size_t rowBytes = - (size_t)(rect->right - rect->left) * bytesPerPixel; - processor->m_transport->WriteFrameBufferRows(fbRes->GetToken(), - fbRes->GetMap(), rowOffset, rowBytes, pitch, - (unsigned)(rect->bottom - rect->top)); - } + if ((product.state == PRODUCT_READY || + (product.state == PRODUCT_RETAINED && + product.productNotified)) && + product.sequence > completeSequence) + completeSequence = product.sequence; + else if ((product.state == PRODUCT_PREPARING || + product.state == PRODUCT_COMPLETING) && + product.sequence > preparingSequence) + preparingSequence = product.sequence; } - indirectCopyTime = CFrameScheduler::Nanotime() - indirectCopyStart; + if (completeSequence && preparingSequence <= completeSequence) + for (const Product& product : m_products) + if (product.state == PRODUCT_READY && + product.sequence == completeSequence) + { + ready = true; + break; + } } + if (!completeSequence || preparingSequence > completeSequence) + return false; + return ready || m_transport->NeedsFrame(); +} - uint64_t gpuStart = 0; - uint64_t gpuEnd = 0; - const uint64_t copyReady = CFrameScheduler::Nanotime(); - const bool gpuTimingValid = slot->GetGPUTimes(gpuStart, gpuEnd); +bool CSoftwareFrameProcessor::Publish( + const FramePlan& plan, uint64_t publishStart) +{ + CSRWSharedLock pipelineLock(*m_pipelineLock); - processor->m_transport->FinalizeFrameBuffer(fbRes->GetToken()); - const uint64_t publishedAt = CFrameScheduler::Nanotime(); - const uint64_t postProcessStart = fbRes->GetPostProcessStart(); - const uint64_t copyStart = fbRes->GetCopyStart(); - uint64_t postProcessTime = copyStart >= postProcessStart ? - copyStart - postProcessStart : 0; - uint64_t copyTime = copyReady >= copyStart ? - copyReady - copyStart : 0; - if (gpuTimingValid && gpuStart >= postProcessStart && - gpuEnd >= gpuStart && gpuEnd <= copyReady) + int selected = -1; + uint64_t newestSequence = 0; + ProductState restoreState = PRODUCT_FREE; { - postProcessTime = gpuStart - postProcessStart; - copyTime = gpuEnd - gpuStart + indirectCopyTime; + CSRWExclusiveLock lock(m_productLock); + for (unsigned i = 0; i < ARRAYSIZE(m_products); ++i) + if (m_products[i].state == PRODUCT_READY && + (selected < 0 || m_products[i].sequence > newestSequence)) + { + selected = static_cast(i); + newestSequence = m_products[i].sequence; + } + + if (selected < 0 && m_transport->NeedsFrame()) + for (unsigned i = 0; i < ARRAYSIZE(m_products); ++i) + if (m_products[i].state == PRODUCT_RETAINED && + m_products[i].productNotified && + (selected < 0 || m_products[i].sequence > newestSequence)) + { + selected = static_cast(i); + newestSequence = m_products[i].sequence; + } + + for (const Product& product : m_products) + if ((product.state == PRODUCT_PREPARING || + product.state == PRODUCT_COMPLETING) && + product.sequence > newestSequence) + { + selected = -1; + break; + } + + if (selected >= 0) + { + Product& product = m_products[static_cast(selected)]; + restoreState = product.state; + product.restoreState = product.state; + product.state = PRODUCT_PUBLISHING; + product.batch = {}; + } } - const uint64_t elapsed = publishedAt >= postProcessStart ? - publishedAt - postProcessStart : 0; - const uint64_t measured = postProcessTime + copyTime; - const uint64_t readyTime = elapsed > measured ? elapsed - measured : 0; + if (selected < 0) + return false; + const unsigned productIndex = static_cast(selected); + Product& product = m_products[productIndex]; + const uint64_t sequence = product.sequence; - processor->m_transport->SetFrameTiming(fbRes->GetToken(), - fbRes->GetCaptureTime(), postProcessTime, copyTime, readyTime, 0, - fbRes->GetSchedule(), publishedAt); - processor->m_transport->CompleteFrameBuffer(fbRes->GetToken(), true); + const auto restoreProduct = + [this, productIndex, sequence, restoreState]() + { + bool available = false; + { + CSRWExclusiveLock lock(m_productLock); + Product& selected = m_products[productIndex]; + if (selected.state == PRODUCT_PUBLISHING && + selected.sequence == sequence) + { + selected.state = restoreState; + selected.restoreState = PRODUCT_FREE; + selected.batch = {}; + available = restoreState == PRODUCT_FREE; + } + } + SignalProductState(available); + }; + + CD3D12CommandSlot * copySlot = m_dx12->GetCopySlot(productIndex); + if (!copySlot) + { + restoreProduct(); + return false; + } + + if (!PrepareBatch(product, productIndex, plan, publishStart)) + { + copySlot->Cancel(); + restoreProduct(); + return false; + } + + const FrameBatchToken token = product.batch.prepared.token; + copySlot->SetCompletionCallback( + &CompletionFunction, this, &product); + copySlot->BeginTiming(); + CPostProcessor& postProcessor = m_postProcessors[productIndex]; + for (unsigned i = 0; i < product.batch.prepared.count; ++i) + if (product.batch.accepted & (1U << i)) + { + CFrameBufferResource * fbRes = product.batch.resources[i]; + postProcessor.CopyFromCandidate(copySlot->GetGfxList(), + fbRes->Get().Get(), product.resource.Get(), + fbRes->GetCopyDirtyRects(), fbRes->GetCopyDirtyRectCount(), + fbRes->IsFullCopy()); + } + copySlot->EndTiming(); + + if (!BeginExecute(productIndex, sequence, PRODUCT_PUBLISHING)) + { + copySlot->Cancel(); + m_transport->FailFrameBatch(token); + return false; + } + const bool executed = copySlot->Execute(); + const bool submittedWork = !executed && copySlot->HasSubmittedWork(); + bool completionDone; + bool completionSucceeded; + EndExecute(productIndex, sequence, + completionDone, completionSucceeded); + if (!executed) + { + if (submittedWork || (completionDone && completionSucceeded)) + m_transport->CommitFrameBatch(token); + else if (!completionDone) + { + m_transport->FailFrameBatch(token); + restoreProduct(); + } + SetFullDamage(); + m_transport->ForceFrame(); + return false; + } + + if (completionDone && !completionSucceeded) + return false; + m_transport->CommitFrameBatch(token); + return true; } bool CSoftwareFrameProcessor::Submit(const FrameSubmission& submission) { - CSRWSharedLock pipelineLock(*m_pipelineLock); - CPostProcessor& postProcessor = m_postProcessors[0]; - const D12FrameFormat& dstFormat = postProcessor.GetOutputFormat(); + if (submission.noImageUpdate && !HasPendingDamage()) + return true; - // A copyable footprint describes a buffer layout, not the physical layout - // of a row-major texture. Use that explicit buffer layout so the pitch sent - // through KVMFR always matches the bytes written into transport memory. - const unsigned pitch = postProcessor.GetOutputPitch(); - const size_t frameSize = postProcessor.GetOutputSize(); - - if (!pitch || !frameSize || frameSize > m_transport->GetMaxFrameSize()) + const int selected = submission.noImageUpdate ? + WaitForProduct() : AcquireProduct(); + if (selected == -2) + return false; + if (selected < 0) { - DEBUG_ERROR("Software frame does not fit in transport memory"); + if (WaitForSingleObject(m_terminateEvent, 0) == WAIT_OBJECT_0) + return true; + if (!submission.noImageUpdate) + m_transport->FrameSuperseded(); + return submission.noImageUpdate; + } + const unsigned productIndex = static_cast(selected); + Product& product = m_products[productIndex]; + + CSRWSharedLock pipelineLock(*m_pipelineLock); + CPostProcessor& postProcessor = m_postProcessors[productIndex]; + const D12FrameFormat& dstFormat = postProcessor.GetOutputFormat(); + const unsigned pitch = postProcessor.GetOutputPitch(); + const size_t frameSize = postProcessor.GetOutputSize(); + if (!pitch || !frameSize || + frameSize > m_transport->GetMaxFrameSize()) + { + RestoreProduct(productIndex, PRODUCT_FREE); + DEBUG_ERROR("Software frame does not fit in primary frame memory"); SetFullDamage(); return false; } - if (submission.noImageUpdate && !HasPendingDamage()) - return true; + RECT currentDirtyRects[LG_MAX_DIRTY_RECTS] = {}; + unsigned nbDirtyRects = 0; + const bool hasDamage = + TakePendingDamage(currentDirtyRects, &nbDirtyRects); + CFrameProcessorUtil::ClipDirtyRects(currentDirtyRects, + &nbDirtyRects, dstFormat.width, dstFormat.height); - for (;;) + RECT previousDirtyRects[LG_MAX_DIRTY_RECTS] = {}; + unsigned nbPreviousDirtyRects = 0; + GetPreviousDamage(previousDirtyRects, &nbPreviousDirtyRects); + + if (!EnsureProductResource(productIndex, frameSize)) { - CFrameScheduler::Schedule commitSchedule = {}; - CFrameScheduler::Schedule deliverySchedule = {}; - PreparedFrameBuffer buffer = {}; - CD3D12CommandSlot * copySlot = nullptr; - RECT currentDirtyRects[LG_MAX_DIRTY_RECTS] = {}; - unsigned nbDirtyRects = 0; - bool hasDamage = false; + RestorePendingDamage(currentDirtyRects, nbDirtyRects, hasDamage); + RestoreProduct(productIndex, PRODUCT_FREE); + SetFullDamage(); + return false; + } - uint64_t ignoredTarget = 0; - bool ignoredPeriodic = false; - bool ignoredRepublish = false; - m_transport->GetPublishTarget(CFrameScheduler::Nanotime(), - ignoredTarget, commitSchedule, ignoredPeriodic, ignoredRepublish); - deliverySchedule = commitSchedule; - deliverySchedule.deliveryDeadlineSerial = 0; - deliverySchedule.phaseEligible = false; - - m_transport->ProcessDeliveries(); - if (!m_transport->FrameBufferAvailable( - deliverySchedule, submission.noImageUpdate)) - { - if (!submission.noImageUpdate) - { - m_transport->FrameSuperseded(); - return true; - } - - if (WaitForSingleObject(m_terminateEvent, 1) == WAIT_OBJECT_0) - return true; - continue; - } - - copySlot = m_dx12->GetCopySlot(); - if (!copySlot) - { - if (!submission.noImageUpdate) - { - m_transport->FrameSuperseded(); - return true; - } - - if (WaitForSingleObject(m_terminateEvent, 1) == WAIT_OBJECT_0) - return true; - continue; - } - - hasDamage = TakePendingDamage(currentDirtyRects, &nbDirtyRects); - CFrameProcessorUtil::ClipDirtyRects( - currentDirtyRects, &nbDirtyRects, - dstFormat.width, dstFormat.height); - buffer = m_transport->PrepareFrameBuffer( - pitch, submission.sourceFormat, dstFormat, - currentDirtyRects, nbDirtyRects, deliverySchedule, - submission.noImageUpdate); - if (!buffer.mem) - { - copySlot->Cancel(); - RestorePendingDamage( - currentDirtyRects, nbDirtyRects, hasDamage); - if (!submission.noImageUpdate) - { - m_transport->FrameSuperseded(); - return true; - } - - if (WaitForSingleObject(m_terminateEvent, 1) == WAIT_OBJECT_0) - return true; - continue; - } - - CFrameBufferResource * fbRes = - m_frameBuffers.Get(buffer, frameSize); - if (!fbRes) - { - copySlot->Cancel(); - m_transport->AbortFrameBuffer(buffer.token); - RestorePendingDamage( - currentDirtyRects, nbDirtyRects, hasDamage); - DEBUG_ERROR("Failed to get a framebuffer for software capture"); - SetFullDamage(); - return false; - } - - if (!submission.source->Signal() || - !submission.source->Sync(*copySlot)) - { - copySlot->Cancel(); - m_transport->AbortFrameBuffer(buffer.token); - RestorePendingDamage( - currentDirtyRects, nbDirtyRects, hasDamage); - SetFullDamage(); - return false; - } - - RECT previousDirtyRects[LG_MAX_DIRTY_RECTS] = {}; - unsigned nbPreviousDirtyRects = 0; - GetPreviousDamage(previousDirtyRects, &nbPreviousDirtyRects); - - RECT copyDirtyRects[LG_MAX_DIRTY_RECTS * 2] = {}; - unsigned nbCopyDirtyRects = 0; - const bool fullCopy = CFrameProcessorUtil::BuildCopyDamage( - postProcessor, buffer.fullCopy, - previousDirtyRects, nbPreviousDirtyRects, - currentDirtyRects, nbDirtyRects, - dstFormat.width, dstFormat.height, - copyDirtyRects, &nbCopyDirtyRects); - - const unsigned bytesPerPixel = - dstFormat.format == FRAME_TYPE_RGBA16F ? 8 : 4; - const uint64_t copyStart = CFrameScheduler::Nanotime(); - fbRes->SetTiming(submission.captureTime, - submission.postProcessStart, copyStart); - fbRes->SetSchedule(deliverySchedule); - fbRes->SetCopyDamage(copyDirtyRects, nbCopyDirtyRects, - fullCopy, pitch, bytesPerPixel); - fbRes->ResetCompletion(); - copySlot->SetCompletionCallback( - &CompletionFunction, this, fbRes); - copySlot->BeginTiming(); - postProcessor.CopyToFrameBuffer(copySlot->GetGfxList(), - fbRes->Get().Get(), submission.source->GetRes().Get(), - copyDirtyRects, nbCopyDirtyRects, fullCopy); - copySlot->EndTiming(); - - bool deliveredToOwner; - if (!m_transport->PublishFrameBuffer( - buffer.token, deliverySchedule, deliveredToOwner)) - { - copySlot->Cancel(); - m_transport->AbortFrameBuffer(buffer.token); - RestorePendingDamage( - currentDirtyRects, nbDirtyRects, hasDamage); - if (!submission.noImageUpdate) - { - m_transport->FrameSuperseded(); - return true; - } - - if (WaitForSingleObject(m_terminateEvent, 1) == WAIT_OBJECT_0) - return true; - continue; - } - - CommitDamage(currentDirtyRects, nbDirtyRects); - if (!copySlot->Execute()) - { - const bool submittedWork = copySlot->HasSubmittedWork(); - const bool completionHandled = fbRes->CompletionHandled(); - if (!submittedWork && !completionHandled) - m_transport->FailFrameBuffer(buffer.token); - RestorePendingDamage( - currentDirtyRects, nbDirtyRects, hasDamage); - if (!submittedWork && !completionHandled) - { - SetFullDamage(); - m_transport->ForceFrame(); - } - return false; - } - - m_transport->CommitFrameBuffer( - buffer.token, commitSchedule, false, deliveredToOwner); + CD3D12CommandSlot * copySlot = m_dx12->GetCopySlot(productIndex); + if (!copySlot) + { + RestorePendingDamage(currentDirtyRects, nbDirtyRects, hasDamage); + RestoreProduct(productIndex, PRODUCT_FREE); + if (!submission.noImageUpdate) + m_transport->FrameSuperseded(); return true; } + + if (!submission.source->Signal() || !submission.source->Sync(*copySlot)) + { + copySlot->Cancel(); + RestorePendingDamage(currentDirtyRects, nbDirtyRects, hasDamage); + RestoreProduct(productIndex, PRODUCT_FREE); + SetFullDamage(); + return false; + } + + product.srcFormat = submission.sourceFormat; + product.dstFormat = dstFormat; + product.nbDirtyRects = nbDirtyRects; + product.nbPreviousDirtyRects = nbPreviousDirtyRects; + product.pitch = pitch; + product.frameSize = frameSize; + product.captureTime = submission.captureTime; + product.postProcessStart = submission.postProcessStart; + product.prepareCopyStart = CFrameScheduler::Nanotime(); + product.prepareReady = 0; + product.prepareGPUStart = 0; + product.prepareGPUEnd = 0; + product.timingStart = submission.timingToken ? + CFrameScheduler::Nanotime() : 0; + product.timingEffectIndex = submission.timingEffectIndex; + product.timingToken = submission.timingToken; + product.prepareTimingValid = false; + if (nbDirtyRects) + memcpy(product.dirtyRects, currentDirtyRects, + nbDirtyRects * sizeof(*product.dirtyRects)); + if (nbPreviousDirtyRects) + memcpy(product.previousDirtyRects, previousDirtyRects, + nbPreviousDirtyRects * sizeof(*product.previousDirtyRects)); + + FramePlan plan = {}; + const uint64_t now = CFrameScheduler::Nanotime(); + if (m_transport->GetImmediateFramePlan(now, plan)) + PrepareBatch(product, productIndex, plan, + product.prepareCopyStart); + + const FrameBatchToken token = product.batch.prepared.token; + const bool hasBatch = product.batch.accepted != 0; + const uint64_t sequence = product.sequence; + bool productValid; + { + CSRWExclusiveLock lock(m_productLock); + productValid = product.state == PRODUCT_PREPARING && + product.sequence == sequence; + if (productValid) + product.batchCommitted = !hasBatch; + } + if (!productValid) + { + copySlot->Cancel(); + if (hasBatch) + m_transport->FailFrameBatch(token); + RestorePendingDamage(currentDirtyRects, nbDirtyRects, hasDamage); + return false; + } + copySlot->SetCompletionCallback( + &CompletionFunction, this, &product); + copySlot->BeginTiming(); + if (hasBatch) + for (unsigned i = 0; i < product.batch.prepared.count; ++i) + if (product.batch.accepted & (1U << i)) + { + CFrameBufferResource * fbRes = product.batch.resources[i]; + postProcessor.CopyToFrameBuffer(copySlot->GetGfxList(), + fbRes->Get().Get(), submission.source->GetRes().Get(), + fbRes->GetCopyDirtyRects(), fbRes->GetCopyDirtyRectCount(), + fbRes->IsFullCopy()); + } + postProcessor.CopyToCandidate(copySlot->GetGfxList(), + product.resource.Get(), submission.source->GetRes().Get()); + copySlot->EndTiming(); + + if (!BeginExecute(productIndex, sequence, PRODUCT_PREPARING)) + { + copySlot->Cancel(); + if (hasBatch) + m_transport->FailFrameBatch(token); + RestorePendingDamage(currentDirtyRects, nbDirtyRects, hasDamage); + return false; + } + const bool executed = copySlot->Execute(); + const bool submittedWork = !executed && copySlot->HasSubmittedWork(); + bool completionDone; + bool completionSucceeded; + EndExecute(productIndex, sequence, + completionDone, completionSucceeded); + if (!executed) + { + if (submittedWork || (completionDone && completionSucceeded)) + { + if (hasBatch) + m_transport->CommitFrameBatch(token); + MarkBatchCommitted(productIndex, sequence); + CommitDamage(currentDirtyRects, nbDirtyRects); + } + else if (!completionDone) + { + if (hasBatch) + m_transport->FailFrameBatch(token); + RestorePendingDamage(currentDirtyRects, nbDirtyRects, hasDamage); + { + CSRWExclusiveLock lock(m_productLock); + if (product.state == PRODUCT_PREPARING && + product.sequence == sequence) + { + product.state = PRODUCT_FREE; + product.restoreState = PRODUCT_FREE; + product.batch = {}; + } + } + SignalProductState(true); + } + SetFullDamage(); + m_transport->ForceFrame(); + return false; + } + + if (completionDone && !completionSucceeded) + { + RestorePendingDamage(currentDirtyRects, nbDirtyRects, hasDamage); + return false; + } + if (hasBatch) + m_transport->CommitFrameBatch(token); + MarkBatchCommitted(productIndex, sequence); + CommitDamage(currentDirtyRects, nbDirtyRects); + return true; +} + +void CSoftwareFrameProcessor::ResetProducts() +{ + { + CSRWExclusiveLock lock(m_productLock); + for (Product& product : m_products) + product = {}; + } + SignalProductState(true); +} + +void CSoftwareFrameProcessor::Reset() +{ + ResetProducts(); + CFrameProcessor::Reset(); +} + +void CSoftwareFrameProcessor::ResetPipeline() +{ + ResetProducts(); + CFrameProcessor::Invalidate(); } diff --git a/idd/LGIdd/capture/CSoftwareFrameProcessor.h b/idd/LGIdd/capture/CSoftwareFrameProcessor.h index 12eead0d..290ce97b 100644 --- a/idd/LGIdd/capture/CSoftwareFrameProcessor.h +++ b/idd/LGIdd/capture/CSoftwareFrameProcessor.h @@ -25,8 +25,74 @@ class CSoftwareFrameProcessor final : public CFrameProcessor { private: + enum ProductState + { + PRODUCT_FREE, + PRODUCT_PREPARING, + PRODUCT_READY, + PRODUCT_PUBLISHING, + PRODUCT_RETAINED, + PRODUCT_COMPLETING, + }; + + struct Product + { + ProductState state = PRODUCT_FREE; + ProductState restoreState = PRODUCT_FREE; + ProductState completedState = PRODUCT_FREE; + ComPtr resource; + D12FrameFormat srcFormat = {}; + D12FrameFormat dstFormat = {}; + RECT dirtyRects[LG_MAX_DIRTY_RECTS] = {}; + unsigned nbDirtyRects = 0; + RECT previousDirtyRects[LG_MAX_DIRTY_RECTS] = {}; + unsigned nbPreviousDirtyRects = 0; + unsigned pitch = 0; + size_t frameSize = 0; + uint64_t sequence = 0; + uint64_t captureTime = 0; + uint64_t postProcessStart = 0; + uint64_t prepareCopyStart = 0; + uint64_t prepareReady = 0; + uint64_t prepareGPUStart = 0; + uint64_t prepareGPUEnd = 0; + uint64_t timingStart = 0; + unsigned timingEffectIndex = 0; + uint64_t timingToken = 0; + bool prepareTimingValid = false; + bool sourceReady = false; + bool batchCommitted = false; + bool productNotified = false; + bool executeActive = false; + bool completionDone = false; + bool completionSucceeded = false; + FrameCopyBatch batch = {}; + }; + + Product m_products[CAPTURE_PIPELINE_SLOTS]; + mutable CSRWLock m_productLock; + Wrappers::Event m_productAvailableEvent; + static void CompletionFunction( CD3D12CommandSlot * slot, bool result, void * param1, void * param2); + int AcquireProduct(); + int WaitForProduct(); + void RestoreProduct(unsigned productIndex, ProductState state); + void FinishProduct( + unsigned productIndex, bool publishing, bool result); + void MarkSourceReady(unsigned productIndex, uint64_t sequence); + void MarkBatchCommitted(unsigned productIndex, uint64_t sequence); + bool EnsureProductResource(unsigned productIndex, size_t frameSize); + bool BeginExecute(unsigned productIndex, uint64_t sequence, + ProductState state); + void EndExecute(unsigned productIndex, uint64_t sequence, + bool& completed, bool& succeeded); + bool PrepareBatch(Product& product, unsigned productIndex, + const FramePlan& plan, uint64_t copyStart); + void CompleteBatch(CD3D12CommandSlot * slot, Product& product, + unsigned productIndex, bool publishing, bool result); + void ResetProducts(); + void SignalProductState(bool available = false); public: CSoftwareFrameProcessor(IFrameTransport * transport, @@ -35,10 +101,10 @@ public: CSRWLock * pipelineLock, HANDLE terminateEvent); bool Submit(const FrameSubmission& submission) override; - bool HasReadyFrame() const override { return false; } - bool Publish(const CFrameScheduler::Schedule&, bool, uint64_t) override - { - return false; - } + bool HasReadyFrame() const override; + bool Publish(const FramePlan& plan, uint64_t publishStart) override; bool UsesCadence() const override { return false; } + bool IsValid() const override; + void Reset() override; + void ResetPipeline() override; }; diff --git a/idd/LGIdd/capture/CSwapChainProcessor.cpp b/idd/LGIdd/capture/CSwapChainProcessor.cpp index 287d39b3..b268af40 100644 --- a/idd/LGIdd/capture/CSwapChainProcessor.cpp +++ b/idd/LGIdd/capture/CSwapChainProcessor.cpp @@ -785,195 +785,66 @@ void CSwapChainProcessor::PublisherThread() for (;;) { - const uint64_t now = CFrameScheduler::Nanotime(); - uint64_t target; - CFrameScheduler::Schedule schedule; - bool periodic; - bool republish; - m_transport.GetPublishTarget( - now, target, schedule, periodic, republish); - + const uint64_t now = CFrameScheduler::Nanotime(); const bool ready = m_frameProcessor->HasReadyFrame(); - if (!ready) - { - m_transport.ProcessDeliveries(); - if (m_frameProcessor->HasReadyFrame()) - continue; + FramePlan plan = {}; + const bool due = m_transport.GetFramePlan(now, ready, plan); - uint64_t current = CFrameScheduler::Nanotime(); - uint64_t cadenceTarget = 0; - if (cadenceEnabled && schedule.deliveryDeadlineSerial && periodic) - { - if (schedule.deadline <= current) - { - m_transport.FrameMissed(schedule, current, periodic); - continue; - } - cadenceTarget = schedule.deadline; - } - - if (republish && m_transport.HasPublishedFrame()) - { - if (m_transport.RepublishFrameBuffer(schedule)) - continue; - - current = CFrameScheduler::Nanotime(); - if (cadenceTarget && cadenceTarget <= current) - { - m_transport.FrameMissed(schedule, current, periodic); - continue; - } - - uint64_t retryTarget = current + PUBLISH_RETRY_NS; - if (cadenceTarget) - retryTarget = min(retryTarget, cadenceTarget); - ArmPublishTimer(m_publishTimer.Get(), retryTarget - current); - if (WaitForMultipleObjects( - ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) == - WAIT_OBJECT_0) - break; - continue; - } - - uint64_t replayTarget; - if (m_transport.GetPendingDeliveryTarget(current, replayTarget)) - { - bool retry = false; - if (replayTarget <= current) - { - if (m_transport.RetryPendingDelivery(current, retry)) - continue; - - current = CFrameScheduler::Nanotime(); - if (cadenceTarget && cadenceTarget <= current) - { - m_transport.FrameMissed(schedule, current, periodic); - continue; - } - - if (retry) - replayTarget = current + PUBLISH_RETRY_NS; - else - { - if (cadenceTarget) - replayTarget = cadenceTarget; - else - { - if (m_publishTimer.Get()) - CancelWaitableTimer(m_publishTimer.Get()); - if (WaitForMultipleObjects( - ARRAYSIZE(idleHandles), idleHandles, FALSE, INFINITE) == - WAIT_OBJECT_0) - break; - continue; - } - } - } - - if (cadenceTarget) - replayTarget = min(replayTarget, cadenceTarget); - - current = CFrameScheduler::Nanotime(); - if (cadenceTarget && cadenceTarget <= current) - { - m_transport.FrameMissed(schedule, current, periodic); - continue; - } - if (replayTarget <= current) - continue; - - ArmPublishTimer(m_publishTimer.Get(), replayTarget - current); - if (WaitForMultipleObjects( - ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) == - WAIT_OBJECT_0) - break; - continue; - } - - if (cadenceTarget) - { - current = CFrameScheduler::Nanotime(); - if (cadenceTarget <= current) - { - m_transport.FrameMissed(schedule, current, periodic); - continue; - } - - ArmPublishTimer(m_publishTimer.Get(), cadenceTarget - current); - if (WaitForMultipleObjects( - ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) == - WAIT_OBJECT_0) - break; - continue; - } - - if (m_publishTimer.Get()) - CancelWaitableTimer(m_publishTimer.Get()); - if (WaitForMultipleObjects( - ARRAYSIZE(idleHandles), idleHandles, FALSE, INFINITE) == - WAIT_OBJECT_0) - break; + if (plan.progressed) continue; + + if (due && ready) + { + const uint64_t publishStart = CFrameScheduler::Nanotime(); + if (m_frameProcessor->Publish(plan, publishStart)) + continue; + const uint64_t retry = publishStart + PUBLISH_RETRY_NS; + if (!plan.nextWake || retry < plan.nextWake) + plan.nextWake = retry; } uint64_t current = CFrameScheduler::Nanotime(); - uint64_t replayTarget; - if (m_transport.GetPendingDeliveryTarget(current, replayTarget) && - replayTarget < target) + bool missed = false; + if (due && !ready && cadenceEnabled) { - if (replayTarget <= current) + for (unsigned i = 0; i < plan.count; ++i) { - m_transport.ProcessDeliveries(); - current = CFrameScheduler::Nanotime(); - bool retry = false; - if (m_transport.RetryPendingDelivery(current, retry)) + const FramePlanTarget& request = plan.targets[i]; + if (!request.periodic || + !request.commitSchedule.deliveryDeadlineSerial) continue; - - current = CFrameScheduler::Nanotime(); - if (retry) - replayTarget = current + PUBLISH_RETRY_NS; - else - replayTarget = target; - } - - replayTarget = min(replayTarget, target); - current = CFrameScheduler::Nanotime(); - if (target > current) - { - if (replayTarget <= current) - continue; - - ArmPublishTimer(m_publishTimer.Get(), replayTarget - current); - if (WaitForMultipleObjects( - ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) == - WAIT_OBJECT_0) - break; - continue; + if (request.commitSchedule.deadline <= current) + missed = true; + else if (!plan.nextWake || + request.commitSchedule.deadline < plan.nextWake) + plan.nextWake = request.commitSchedule.deadline; } } + if (missed) + { + m_transport.MissFramePlan(plan, current); + continue; + } current = CFrameScheduler::Nanotime(); - if (target > current) + if (plan.nextWake && plan.nextWake > current) { - ArmPublishTimer(m_publishTimer.Get(), target - current); + ArmPublishTimer(m_publishTimer.Get(), plan.nextWake - current); if (WaitForMultipleObjects( ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) == - WAIT_OBJECT_0) + WAIT_OBJECT_0) break; continue; } - const uint64_t publishStart = CFrameScheduler::Nanotime(); - m_transport.ProcessDeliveries(); - if (!m_transport.FrameBufferAvailable(schedule) || - !m_frameProcessor->Publish(schedule, periodic, publishStart)) - { - ArmPublishTimer(m_publishTimer.Get(), PUBLISH_RETRY_NS); - if (WaitForMultipleObjects( - ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) == - WAIT_OBJECT_0) - break; - } + if (plan.nextWake) + continue; + if (m_publishTimer.Get()) + CancelWaitableTimer(m_publishTimer.Get()); + if (WaitForMultipleObjects( + ARRAYSIZE(idleHandles), idleHandles, FALSE, INFINITE) == + WAIT_OBJECT_0) + break; } if (avTaskHandle) diff --git a/idd/LGIdd/transport/CFrameHub.cpp b/idd/LGIdd/transport/CFrameHub.cpp index 201b39e0..042cf999 100644 --- a/idd/LGIdd/transport/CFrameHub.cpp +++ b/idd/LGIdd/transport/CFrameHub.cpp @@ -7,301 +7,892 @@ * under the terms of the GNU General Public License as published by the Free * Software Foundation; either version 2 of the License, or (at your option) * any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., 59 - * Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include "transport/CFrameHub.h" -bool CFrameHub::Bind( - BackendId backend, uint32_t epoch, IFrameSink& sink) +#include "CDebug.h" + +static const uint64_t RETRY_NS = 1000000ULL; + +static void Earlier(uint64_t value, uint64_t& target) { - if (!backend || !epoch) + if (value && (!target || value < target)) + target = value; +} + +static bool ContentAfter(uint64_t value, uint64_t current) +{ + return value && (!current || + static_cast(value - current) > 0); +} + +CFrameHub::CFrameHub() +{ + m_wakeEvent = CreateEvent(nullptr, FALSE, FALSE, nullptr); + for (Sink& sink : m_sinks) + sink.drained = CreateEvent(nullptr, TRUE, TRUE, nullptr); +} + +CFrameHub::~CFrameHub() +{ + for (Sink& sink : m_sinks) + { + const BackendId backend = + sink.backend.load(std::memory_order_acquire); + const uint32_t epoch = sink.epoch.load(std::memory_order_acquire); + if (backend && epoch) + Unbind(backend, epoch); + } + + for (Sink& sink : m_sinks) + { + if (sink.drained) + CloseHandle(sink.drained); + } + if (m_wakeEvent) + CloseHandle(m_wakeEvent); +} + +bool CFrameHub::Bind(BackendId backend, uint32_t epoch, bool primary, + IFrameSink& target) +{ + if (!backend || !epoch || !m_wakeEvent) return false; - CSRWExclusiveLock lock(m_lock); - if (m_sink) - return false; - m_sink = &sink; - m_backend = backend; - m_epoch = epoch; - m_slots.clear(); + Sink * selected = nullptr; + { + CSRWExclusiveLock lock(m_listLock); + if (primary) + { + if (!m_sinks[0].active.load(std::memory_order_acquire) && + !m_sinks[0].reserved) + selected = &m_sinks[0]; + } + else + for (unsigned i = 1; i < FRAME_MAX_SINKS; ++i) + if (!m_sinks[i].active.load(std::memory_order_acquire) && + !m_sinks[i].reserved) + { + selected = &m_sinks[i]; + break; + } + + if (!selected || !selected->drained) + return false; + selected->reserved = true; + } + + { + CSRWExclusiveLock lock(selected->callLock); + selected->target = ⌖ + selected->backend.store(backend, std::memory_order_release); + selected->epoch.store(epoch, std::memory_order_release); + selected->primary = primary; + selected->needsFullCopy = true; + selected->lastContent = 0; + selected->blockedContent = 0; + selected->blockedFrameSize = 0; + selected->blockedAllocation = false; + selected->pitch = 0; + selected->width = 0; + selected->height = 0; + selected->format = DXGI_FORMAT_UNKNOWN; + selected->frameType = FRAME_TYPE_INVALID; + for (Sink::ResourceLane& lane : selected->lanes) + lane = {}; + selected->outstanding.store(0, std::memory_order_release); + SetEvent(selected->drained); + target.SetFrameScheduleEvent(m_wakeEvent); + } + + { + CSRWExclusiveLock lock(m_listLock); + selected->active.store(true, std::memory_order_release); + selected->reserved = false; + } + target.ForceFrame(); + SetEvent(m_wakeEvent); return true; } void CFrameHub::Unbind(BackendId backend, uint32_t epoch) { - CSRWExclusiveLock lock(m_lock); - if (m_backend != backend || m_epoch != epoch) + Sink * selected = nullptr; + { + CSRWExclusiveLock lock(m_listLock); + for (Sink& sink : m_sinks) + if (sink.active.load(std::memory_order_acquire) && + sink.backend.load(std::memory_order_acquire) == backend && + sink.epoch.load(std::memory_order_acquire) == epoch) + { + sink.active.store(false, std::memory_order_release); + sink.reserved = true; + selected = &sink; + break; + } + } + if (!selected) return; - m_sink = nullptr; - m_backend = 0; - m_epoch = 0; - m_slots.clear(); + + { + CSRWExclusiveLock lock(selected->callLock); + if (selected->target) + selected->target->SetFrameScheduleEvent(nullptr); + } + + WaitForSingleObject(selected->drained, INFINITE); + { + CSRWExclusiveLock lock(selected->callLock); + selected->target = nullptr; + selected->backend.store(0, std::memory_order_release); + selected->epoch.store(0, std::memory_order_release); + selected->primary = false; + selected->needsFullCopy = true; + selected->lastContent = 0; + selected->blockedContent = 0; + selected->blockedFrameSize = 0; + selected->blockedAllocation = false; + for (Sink::ResourceLane& lane : selected->lanes) + lane = {}; + } + { + CSRWExclusiveLock lock(m_listLock); + selected->reserved = false; + } + SetEvent(m_wakeEvent); } -bool CFrameHub::Valid(const FrameToken& token) const +unsigned CFrameHub::Snapshot(SinkRef refs[FRAME_MAX_SINKS]) const { - return m_sink && token.sink == m_backend && token.epoch == m_epoch && - token.slot < m_slots.size() && token.serial && - m_slots[token.slot].serial == token.serial; + unsigned count = 0; + CSRWSharedLock lock(m_listLock); + for (unsigned i = 0; i < FRAME_MAX_SINKS; ++i) + if (m_sinks[i].active.load(std::memory_order_acquire)) + refs[count++] = { + const_cast(&m_sinks[i]), i, m_sinks[i].primary }; + return count; } -void CFrameHub::Invalidate(const FrameToken& token) +bool CFrameHub::BatchValid( + const Batch& batch, const FrameBatchToken& token) const { - if (Valid(token)) - m_slots[token.slot] = {}; + return token.serial && batch.active && batch.serial == token.serial; +} + +void CFrameHub::ReleaseTarget(Batch& batch, BatchTarget& target) +{ + if (!target.active || target.releasePending) + return; + target.releasePending = true; + if (target.resourceLane < FRAME_SINK_BUFFERS) + target.sink->lanes[target.resourceLane].busy = false; + if (target.sink->outstanding.fetch_sub( + 1, std::memory_order_acq_rel) == 1) + SetEvent(target.sink->drained); + + target.active = false; + + for (unsigned i = 0; i < batch.count; ++i) + if (batch.targets[i].active) + return; + batch.active = false; +} + +void CFrameHub::CompleteTarget( + Batch& batch, BatchTarget& target, bool succeeded) +{ + CSRWExclusiveLock call(target.sink->callLock); + if (target.sink->target && + target.sink->backend.load(std::memory_order_acquire) == + target.backend && + target.sink->epoch.load(std::memory_order_acquire) == target.epoch) + { + target.sink->target->CompleteFrameBuffer(target.localSlot, succeeded); + if (succeeded) + { + if (ContentAfter(target.content, target.sink->lastContent)) + { + target.sink->lastContent = target.content; + target.sink->pitch = target.pitch; + target.sink->width = target.width; + target.sink->height = target.height; + target.sink->format = target.format; + target.sink->frameType = target.frameType; + target.sink->needsFullCopy = false; + target.sink->blockedContent = 0; + target.sink->blockedFrameSize = 0; + target.sink->blockedAllocation = false; + } + + } + else + target.sink->needsFullCopy = true; + } + ReleaseTarget(batch, target); } size_t CFrameHub::GetMaxFrameSize() const { - CSRWSharedLock lock(m_lock); - return m_sink ? m_sink->GetMaxFrameSize() : 0; + SinkRef refs[FRAME_MAX_SINKS]; + const unsigned count = Snapshot(refs); + for (unsigned i = 0; i < count; ++i) + if (refs[i].primary) + { + CSRWSharedLock call(refs[i].sink->callLock); + if (refs[i].sink->active.load(std::memory_order_acquire) && + refs[i].sink->target) + return refs[i].sink->target->GetMaxFrameSize(); + } + return 0; } -bool CFrameHub::FrameBufferAvailable( - const CFrameScheduler::Schedule& schedule, bool allowReadyReplacement) +uint64_t CFrameHub::NextContentSerial() { - CSRWSharedLock lock(m_lock); - return m_sink && - m_sink->FrameBufferAvailable(schedule, allowReadyReplacement); + uint64_t serial = m_nextContent.fetch_add( + 1, std::memory_order_acq_rel) + 1; + if (!serial) + serial = m_nextContent.fetch_add( + 1, std::memory_order_acq_rel) + 1; + return serial; } -bool CFrameHub::HasPublishedFrame() const +void CFrameHub::FrameProductReady(uint64_t contentSerial) { - CSRWSharedLock lock(m_lock); - return m_sink && m_sink->HasPublishedFrame(); + uint64_t newest = m_newestContent.load(std::memory_order_acquire); + while (ContentAfter(contentSerial, newest) && + !m_newestContent.compare_exchange_weak(newest, + contentSerial, std::memory_order_acq_rel, + std::memory_order_acquire)) + { + } + SetEvent(m_wakeEvent); } -void CFrameHub::ProcessDeliveries() +bool CFrameHub::NeedsFrame() const { - CSRWSharedLock lock(m_lock); - if (m_sink) - m_sink->ProcessDeliveries(); + const uint64_t newest = + m_newestContent.load(std::memory_order_acquire); + SinkRef refs[FRAME_MAX_SINKS]; + const unsigned count = Snapshot(refs); + for (unsigned i = 0; i < count; ++i) + { + CSRWSharedLock call(refs[i].sink->callLock); + if (!refs[i].sink->active.load(std::memory_order_acquire) || + !refs[i].sink->target) + continue; + if (refs[i].sink->blockedContent == newest && + refs[i].sink->blockedFrameSize && + (refs[i].sink->blockedAllocation || + refs[i].sink->target->GetMaxFrameSize() < + refs[i].sink->blockedFrameSize)) + continue; + if (refs[i].sink->needsFullCopy || + ContentAfter(newest, refs[i].sink->lastContent)) + return true; + } + return false; } -bool CFrameHub::GetPendingDeliveryTarget(uint64_t now, uint64_t& target) +bool CFrameHub::GetFramePlan( + uint64_t now, bool, FramePlan& plan) { - CSRWSharedLock lock(m_lock); - return m_sink && m_sink->GetPendingDeliveryTarget(now, target); + plan = {}; + SinkRef refs[FRAME_MAX_SINKS]; + const unsigned count = Snapshot(refs); + for (unsigned i = 0; i < count; ++i) + { + Sink& sink = *refs[i].sink; + CSRWSharedLock call(sink.callLock); + if (!sink.active.load(std::memory_order_acquire) || !sink.target) + continue; + + sink.target->ProcessDeliveries(); + uint64_t deliveryTarget = 0; + if (sink.target->GetPendingDeliveryTarget(now, deliveryTarget)) + { + if (deliveryTarget <= now) + { + bool retry = false; + if (sink.target->RetryPendingDelivery(now, retry)) + plan.progressed = true; + else if (retry) + Earlier(now + RETRY_NS, plan.nextWake); + } + else + Earlier(deliveryTarget, plan.nextWake); + } + + uint64_t target = 0; + CFrameScheduler::Schedule schedule = {}; + bool periodic = false; + bool republish = false; + if (!sink.target->GetPublishTarget( + now, target, schedule, periodic, republish)) + continue; + + if (target > now) + { + Earlier(target, plan.nextWake); + continue; + } + + if (republish && sink.target->HasPublishedFrame()) + { + if (sink.target->RepublishFrameBuffer(schedule)) + { + plan.progressed = true; + continue; + } + else + { + Earlier(now + RETRY_NS, plan.nextWake); + continue; + } + } + + if (plan.count == FRAME_MAX_SINKS) + continue; + FramePlanTarget& request = plan.targets[plan.count++]; + request.sink = refs[i].index; + request.backend = sink.backend.load(std::memory_order_acquire); + request.epoch = sink.epoch.load(std::memory_order_acquire); + request.schedule = schedule; + request.commitSchedule = schedule; + request.periodic = periodic; + request.primary = sink.primary; + } + return plan.count != 0; } -bool CFrameHub::RetryPendingDelivery(uint64_t now, bool& retry) +bool CFrameHub::GetImmediateFramePlan(uint64_t now, FramePlan& plan) { - CSRWSharedLock lock(m_lock); - return m_sink && m_sink->RetryPendingDelivery(now, retry); + plan = {}; + SinkRef refs[FRAME_MAX_SINKS]; + const unsigned count = Snapshot(refs); + for (unsigned i = 0; i < count; ++i) + { + Sink& sink = *refs[i].sink; + CSRWSharedLock call(sink.callLock); + if (!sink.active.load(std::memory_order_acquire) || !sink.target) + continue; + + sink.target->ProcessDeliveries(); + uint64_t deliveryTarget = 0; + if (sink.target->GetPendingDeliveryTarget(now, deliveryTarget) && + deliveryTarget <= now) + { + bool retry = false; + if (sink.target->RetryPendingDelivery(now, retry)) + plan.progressed = true; + } + + uint64_t target = 0; + CFrameScheduler::Schedule schedule = {}; + bool periodic = false; + bool republish = false; + sink.target->GetPublishTarget( + now, target, schedule, periodic, republish); + + if (plan.count == FRAME_MAX_SINKS) + continue; + FramePlanTarget& request = plan.targets[plan.count++]; + request.sink = refs[i].index; + request.backend = sink.backend.load(std::memory_order_acquire); + request.epoch = sink.epoch.load(std::memory_order_acquire); + request.schedule = schedule; + request.schedule.deliveryDeadlineSerial = 0; + request.schedule.phaseEligible = false; + request.commitSchedule = schedule; + request.periodic = false; + request.primary = sink.primary; + } + return plan.count != 0; } -PreparedFrameBuffer CFrameHub::PrepareFrameBuffer(unsigned pitch, +void CFrameHub::MissFramePlan(const FramePlan& plan, uint64_t now) +{ + for (unsigned i = 0; i < plan.count; ++i) + { + const FramePlanTarget& request = plan.targets[i]; + if (!request.periodic || + !request.schedule.deliveryDeadlineSerial || + request.schedule.deadline > now || + request.sink >= FRAME_MAX_SINKS) + continue; + Sink& sink = m_sinks[request.sink]; + CSRWSharedLock call(sink.callLock); + if (sink.active.load(std::memory_order_acquire) && sink.target && + sink.backend.load(std::memory_order_acquire) == request.backend && + sink.epoch.load(std::memory_order_acquire) == request.epoch) + sink.target->FrameMissed( + request.commitSchedule, now, request.periodic); + } +} + +bool CFrameHub::PrepareFrameBatch(const FramePlan& plan, + uint64_t contentSerial, unsigned pitch, size_t frameSize, const D12FrameFormat& srcFormat, const D12FrameFormat& dstFormat, const RECT * dirtyRects, unsigned nbDirtyRects, - const CFrameScheduler::Schedule& schedule, bool allowReadyReplacement) + bool allowReadyReplacement, PreparedFrameBatch& prepared) { - PreparedFrameBuffer result = {}; - CSRWExclusiveLock lock(m_lock); - if (!m_sink) - return result; + prepared = {}; + if (!contentSerial || !pitch || !frameSize) + return false; - const SinkTarget target = m_sink->PrepareFrameBuffer( - pitch, srcFormat, dstFormat, dirtyRects, nbDirtyRects, schedule, - allowReadyReplacement); - if (!target.mem || target.slot >= MAX_SLOTS) + Batch * batch = nullptr; + unsigned batchSlot = 0; + for (; batchSlot < FRAME_BATCHES; ++batchSlot) { - if (target.mem) - m_sink->AbortFrameBuffer(target.slot); - return result; + Batch& candidate = m_batches[batchSlot]; + CSRWExclusiveLock lock = CSRWExclusiveLock::Try(candidate.lock); + if (!lock || candidate.active) + continue; + candidate.active = true; + candidate.count = 0; + candidate.serial = m_nextSerial.fetch_add( + 1, std::memory_order_acq_rel) + 1; + if (!candidate.serial) + candidate.serial = m_nextSerial.fetch_add( + 1, std::memory_order_acq_rel) + 1; + for (BatchTarget& target : candidate.targets) + target = {}; + prepared.token = { + static_cast(batchSlot), candidate.serial }; + batch = &candidate; + lock.Unlock(); + break; } + if (!batch) + return false; - if (target.slot >= m_slots.size()) - m_slots.resize(target.slot + 1); - if (++m_nextSerial == 0) - ++m_nextSerial; - m_slots[target.slot].serial = m_nextSerial; - - result.token.sink = m_backend; - result.token.epoch = m_epoch; - result.token.slot = target.slot; - result.token.serial = m_nextSerial; - result.resourceSlot = target.slot; - result.mem = target.mem; - result.heapOffset = target.heapOffset; - result.fullCopy = target.fullCopy; - return result; -} - -bool CFrameHub::PublishFrameBuffer(const FrameToken& token, - const CFrameScheduler::Schedule& schedule, bool& deliveredToOwner) -{ - CSRWSharedLock lock(m_lock); - return Valid(token) && - m_sink->PublishFrameBuffer(token.slot, schedule, deliveredToOwner); -} - -bool CFrameHub::RepublishFrameBuffer( - const CFrameScheduler::Schedule& schedule) -{ - CSRWSharedLock lock(m_lock); - return m_sink && m_sink->RepublishFrameBuffer(schedule); -} - -bool CFrameHub::TryFrameSubmitted(const FrameToken& token, - const CFrameScheduler::Schedule& schedule) -{ - CSRWSharedLock lock(m_lock); - return Valid(token) && - m_sink->TryFrameSubmitted(token.slot, schedule); -} - -void CFrameHub::CommitFrameBuffer(const FrameToken& token, - const CFrameScheduler::Schedule& schedule, bool periodic, - bool deliveredToOwner) -{ - CSRWExclusiveLock lock(m_lock); - if (Valid(token)) + CSRWExclusiveLock batchLock(batch->lock); + for (unsigned i = 0; i < plan.count && + batch->count < FRAME_MAX_SINKS; ++i) { - m_sink->CommitFrameBuffer( - token.slot, schedule, periodic, deliveredToOwner); - Slot& slot = m_slots[token.slot]; - slot.committed = true; - if (slot.completionPending) + const FramePlanTarget& request = plan.targets[i]; + if (request.sink >= FRAME_MAX_SINKS) + continue; + Sink& sink = m_sinks[request.sink]; + CSRWExclusiveLock call(sink.callLock); + if (!sink.active.load(std::memory_order_acquire) || !sink.target || + sink.backend.load(std::memory_order_acquire) != request.backend || + sink.epoch.load(std::memory_order_acquire) != request.epoch || + !sink.target->FrameBufferAvailable( + request.schedule, allowReadyReplacement)) + continue; + + const size_t maxFrameSize = sink.target->GetMaxFrameSize(); + if (!maxFrameSize || frameSize > maxFrameSize) { - m_sink->CompleteFrameBuffer( - token.slot, slot.completionSucceeded); - Invalidate(token); + sink.blockedContent = contentSerial; + sink.blockedFrameSize = frameSize; + sink.blockedAllocation = false; + continue; } - } -} -void CFrameHub::AbortFrameBuffer(const FrameToken& token) -{ - CSRWExclusiveLock lock(m_lock); - if (Valid(token)) - { - m_sink->AbortFrameBuffer(token.slot); - Invalidate(token); - } -} - -void CFrameHub::FailFrameBuffer(const FrameToken& token) -{ - CSRWExclusiveLock lock(m_lock); - if (Valid(token)) - { - m_sink->FailFrameBuffer(token.slot); - Invalidate(token); - } -} - -void CFrameHub::CompleteFrameBuffer( - const FrameToken& token, bool succeeded) -{ - CSRWExclusiveLock lock(m_lock); - if (Valid(token)) - { - Slot& slot = m_slots[token.slot]; - if (slot.committed) + const bool layoutChanged = sink.pitch != pitch || + sink.width != dstFormat.width || sink.height != dstFormat.height || + sink.format != dstFormat.desc.Format || + sink.frameType != dstFormat.format; + const bool forceFull = sink.needsFullCopy || layoutChanged || + sink.lastContent + 1 != contentSerial; + const RECT * damage = forceFull ? nullptr : dirtyRects; + const unsigned damageCount = forceFull ? 0 : nbDirtyRects; + SinkTarget result = sink.target->PrepareFrameBuffer( + pitch, srcFormat, dstFormat, damage, damageCount, + request.schedule, allowReadyReplacement); + if (!result.mem || result.capacity < frameSize) { - m_sink->CompleteFrameBuffer(token.slot, succeeded); - Invalidate(token); + if (result.mem) + { + sink.blockedContent = contentSerial; + sink.blockedFrameSize = frameSize; + sink.blockedAllocation = true; + sink.target->AbortFrameBuffer(result.slot); + } + continue; } - else + + unsigned resourceLane = FRAME_SINK_BUFFERS; + for (unsigned lane = 0; lane < FRAME_SINK_BUFFERS; ++lane) { - slot.completionPending = true; - slot.completionSucceeded = succeeded; + const Sink::ResourceLane& candidate = sink.lanes[lane]; + if (!candidate.busy && candidate.valid && + candidate.mem == result.mem && + candidate.heapOffset == result.heapOffset && + candidate.localSlot == result.slot && + candidate.direct == request.primary) + { + resourceLane = lane; + break; + } } + if (resourceLane == FRAME_SINK_BUFFERS) + for (unsigned lane = 0; lane < FRAME_SINK_BUFFERS; ++lane) + if (!sink.lanes[lane].busy && !sink.lanes[lane].valid) + { + resourceLane = lane; + break; + } + if (resourceLane == FRAME_SINK_BUFFERS) + for (unsigned lane = 0; lane < FRAME_SINK_BUFFERS; ++lane) + if (!sink.lanes[lane].busy) + { + resourceLane = lane; + break; + } + if (resourceLane == FRAME_SINK_BUFFERS) + { + sink.target->AbortFrameBuffer(result.slot); + continue; + } + + Sink::ResourceLane& lane = sink.lanes[resourceLane]; + lane.mem = result.mem; + lane.heapOffset = result.heapOffset; + lane.localSlot = result.slot; + lane.direct = request.primary; + lane.valid = true; + lane.busy = true; + + unsigned index = batch->count; + if (sink.primary && index) + { + for (unsigned move = index; move != 0; --move) + { + batch->targets[move] = batch->targets[move - 1]; + prepared.targets[move] = prepared.targets[move - 1]; + } + index = 0; + } + ++batch->count; + BatchTarget& target = batch->targets[index]; + target.sink = &sink; + target.backend = request.backend; + target.epoch = request.epoch; + target.localSlot = result.slot; + target.resourceLane = resourceLane; + target.schedule = request.commitSchedule; + target.deliverySchedule = request.schedule; + target.content = contentSerial; + target.pitch = pitch; + target.width = dstFormat.width; + target.height = dstFormat.height; + target.format = dstFormat.desc.Format; + target.frameType = dstFormat.format; + target.periodic = request.periodic; + target.active = true; + if (sink.outstanding.fetch_add( + 1, std::memory_order_acq_rel) == 0) + ResetEvent(sink.drained); + + PreparedFrameBuffer& output = prepared.targets[index]; + output.token.sink = request.backend; + output.token.epoch = request.epoch; + output.token.slot = result.slot; + output.token.serial = batch->serial; + output.resourceSlot = + request.sink * FRAME_SINK_BUFFERS + resourceLane; + output.mem = result.mem; + output.heapOffset = result.heapOffset; + output.capacity = result.capacity; + output.direct = request.primary; + output.fullCopy = forceFull || result.fullCopy; + } + prepared.count = batch->count; + if (!batch->count) + { + batch->active = false; + prepared = {}; + return false; + } + return true; +} + +uint32_t CFrameHub::PublishFrameBatch(const FrameBatchToken& token) +{ + if (token.slot >= FRAME_BATCHES) + return 0; + Batch& batch = m_batches[token.slot]; + CSRWExclusiveLock lock(batch.lock); + if (!BatchValid(batch, token)) + return 0; + + uint32_t accepted = 0; + for (unsigned i = 0; i < batch.count; ++i) + { + BatchTarget& target = batch.targets[i]; + if (!target.active) + continue; + CSRWExclusiveLock call(target.sink->callLock); + bool delivered = false; + const bool valid = target.sink->target && + target.sink->backend.load(std::memory_order_acquire) == + target.backend && + target.sink->epoch.load(std::memory_order_acquire) == target.epoch; + if (!valid || !target.sink->target->PublishFrameBuffer( + target.localSlot, target.deliverySchedule, delivered)) + { + if (valid) + target.sink->target->AbortFrameBuffer(target.localSlot); + target.sink->needsFullCopy = true; + ReleaseTarget(batch, target); + continue; + } + + target.published = true; + target.delivered = delivered; + target.submitted = delivered && + target.sink->target->TryFrameSubmitted( + target.localSlot, target.deliverySchedule); + if (!target.submitted) + target.schedule.phaseEligible = false; + accepted |= 1U << i; + } + return accepted; +} + +void CFrameHub::CommitFrameBatch(const FrameBatchToken& token) +{ + if (token.slot >= FRAME_BATCHES) + return; + Batch& batch = m_batches[token.slot]; + CSRWExclusiveLock lock(batch.lock); + if (!BatchValid(batch, token)) + return; + for (unsigned i = 0; i < batch.count; ++i) + { + BatchTarget& target = batch.targets[i]; + if (!target.active || !target.published) + continue; + { + CSRWExclusiveLock call(target.sink->callLock); + if (target.sink->target && + target.sink->backend.load(std::memory_order_acquire) == + target.backend && + target.sink->epoch.load(std::memory_order_acquire) == + target.epoch) + target.sink->target->CommitFrameBuffer(target.localSlot, + target.schedule, target.periodic, target.delivered); + } + target.committed = true; + if (target.completionPending) + CompleteTarget(batch, target, target.completionSucceeded); } } -void CFrameHub::SetFrameTiming(const FrameToken& token, - uint64_t captureTime, uint64_t postProcessTime, uint64_t copyTime, - uint64_t readyTime, uint64_t holdTime, - const CFrameScheduler::Schedule& schedule, uint64_t completedAt) +void CFrameHub::AbortFrameBatch(const FrameBatchToken& token) { - CSRWSharedLock lock(m_lock); - if (Valid(token)) - m_sink->SetFrameTiming(token.slot, captureTime, postProcessTime, - copyTime, readyTime, holdTime, schedule, completedAt); + if (token.slot >= FRAME_BATCHES) + return; + Batch& batch = m_batches[token.slot]; + CSRWExclusiveLock lock(batch.lock); + if (!BatchValid(batch, token)) + return; + for (unsigned i = 0; i < batch.count; ++i) + { + BatchTarget& target = batch.targets[i]; + if (!target.active) + continue; + CSRWExclusiveLock call(target.sink->callLock); + if (target.sink->target && + target.sink->backend.load(std::memory_order_acquire) == + target.backend && + target.sink->epoch.load(std::memory_order_acquire) == target.epoch) + target.sink->target->AbortFrameBuffer(target.localSlot); + target.sink->needsFullCopy = true; + ReleaseTarget(batch, target); + } } -void CFrameHub::WriteFrameBuffer(const FrameToken& token, void * src, - size_t offset, size_t len, bool setWritePos) const +void CFrameHub::FailFrameBatch(const FrameBatchToken& token) { - CSRWSharedLock lock(m_lock); - if (Valid(token)) - m_sink->WriteFrameBuffer( - token.slot, src, offset, len, setWritePos); + if (token.slot >= FRAME_BATCHES) + return; + Batch& batch = m_batches[token.slot]; + CSRWExclusiveLock lock(batch.lock); + if (!BatchValid(batch, token)) + return; + for (unsigned i = 0; i < batch.count; ++i) + { + BatchTarget& target = batch.targets[i]; + if (!target.active) + continue; + CSRWExclusiveLock call(target.sink->callLock); + if (target.sink->target && + target.sink->backend.load(std::memory_order_acquire) == + target.backend && + target.sink->epoch.load(std::memory_order_acquire) == target.epoch) + { + if (target.published) + target.sink->target->FailFrameBuffer(target.localSlot); + else + target.sink->target->AbortFrameBuffer(target.localSlot); + } + target.sink->needsFullCopy = true; + ReleaseTarget(batch, target); + } } -void CFrameHub::WriteFrameBufferRows(const FrameToken& token, void * src, - size_t offset, size_t rowBytes, size_t pitch, unsigned rows) const +void CFrameHub::WriteFrameTarget(const FrameBatchToken& token, + unsigned index, void * src, size_t offset, size_t len, + bool setWritePos) const { - CSRWSharedLock lock(m_lock); - if (Valid(token)) - m_sink->WriteFrameBufferRows( - token.slot, src, offset, rowBytes, pitch, rows); + if (token.slot >= FRAME_BATCHES) + return; + Batch& batch = const_cast(m_batches[token.slot]); + CSRWSharedLock lock(batch.lock); + if (!BatchValid(batch, token) || index >= batch.count || + !batch.targets[index].active) + return; + BatchTarget& target = batch.targets[index]; + CSRWSharedLock call(target.sink->callLock); + if (target.sink->target && + target.sink->backend.load(std::memory_order_acquire) == + target.backend && + target.sink->epoch.load(std::memory_order_acquire) == target.epoch) + target.sink->target->WriteFrameBuffer( + target.localSlot, src, offset, len, setWritePos); } -void CFrameHub::FinalizeFrameBuffer(const FrameToken& token) const +void CFrameHub::WriteFrameTargetRows(const FrameBatchToken& token, + unsigned index, void * src, size_t offset, size_t rowBytes, + size_t pitch, unsigned rows) const { - CSRWSharedLock lock(m_lock); - if (Valid(token)) - m_sink->FinalizeFrameBuffer(token.slot); + if (token.slot >= FRAME_BATCHES) + return; + Batch& batch = const_cast(m_batches[token.slot]); + CSRWSharedLock lock(batch.lock); + if (!BatchValid(batch, token) || index >= batch.count || + !batch.targets[index].active) + return; + BatchTarget& target = batch.targets[index]; + CSRWSharedLock call(target.sink->callLock); + if (target.sink->target && + target.sink->backend.load(std::memory_order_acquire) == + target.backend && + target.sink->epoch.load(std::memory_order_acquire) == target.epoch) + target.sink->target->WriteFrameBufferRows(target.localSlot, src, + offset, rowBytes, pitch, rows); +} + +void CFrameHub::FinalizeFrameTarget( + const FrameBatchToken& token, unsigned index) const +{ + if (token.slot >= FRAME_BATCHES) + return; + Batch& batch = const_cast(m_batches[token.slot]); + CSRWSharedLock lock(batch.lock); + if (!BatchValid(batch, token) || index >= batch.count || + !batch.targets[index].active) + return; + BatchTarget& target = batch.targets[index]; + CSRWSharedLock call(target.sink->callLock); + if (target.sink->target && + target.sink->backend.load(std::memory_order_acquire) == + target.backend && + target.sink->epoch.load(std::memory_order_acquire) == target.epoch) + target.sink->target->FinalizeFrameBuffer(target.localSlot); +} + +void CFrameHub::SetFrameTargetTiming(const FrameBatchToken& token, + unsigned index, uint64_t captureTime, uint64_t postProcessTime, + uint64_t copyTime, uint64_t readyTime, uint64_t holdTime, + uint64_t completedAt) +{ + if (token.slot >= FRAME_BATCHES) + return; + Batch& batch = m_batches[token.slot]; + CSRWSharedLock lock(batch.lock); + if (!BatchValid(batch, token) || index >= batch.count || + !batch.targets[index].active) + return; + BatchTarget& target = batch.targets[index]; + CSRWSharedLock call(target.sink->callLock); + if (target.sink->target && + target.sink->backend.load(std::memory_order_acquire) == + target.backend && + target.sink->epoch.load(std::memory_order_acquire) == target.epoch) + target.sink->target->SetFrameTiming(target.localSlot, captureTime, + postProcessTime, copyTime, readyTime, holdTime, + target.schedule, completedAt); +} + +void CFrameHub::TryRecordFrameTiming(const FrameBatchToken& token, + unsigned index, uint64_t duration) +{ + if (token.slot >= FRAME_BATCHES) + return; + Batch& batch = m_batches[token.slot]; + CSRWSharedLock lock(batch.lock); + if (!BatchValid(batch, token) || index >= batch.count || + !batch.targets[index].active) + return; + BatchTarget& target = batch.targets[index]; + CSRWSharedLock call(target.sink->callLock); + if (target.sink->target && + target.sink->backend.load(std::memory_order_acquire) == + target.backend && + target.sink->epoch.load(std::memory_order_acquire) == target.epoch) + target.sink->target->TryRecordFrameTiming(duration); +} + +void CFrameHub::CompleteFrameTarget(const FrameBatchToken& token, + unsigned index, bool succeeded) +{ + if (token.slot >= FRAME_BATCHES) + return; + Batch& batch = m_batches[token.slot]; + CSRWExclusiveLock lock(batch.lock); + if (!BatchValid(batch, token) || index >= batch.count || + !batch.targets[index].active) + return; + BatchTarget& target = batch.targets[index]; + if (!target.committed) + { + target.completionPending = true; + target.completionSucceeded = succeeded; + return; + } + CompleteTarget(batch, target, succeeded); } void CFrameHub::ObserveFrame(uint64_t now) { - CSRWSharedLock lock(m_lock); - if (m_sink) - m_sink->ObserveFrame(now); + SinkRef refs[FRAME_MAX_SINKS]; + const unsigned count = Snapshot(refs); + for (unsigned i = 0; i < count; ++i) + { + CSRWSharedLock call(refs[i].sink->callLock); + if (refs[i].sink->active.load(std::memory_order_acquire) && + refs[i].sink->target) + refs[i].sink->target->ObserveFrame(now); + } } void CFrameHub::ForceFrame() { - CSRWSharedLock lock(m_lock); - if (m_sink) - m_sink->ForceFrame(); -} - -bool CFrameHub::GetPublishTarget(uint64_t now, uint64_t& target, - CFrameScheduler::Schedule& schedule, bool& periodic, bool& republish) -{ - CSRWSharedLock lock(m_lock); - return m_sink && m_sink->GetPublishTarget( - now, target, schedule, periodic, republish); -} - -void CFrameHub::FrameMissed(const CFrameScheduler::Schedule& schedule, - uint64_t now, bool periodic) -{ - CSRWSharedLock lock(m_lock); - if (m_sink) - m_sink->FrameMissed(schedule, now, periodic); + SinkRef refs[FRAME_MAX_SINKS]; + const unsigned count = Snapshot(refs); + for (unsigned i = 0; i < count; ++i) + { + CSRWSharedLock call(refs[i].sink->callLock); + if (refs[i].sink->active.load(std::memory_order_acquire) && + refs[i].sink->target) + refs[i].sink->target->ForceFrame(); + } } void CFrameHub::FrameSuperseded() { - CSRWSharedLock lock(m_lock); - if (m_sink) - m_sink->FrameSuperseded(); -} - -HANDLE CFrameHub::GetFrameScheduleEvent() const -{ - CSRWSharedLock lock(m_lock); - return m_sink ? m_sink->GetFrameScheduleEvent() : nullptr; -} - -void CFrameHub::TryRecordFrameTiming( - const FrameToken& token, uint64_t duration) -{ - CSRWSharedLock lock(m_lock); - if (Valid(token)) - m_sink->TryRecordFrameTiming(duration); + SinkRef refs[FRAME_MAX_SINKS]; + const unsigned count = Snapshot(refs); + for (unsigned i = 0; i < count; ++i) + { + CSRWSharedLock call(refs[i].sink->callLock); + if (refs[i].sink->active.load(std::memory_order_acquire) && + refs[i].sink->target) + refs[i].sink->target->FrameSuperseded(); + } } diff --git a/idd/LGIdd/transport/CFrameHub.h b/idd/LGIdd/transport/CFrameHub.h index 6ccfeece..08e686c7 100644 --- a/idd/LGIdd/transport/CFrameHub.h +++ b/idd/LGIdd/transport/CFrameHub.h @@ -25,81 +25,147 @@ #include "transport/IFrameTransport.h" #include "transport/ITransport.h" -#include +#include class CFrameHub final : public IFrameTransport { private: - static const unsigned MAX_SLOTS = 1024; - - struct Slot + struct Sink { - uint64_t serial = 0; - bool committed = false; - bool completionPending = false; - bool completionSucceeded = false; + struct ResourceLane + { + uint8_t * mem = nullptr; + uint64_t heapOffset = 0; + unsigned localSlot = 0; + bool direct = false; + bool valid = false; + bool busy = false; + }; + + CSRWLock callLock; + IFrameSink * target = nullptr; + HANDLE drained = nullptr; + std::atomic outstanding = 0; + std::atomic_bool active = false; + std::atomic backend = 0; + std::atomic epoch = 0; + bool reserved = false; + bool primary = false; + bool needsFullCopy = true; + uint64_t lastContent = 0; + uint64_t blockedContent = 0; + size_t blockedFrameSize = 0; + bool blockedAllocation = false; + unsigned pitch = 0; + unsigned width = 0; + unsigned height = 0; + DXGI_FORMAT format = DXGI_FORMAT_UNKNOWN; + FrameType frameType = FRAME_TYPE_INVALID; + ResourceLane lanes[FRAME_SINK_BUFFERS] = {}; }; - mutable CSRWLock m_lock; - IFrameSink * m_sink = nullptr; - BackendId m_backend = 0; - uint32_t m_epoch = 0; - uint64_t m_nextSerial = 0; - std::vector m_slots; + struct BatchTarget + { + Sink * sink = nullptr; + BackendId backend = 0; + uint32_t epoch = 0; + unsigned localSlot = 0; + unsigned resourceLane = 0; + CFrameScheduler::Schedule schedule = {}; + CFrameScheduler::Schedule deliverySchedule = {}; + uint64_t content = 0; + unsigned pitch = 0; + unsigned width = 0; + unsigned height = 0; + DXGI_FORMAT format = DXGI_FORMAT_UNKNOWN; + FrameType frameType = FRAME_TYPE_INVALID; + bool periodic = false; + bool published = false; + bool delivered = false; + bool submitted = false; + bool committed = false; + bool completionPending = false; + bool completionSucceeded = false; + bool releasePending = false; + bool active = false; + }; - bool Valid(const FrameToken& token) const; - void Invalidate(const FrameToken& token); + struct Batch + { + CSRWLock lock; + uint64_t serial = 0; + unsigned count = 0; + bool active = false; + BatchTarget targets[FRAME_MAX_SINKS] = {}; + }; + + struct SinkRef + { + Sink * sink; + unsigned index; + bool primary; + }; + + mutable CSRWLock m_listLock; + Sink m_sinks[FRAME_MAX_SINKS]; + Batch m_batches[FRAME_BATCHES]; + std::atomic m_nextSerial = 0; + std::atomic m_nextContent = 0; + std::atomic m_newestContent = 0; + HANDLE m_wakeEvent = nullptr; + + unsigned Snapshot(SinkRef refs[FRAME_MAX_SINKS]) const; + bool BatchValid(const Batch& batch, const FrameBatchToken& token) const; + void ReleaseTarget(Batch& batch, BatchTarget& target); + void CompleteTarget(Batch& batch, BatchTarget& target, bool succeeded); public: - bool Bind(BackendId backend, uint32_t epoch, IFrameSink& sink); + CFrameHub(); + ~CFrameHub() override; + + CFrameHub(const CFrameHub&) = delete; + CFrameHub& operator=(const CFrameHub&) = delete; + + bool Bind(BackendId backend, uint32_t epoch, bool primary, + IFrameSink& sink); void Unbind(BackendId backend, uint32_t epoch); size_t GetMaxFrameSize() const override; - bool FrameBufferAvailable(const CFrameScheduler::Schedule& schedule, - bool allowReadyReplacement = true) override; - bool HasPublishedFrame() const override; - void ProcessDeliveries() override; - bool GetPendingDeliveryTarget(uint64_t now, uint64_t& target) override; - bool RetryPendingDelivery(uint64_t now, bool& retry) override; - PreparedFrameBuffer PrepareFrameBuffer(unsigned pitch, + uint64_t NextContentSerial() override; + void FrameProductReady(uint64_t contentSerial) override; + bool NeedsFrame() const override; + bool GetFramePlan( + uint64_t now, bool productReady, FramePlan& plan) override; + bool GetImmediateFramePlan( + uint64_t now, FramePlan& plan) override; + void MissFramePlan(const FramePlan& plan, uint64_t now) override; + bool PrepareFrameBatch(const FramePlan& plan, + uint64_t contentSerial, unsigned pitch, size_t frameSize, const D12FrameFormat& srcFormat, const D12FrameFormat& dstFormat, const RECT * dirtyRects, unsigned nbDirtyRects, - const CFrameScheduler::Schedule& schedule, - bool allowReadyReplacement = true) override; - bool PublishFrameBuffer(const FrameToken& token, - const CFrameScheduler::Schedule& schedule, - bool& deliveredToOwner) override; - bool RepublishFrameBuffer( - const CFrameScheduler::Schedule& schedule) override; - bool TryFrameSubmitted(const FrameToken& token, - const CFrameScheduler::Schedule& schedule) override; - void CommitFrameBuffer(const FrameToken& token, - const CFrameScheduler::Schedule& schedule, bool periodic, - bool deliveredToOwner) override; - void AbortFrameBuffer(const FrameToken& token) override; - void FailFrameBuffer(const FrameToken& token) override; - void CompleteFrameBuffer( - const FrameToken& token, bool succeeded) override; - void SetFrameTiming(const FrameToken& token, uint64_t captureTime, - uint64_t postProcessTime, uint64_t copyTime, uint64_t readyTime, - uint64_t holdTime, const CFrameScheduler::Schedule& schedule, + bool allowReadyReplacement, PreparedFrameBatch& batch) override; + uint32_t PublishFrameBatch(const FrameBatchToken& token) override; + void CommitFrameBatch(const FrameBatchToken& token) override; + void AbortFrameBatch(const FrameBatchToken& token) override; + void FailFrameBatch(const FrameBatchToken& token) override; + void WriteFrameTarget(const FrameBatchToken& token, + unsigned target, void * src, size_t offset, size_t len, + bool setWritePos) const override; + void WriteFrameTargetRows(const FrameBatchToken& token, + unsigned target, void * src, size_t offset, size_t rowBytes, + size_t pitch, unsigned rows) const override; + void FinalizeFrameTarget(const FrameBatchToken& token, + unsigned target) const override; + void SetFrameTargetTiming(const FrameBatchToken& token, + unsigned target, uint64_t captureTime, uint64_t postProcessTime, + uint64_t copyTime, uint64_t readyTime, uint64_t holdTime, uint64_t completedAt) override; - void WriteFrameBuffer(const FrameToken& token, void * src, - size_t offset, size_t len, bool setWritePos) const override; - void WriteFrameBufferRows(const FrameToken& token, void * src, - size_t offset, size_t rowBytes, size_t pitch, - unsigned rows) const override; - void FinalizeFrameBuffer(const FrameToken& token) const override; - + void TryRecordFrameTiming(const FrameBatchToken& token, + unsigned target, uint64_t duration) override; + void CompleteFrameTarget(const FrameBatchToken& token, + unsigned target, bool succeeded) override; void ObserveFrame(uint64_t now) override; void ForceFrame() override; - bool GetPublishTarget(uint64_t now, uint64_t& target, - CFrameScheduler::Schedule& schedule, bool& periodic, - bool& republish) override; - void FrameMissed(const CFrameScheduler::Schedule& schedule, - uint64_t now, bool periodic) override; void FrameSuperseded() override; - HANDLE GetFrameScheduleEvent() const override; - void TryRecordFrameTiming( - const FrameToken& token, uint64_t duration) override; + HANDLE GetFrameScheduleEvent() const override { return m_wakeEvent; } }; diff --git a/idd/LGIdd/transport/CTransportManager.cpp b/idd/LGIdd/transport/CTransportManager.cpp index 284765b5..25e0aca0 100644 --- a/idd/LGIdd/transport/CTransportManager.cpp +++ b/idd/LGIdd/transport/CTransportManager.cpp @@ -68,23 +68,191 @@ public: } }; +CTransportManager::Entry::Entry() +{ + idleEvent = CreateEvent(nullptr, TRUE, TRUE, nullptr); +} + +CTransportManager::Entry::~Entry() +{ + if (idleEvent) + CloseHandle(idleEvent); +} + +CTransportManager::CTransportManager() +{ + m_phaseIdle = CreateEvent(nullptr, TRUE, TRUE, nullptr); + m_stoppedEvent = CreateEvent(nullptr, TRUE, FALSE, nullptr); +} + CTransportManager::~CTransportManager() { Stop(); + if (m_stoppedEvent) + CloseHandle(m_stoppedEvent); + if (m_phaseIdle) + CloseHandle(m_phaseIdle); +} + +unsigned CTransportManager::Entries( + Entry * entries[FRAME_MAX_SINKS]) const +{ + CSRWSharedLock managerLock(m_lock); + for (unsigned i = 0; i < m_entryCount; ++i) + entries[i] = m_entries[i].get(); + return m_entryCount; +} + +CTransportManager::Entry * CTransportManager::Primary() const +{ + CSRWSharedLock managerLock(m_lock); + return m_primary; +} + +bool CTransportManager::BeginPhase( + Phase phase, bool wait, bool stopping) +{ + const DWORD thread = GetCurrentThreadId(); + for (;;) + { + HANDLE idleEvent = nullptr; + { + CSRWExclusiveLock managerLock(m_lock); + if ((!stopping && (m_stopping || m_stopped)) || + (stopping && m_stopped)) + return false; + + if (m_phase == Phase::IDLE) + { + m_phase = phase; + m_phaseOwner = thread; + ResetEvent(m_phaseIdle); + return true; + } + + if (!wait || m_phaseOwner == thread) + return false; + idleEvent = m_phaseIdle; + } + + if (!idleEvent || + WaitForSingleObject(idleEvent, INFINITE) != WAIT_OBJECT_0) + return false; + } +} + +void CTransportManager::EndPhase() +{ + CSRWExclusiveLock managerLock(m_lock); + m_phase = Phase::IDLE; + m_phaseOwner = 0; + SetEvent(m_phaseIdle); +} + +bool CTransportManager::BeginCall( + Entry& entry, Call call, bool wait) +{ + const DWORD thread = GetCurrentThreadId(); + for (;;) + { + HANDLE idleEvent = nullptr; + { + CSRWExclusiveLock entryLock(entry.lock); + if (entry.stopRequested || entry.state == State::STOPPED) + return false; + + if (entry.call == Call::IDLE) + { + entry.call = call; + entry.callOwner = thread; + ResetEvent(entry.idleEvent); + return true; + } + + if (!wait || entry.callOwner == thread) + return false; + idleEvent = entry.idleEvent; + } + + if (!idleEvent || + WaitForSingleObject(idleEvent, INFINITE) != WAIT_OBJECT_0) + return false; + } +} + +void CTransportManager::DrainRecovery(Entry& entry, + const std::shared_ptr& transport) +{ + static const unsigned MAX_DRAIN = 8; + for (unsigned i = 0; i < MAX_DRAIN; ++i) + { + bool sync = false; + bool update = false; + RecoveryUpdate recovery; + { + CSRWExclusiveLock entryLock(entry.lock); + if (entry.stopRequested || !transport || + transport != entry.transport || + (entry.state != State::INITIALIZED && + entry.state != State::READY)) + { + entry.syncPending = false; + entry.recoveryPending = false; + return; + } + + sync = entry.syncPending; + update = entry.recoveryPending; + recovery = entry.recovery; + entry.syncPending = false; + entry.recoveryPending = false; + } + + if (!sync && !update) + return; + if (sync) + transport->SyncRecovery(); + if (update) + transport->RecoveryStatus(recovery.session, recovery.serial, + recovery.active, recovery.state, recovery.error); + } +} + +void CTransportManager::EndCall(Entry& entry, + const std::shared_ptr& transport, bool drain) +{ + for (;;) + { + if (drain) + DrainRecovery(entry, transport); + + CSRWExclusiveLock entryLock(entry.lock); + if (drain && (entry.syncPending || entry.recoveryPending)) + continue; + + entry.call = Call::IDLE; + entry.callOwner = 0; + SetEvent(entry.idleEvent); + return; + } } bool CTransportManager::Add(BackendId id, const char * name, bool required, bool primary, CreateFn create) { - if (!id || !name || !create || (primary && m_primary)) + CSRWExclusiveLock managerLock(m_lock); + if (!m_phaseIdle || !m_stoppedEvent || m_phase != Phase::IDLE || + m_started || m_stopping || + m_stopped || !id || !name || !create || + m_entryCount == FRAME_MAX_SINKS || (primary && m_primary)) return false; - for (const auto& current : m_entries) - if (current->id == id) + for (unsigned i = 0; i < m_entryCount; ++i) + if (m_entries[i]->id == id) return false; std::unique_ptr entry(new (std::nothrow) Entry); - if (!entry) + if (!entry || !entry->idleEvent) return false; entry->id = id; @@ -94,7 +262,7 @@ bool CTransportManager::Add(BackendId id, const char * name, bool required, entry->create = create; Entry * raw = entry.get(); - m_entries.push_back(std::move(entry)); + m_entries[m_entryCount++] = std::move(entry); if (primary) m_primary = raw; return true; @@ -102,166 +270,242 @@ bool CTransportManager::Add(BackendId id, const char * name, bool required, ITransport::OpenResult CTransportManager::OpenEntry(Entry& entry) { - if (!entry.transport) - entry.transport = entry.create(); - if (!entry.transport) + std::shared_ptr transport; + CreateFn create = nullptr; { + CSRWSharedLock entryLock(entry.lock); + transport = entry.transport; + create = entry.create; + } + + if (!transport) + { + std::unique_ptr created = create(); + transport.reset(created.release()); + CSRWExclusiveLock entryLock(entry.lock); + entry.transport = transport; + } + + if (!transport) + { + CSRWExclusiveLock entryLock(entry.lock); entry.state = State::FAILED; return OpenResult::FAILURE; } - const OpenResult result = entry.transport->Open(); + const OpenResult result = transport->Open(); switch (result) { case OpenResult::SUCCESS: + { + CSRWExclusiveLock entryLock(entry.lock); entry.state = State::OPEN; break; + } case OpenResult::RETRY: ScheduleRetry(entry); break; case OpenResult::FAILURE: + { + CSRWExclusiveLock entryLock(entry.lock); entry.state = State::FAILED; break; + } } return result; } bool CTransportManager::InitializeEntry(Entry& entry) { - if (entry.state == State::INITIALIZED || entry.state == State::READY) - return true; - if (entry.state != State::OPEN || !entry.transport->Initialize()) + std::shared_ptr transport; { + CSRWSharedLock entryLock(entry.lock); + if (entry.state == State::INITIALIZED || entry.state == State::READY) + return true; + if (entry.state != State::OPEN) + return false; + transport = entry.transport; + } + + if (!transport || !transport->Initialize()) + { + CSRWExclusiveLock entryLock(entry.lock); entry.state = State::FAILED; return false; } - if (!m_control.Add(entry.id, entry.epoch, entry.transport->Control())) + const FrameMemoryLimits limits = transport->GetMemoryLimits(); { - entry.state = State::FAILED; - return false; + CSRWExclusiveLock entryLock(entry.lock); + entry.limits = limits; + entry.limitsValid = true; + entry.state = State::INITIALIZED; } - - entry.controlAdded = true; - if (entry.primary && - !m_frames.Bind(entry.id, entry.epoch, entry.transport->FrameSink())) - { - m_control.Remove(entry.id, entry.epoch); - entry.controlAdded = false; - entry.state = State::FAILED; - return false; - } - - entry.frameAdded = entry.primary; - entry.state = State::INITIALIZED; return true; } -bool CTransportManager::SetupEntry(Entry& entry) +bool CTransportManager::AddServices(Entry& entry) { - if (entry.state == State::READY) - return true; - if (entry.state != State::INITIALIZED || - !entry.transport->Setup(m_alignment)) + std::shared_ptr transport; + BackendId id = 0; + uint32_t epoch = 0; + bool primary = false; + bool controlAdded = false; + bool frameAdded = false; { + CSRWSharedLock entryLock(entry.lock); + transport = entry.transport; + id = entry.id; + epoch = entry.epoch; + primary = entry.primary; + controlAdded = entry.controlAdded; + frameAdded = entry.frameAdded; + } + + if (!transport) + return false; + + if (!controlAdded) + { + if (!m_control.Add(id, epoch, transport->Control())) + return false; + controlAdded = true; + CSRWExclusiveLock entryLock(entry.lock); + entry.controlAdded = true; + } + + if (!frameAdded && + m_frames.Bind(id, epoch, primary, transport->FrameSink())) + { + CSRWExclusiveLock entryLock(entry.lock); + entry.frameAdded = true; + return true; + } + + if (frameAdded) + return true; + + m_control.Remove(id, epoch); + { + CSRWExclusiveLock entryLock(entry.lock); + entry.controlAdded = false; + } + return false; +} + +bool CTransportManager::SetupEntry(Entry& entry, size_t alignment) +{ + std::shared_ptr transport; + { + CSRWSharedLock entryLock(entry.lock); + if (entry.state == State::READY) + return true; + if (entry.state != State::INITIALIZED) + return false; + transport = entry.transport; + } + + bool setupDone = false; + { + CSRWSharedLock entryLock(entry.lock); + setupDone = entry.setupDone; + } + + if (!transport || (!setupDone && !transport->Setup(alignment))) + { + CSRWExclusiveLock entryLock(entry.lock); entry.state = State::FAILED; return false; } + { + CSRWExclusiveLock entryLock(entry.lock); + entry.setupDone = true; + } + if (!AddServices(entry)) + { + CSRWExclusiveLock entryLock(entry.lock); + entry.state = State::INITIALIZED; + return false; + } + + CSRWExclusiveLock entryLock(entry.lock); entry.state = State::READY; return true; } void CTransportManager::ScheduleRetry(Entry& entry) { + CSRWExclusiveLock entryLock(entry.lock); entry.state = State::RETRY; entry.retryAt = GetTickCount64() + RETRY_DELAY_MS; } -ITransport::OpenResult CTransportManager::Open() +void CTransportManager::RemoveServices(Entry& entry) { - if (!m_primary) - return OpenResult::FAILURE; - - OpenResult aggregate = OpenResult::SUCCESS; - for (const auto& current : m_entries) + BackendId id = 0; + uint32_t epoch = 0; + bool frameAdded = false; + bool controlAdded = false; { - Entry& entry = *current; - if (entry.state == State::OPEN || - entry.state == State::INITIALIZED || entry.state == State::READY) - continue; - - const OpenResult result = OpenEntry(entry); - if (!entry.required || result == OpenResult::SUCCESS) - continue; - if (result == OpenResult::FAILURE) - return OpenResult::FAILURE; - aggregate = OpenResult::RETRY; - } - return aggregate; -} - -bool CTransportManager::Initialize() -{ - for (const auto& current : m_entries) - { - Entry& entry = *current; - if (entry.state != State::OPEN) - continue; - if (!InitializeEntry(entry)) - { - if (entry.required) - return false; - ScheduleRetry(entry); - } + CSRWExclusiveLock entryLock(entry.lock); + id = entry.id; + epoch = entry.epoch; + frameAdded = entry.frameAdded; + controlAdded = entry.controlAdded; + entry.frameAdded = false; + entry.controlAdded = false; } - m_initialized = true; - return m_primary && - (m_primary->state == State::INITIALIZED || - m_primary->state == State::READY); + if (frameAdded) + m_frames.Unbind(id, epoch); + if (controlAdded) + m_control.Remove(id, epoch); } -bool CTransportManager::Setup(size_t alignment) +void CTransportManager::RetryEntry(Entry& entry, uint64_t now, + bool initialized, bool setup, size_t alignment) { - if (!m_initialized || !m_primary) - return false; - - m_alignment = alignment; - if (!SetupEntry(*m_primary)) - return false; - - m_setup = true; - return true; -} - -void CTransportManager::RetryEntry(Entry& entry, uint64_t now) -{ - if (entry.state != State::RETRY || now < entry.retryAt) - return; - - if (entry.primary && m_exposed) - return; + std::shared_ptr transport; + { + CSRWSharedLock entryLock(entry.lock); + if (entry.state != State::RETRY || now < entry.retryAt || + (entry.primary && entry.exposed)) + return; + transport = entry.transport; + } RemoveServices(entry); - if (entry.transport) - entry.transport->Stop(); - entry.transport.reset(); - ++entry.epoch; - if (!entry.epoch) + if (transport) + transport->Stop(); + + { + CSRWExclusiveLock entryLock(entry.lock); + entry.transport.reset(); + entry.limits = FrameMemoryLimits {}; + entry.limitsValid = false; + entry.directMemory = DirectFrameBufferMemory {}; + entry.directMemoryValid = false; + entry.setupDone = false; ++entry.epoch; + if (!entry.epoch) + ++entry.epoch; + } if (OpenEntry(entry) != OpenResult::SUCCESS) return; - if (m_initialized && !InitializeEntry(entry)) + if (initialized && !InitializeEntry(entry)) { ScheduleRetry(entry); return; } - if (m_setup && entry.primary && !SetupEntry(entry)) + if (setup && !SetupEntry(entry, alignment)) + { + RemoveServices(entry); ScheduleRetry(entry); + } } void CTransportManager::HandleProcessResult( @@ -270,109 +514,579 @@ void CTransportManager::HandleProcessResult( if (result == ProcessResult::OK) return; - if (entry.primary && m_exposed) + bool exposed = false; + bool primary = false; + bool required = false; + const char * name = nullptr; { - DEBUG_WARN("Transport %s requested a restart while its frame interfaces " - "are active", entry.name); - return; + CSRWSharedLock entryLock(entry.lock); + exposed = entry.exposed; + primary = entry.primary; + required = entry.required; + name = entry.name; } - if (result == ProcessResult::RETRY || !entry.required) + if (primary && exposed) { - RemoveServices(entry); - ScheduleRetry(entry); + (void)name; + DEBUG_WARN("Transport %s requested a restart while its frame interfaces " + "are active", name); return; } RemoveServices(entry); + if (result == ProcessResult::RETRY || !required) + { + ScheduleRetry(entry); + return; + } + + CSRWExclusiveLock entryLock(entry.lock); entry.state = State::FAILED; } +void CTransportManager::Expose(Entry& entry) +{ + CSRWExclusiveLock entryLock(entry.lock); + entry.exposed = true; + entry.exposedTransport = entry.transport; +} + +ITransport::OpenResult CTransportManager::Open() +{ + if (!BeginPhase(Phase::OPEN, true)) + return OpenResult::FAILURE; + + { + CSRWExclusiveLock managerLock(m_lock); + m_started = true; + } + + Entry * entries[FRAME_MAX_SINKS] = {}; + const unsigned count = Entries(entries); + OpenResult aggregate = Primary() ? + OpenResult::SUCCESS : OpenResult::FAILURE; + for (unsigned i = 0; i < count && aggregate != OpenResult::FAILURE; ++i) + { + Entry& entry = *entries[i]; + if (!BeginCall(entry, Call::LIFECYCLE, true)) + { + if (entry.required) + aggregate = OpenResult::FAILURE; + continue; + } + + bool alreadyOpen = false; + std::shared_ptr transport; + { + CSRWSharedLock entryLock(entry.lock); + alreadyOpen = entry.state == State::OPEN || + entry.state == State::INITIALIZED || entry.state == State::READY; + transport = entry.transport; + } + + const OpenResult result = alreadyOpen ? + OpenResult::SUCCESS : OpenEntry(entry); + { + CSRWSharedLock entryLock(entry.lock); + transport = entry.transport; + } + EndCall(entry, transport); + + if (entry.required && result != OpenResult::SUCCESS) + aggregate = result; + } + + EndPhase(); + return aggregate; +} + +bool CTransportManager::Initialize() +{ + if (!BeginPhase(Phase::INITIALIZE, true)) + return false; + + Entry * entries[FRAME_MAX_SINKS] = {}; + const unsigned count = Entries(entries); + bool success = true; + for (unsigned i = 0; i < count && success; ++i) + { + Entry& entry = *entries[i]; + if (!BeginCall(entry, Call::LIFECYCLE, true)) + { + if (entry.required) + success = false; + continue; + } + + State state; + std::shared_ptr transport; + { + CSRWSharedLock entryLock(entry.lock); + state = entry.state; + transport = entry.transport; + } + + if (state == State::OPEN && !InitializeEntry(entry)) + { + if (entry.required) + success = false; + else + ScheduleRetry(entry); + } + { + CSRWSharedLock entryLock(entry.lock); + transport = entry.transport; + } + EndCall(entry, transport); + } + + Entry * primary = Primary(); + if (success && primary) + { + CSRWSharedLock entryLock(primary->lock); + success = primary->state == State::INITIALIZED || + primary->state == State::READY; + } + else + success = false; + + if (success) + { + CSRWExclusiveLock managerLock(m_lock); + m_initialized = true; + } + EndPhase(); + return success; +} + +bool CTransportManager::Setup(size_t alignment) +{ + if (!BeginPhase(Phase::SETUP, true)) + return false; + + bool initialized = false; + { + CSRWExclusiveLock managerLock(m_lock); + initialized = m_initialized && m_primary; + if (initialized) + m_alignment = alignment; + } + + Entry * entries[FRAME_MAX_SINKS] = {}; + const unsigned count = Entries(entries); + bool success = initialized; + for (unsigned i = 0; i < count && success; ++i) + { + Entry& entry = *entries[i]; + if (!BeginCall(entry, Call::LIFECYCLE, true)) + { + if (entry.required) + success = false; + continue; + } + + State state; + std::shared_ptr transport; + { + CSRWSharedLock entryLock(entry.lock); + state = entry.state; + transport = entry.transport; + } + + if (state == State::INITIALIZED && !SetupEntry(entry, alignment)) + { + RemoveServices(entry); + if (entry.required) + success = false; + else + ScheduleRetry(entry); + } + { + CSRWSharedLock entryLock(entry.lock); + transport = entry.transport; + } + EndCall(entry, transport); + } + + for (unsigned i = 0; i < count && success; ++i) + { + CSRWSharedLock entryLock(entries[i]->lock); + if (entries[i]->required && entries[i]->state != State::READY) + success = false; + } + + if (success) + { + CSRWExclusiveLock managerLock(m_lock); + m_setup = true; + } + EndPhase(); + return success; +} + ITransport::ProcessResult CTransportManager::Process( ITransportEvents& events) { - const uint64_t now = GetTickCount64(); - for (const auto& current : m_entries) + if (!BeginPhase(Phase::PROCESS, false)) { - Entry& entry = *current; - RetryEntry(entry, now); - if (entry.state != State::INITIALIZED && entry.state != State::READY) + CSRWSharedLock managerLock(m_lock); + return m_stopping || m_stopped ? + ProcessResult::FAILURE : ProcessResult::OK; + } + + bool initialized = false; + bool setup = false; + size_t alignment = 0; + { + CSRWSharedLock managerLock(m_lock); + initialized = m_initialized; + setup = m_setup; + alignment = m_alignment; + } + + const uint64_t now = GetTickCount64(); + Entry * entries[FRAME_MAX_SINKS] = {}; + const unsigned count = Entries(entries); + for (unsigned i = 0; i < count; ++i) + { + Entry& entry = *entries[i]; + if (!BeginCall(entry, Call::PROCESS, false)) continue; - CSourceEvents sourceEvents(entry.id, entry.epoch, events); - const ProcessResult result = entry.transport->Process(sourceEvents); + RetryEntry(entry, now, initialized, setup, alignment); + + std::shared_ptr transport; + BackendId id = 0; + uint32_t epoch = 0; + bool process = false; + { + CSRWSharedLock entryLock(entry.lock); + transport = entry.transport; + id = entry.id; + epoch = entry.epoch; + process = transport && + (entry.state == State::INITIALIZED || entry.state == State::READY); + } + + if (!process) + { + EndCall(entry, transport); + continue; + } + + bool setupDone = false; + { + CSRWSharedLock entryLock(entry.lock); + setupDone = entry.setupDone; + } + if (setup && !setupDone) + { + DrainRecovery(entry, transport); + if (!transport || !transport->Setup(alignment)) + { + HandleProcessResult(entry, ProcessResult::FAILURE); + EndCall(entry, transport); + continue; + } + CSRWExclusiveLock entryLock(entry.lock); + entry.setupDone = true; + } + + DrainRecovery(entry, transport); + CSourceEvents sourceEvents(id, epoch, events); + const ProcessResult result = transport->Process(sourceEvents); + DrainRecovery(entry, transport); HandleProcessResult(entry, result); + EndCall(entry, transport); } + + EndPhase(); return ProcessResult::OK; } void CTransportManager::Stop() { - for (auto current = m_entries.rbegin(); current != m_entries.rend(); - ++current) + const DWORD thread = GetCurrentThreadId(); + bool wait = false; { - Entry& entry = **current; + CSRWExclusiveLock managerLock(m_lock); + if (m_stopped) + return; + if (m_stopping) + wait = true; + else + { + if (m_phase != Phase::IDLE && m_phaseOwner == thread) + return; + m_stopping = true; + ResetEvent(m_stoppedEvent); + } + } + + if (wait) + { + WaitForSingleObject(m_stoppedEvent, INFINITE); + return; + } + + if (!BeginPhase(Phase::STOP, true, true)) + return; + + Entry * entries[FRAME_MAX_SINKS] = {}; + const unsigned count = Entries(entries); + bool failed = false; + for (unsigned i = 0; i < count; ++i) + { + CSRWExclusiveLock entryLock(entries[i]->lock); + entries[i]->stopRequested = true; + entries[i]->syncPending = false; + entries[i]->recoveryPending = false; + } + + for (unsigned i = 0; i < count; ++i) + { + for (;;) + { + HANDLE idleEvent = nullptr; + { + CSRWSharedLock entryLock(entries[i]->lock); + if (entries[i]->call == Call::IDLE) + break; + idleEvent = entries[i]->idleEvent; + } + if (!idleEvent || + WaitForSingleObject(idleEvent, INFINITE) != WAIT_OBJECT_0) + { + failed = true; + break; + } + } + } + + if (failed) + { + for (unsigned i = 0; i < count; ++i) + { + CSRWExclusiveLock entryLock(entries[i]->lock); + entries[i]->stopRequested = false; + } + CSRWExclusiveLock managerLock(m_lock); + m_stopping = false; + m_phase = Phase::IDLE; + m_phaseOwner = 0; + SetEvent(m_phaseIdle); + SetEvent(m_stoppedEvent); + return; + } + + for (unsigned i = count; i > 0; --i) + { + Entry& entry = *entries[i - 1]; + std::shared_ptr transport; + State state; + { + CSRWSharedLock entryLock(entry.lock); + transport = entry.transport; + state = entry.state; + } + RemoveServices(entry); - if (entry.transport && entry.state != State::STOPPED) - entry.transport->Stop(); - entry.state = State::STOPPED; - } -} - -void CTransportManager::RemoveServices(Entry& entry) -{ - if (entry.frameAdded) - { - m_frames.Unbind(entry.id, entry.epoch); - entry.frameAdded = false; + if (transport && state != State::STOPPED) + transport->Stop(); + { + CSRWExclusiveLock entryLock(entry.lock); + entry.state = State::STOPPED; + } } - if (entry.controlAdded) { - m_control.Remove(entry.id, entry.epoch); - entry.controlAdded = false; + CSRWExclusiveLock managerLock(m_lock); + m_initialized = false; + m_setup = false; + m_stopped = true; + m_phase = Phase::IDLE; + m_phaseOwner = 0; + SetEvent(m_phaseIdle); + SetEvent(m_stoppedEvent); } } void CTransportManager::SyncRecovery() { - for (const auto& current : m_entries) - if (current->transport && - (current->state == State::INITIALIZED || - current->state == State::READY)) - current->transport->SyncRecovery(); + { + CSRWSharedLock managerLock(m_lock); + if (m_stopping || m_stopped) + return; + } + + Entry * entries[FRAME_MAX_SINKS] = {}; + const unsigned count = Entries(entries); + for (unsigned i = 0; i < count; ++i) + { + Entry& entry = *entries[i]; + std::shared_ptr transport; + bool call = false; + { + CSRWExclusiveLock entryLock(entry.lock); + if (entry.stopRequested || !entry.transport || + (entry.state != State::INITIALIZED && entry.state != State::READY)) + continue; + + if (entry.call != Call::IDLE) + { + entry.syncPending = true; + continue; + } + + entry.call = Call::RECOVERY; + entry.callOwner = GetCurrentThreadId(); + ResetEvent(entry.idleEvent); + transport = entry.transport; + call = true; + } + + if (call) + transport->SyncRecovery(); + EndCall(entry, transport); + } } void CTransportManager::RecoveryStatus(uint64_t session, uint32_t serial, bool active, Recovery state, uint32_t error) { - for (const auto& current : m_entries) - if (current->transport && - (current->state == State::INITIALIZED || - current->state == State::READY)) - current->transport->RecoveryStatus( - session, serial, active, state, error); -} + { + CSRWSharedLock managerLock(m_lock); + if (m_stopping || m_stopped) + return; + } -ITransport& CTransportManager::Primary() -{ - m_exposed = true; - return *m_primary->transport; + Entry * entries[FRAME_MAX_SINKS] = {}; + const unsigned count = Entries(entries); + for (unsigned i = 0; i < count; ++i) + { + Entry& entry = *entries[i]; + std::shared_ptr transport; + bool call = false; + { + CSRWExclusiveLock entryLock(entry.lock); + if (entry.stopRequested || !entry.transport || + (entry.state != State::INITIALIZED && entry.state != State::READY)) + continue; + + if (entry.call != Call::IDLE) + { + entry.recovery.session = session; + entry.recovery.serial = serial; + entry.recovery.active = active; + entry.recovery.state = state; + entry.recovery.error = error; + entry.recoveryPending = true; + continue; + } + + entry.call = Call::RECOVERY; + entry.callOwner = GetCurrentThreadId(); + ResetEvent(entry.idleEvent); + transport = entry.transport; + call = true; + } + + if (call) + transport->RecoveryStatus(session, serial, active, state, error); + EndCall(entry, transport); + } } FrameMemoryLimits CTransportManager::GetMemoryLimits() const { - return m_primary->transport->GetMemoryLimits(); + { + CSRWSharedLock managerLock(m_lock); + if (m_stopping || m_stopped) + return FrameMemoryLimits {}; + } + + Entry * primary = Primary(); + if (!primary) + return FrameMemoryLimits {}; + + CSRWSharedLock entryLock(primary->lock); + return primary->limitsValid ? primary->limits : FrameMemoryLimits {}; } DirectFrameBufferMemory CTransportManager::GetDirectMemory() const { - return m_primary->transport->GetDirectMemory(); + CTransportManager * manager = const_cast(this); + { + CSRWSharedLock managerLock(m_lock); + if (m_stopping || m_stopped) + return DirectFrameBufferMemory {}; + } + + Entry * primary = manager->Primary(); + if (!primary) + return DirectFrameBufferMemory {}; + + { + CSRWSharedLock entryLock(primary->lock); + if (primary->directMemoryValid) + return primary->directMemory; + } + + if (!manager->BeginPhase(Phase::ACCESS, true)) + return DirectFrameBufferMemory {}; + + { + CSRWSharedLock entryLock(primary->lock); + if (primary->directMemoryValid) + { + const DirectFrameBufferMemory memory = primary->directMemory; + manager->EndPhase(); + return memory; + } + } + + if (!manager->BeginCall(*primary, Call::ACCESS, true)) + { + manager->EndPhase(); + return DirectFrameBufferMemory {}; + } + + std::shared_ptr transport; + State state; + { + CSRWSharedLock entryLock(primary->lock); + transport = primary->transport; + state = primary->state; + } + + DirectFrameBufferMemory memory; + if (transport && state != State::FAILED && state != State::STOPPED) + memory = transport->GetDirectMemory(); + { + CSRWExclusiveLock entryLock(primary->lock); + primary->directMemory = memory; + primary->directMemoryValid = true; + primary->exposed = true; + primary->exposedTransport = transport; + } + + manager->EndCall(*primary, transport); + manager->EndPhase(); + return memory; } IFrameTransport& CTransportManager::Frames() { - m_exposed = true; + Entry * primary = Primary(); + if (!primary) + return m_frames; + + if (BeginPhase(Phase::ACCESS, true)) + { + Expose(*primary); + EndPhase(); + } return m_frames; } @@ -383,5 +1097,23 @@ IControlTransport& CTransportManager::Control() IInputTransport * CTransportManager::Input() { - return Primary().Input(); + Entry * primary = Primary(); + if (!primary || !BeginPhase(Phase::ACCESS, true)) + return nullptr; + if (!BeginCall(*primary, Call::ACCESS, true)) + { + EndPhase(); + return nullptr; + } + + std::shared_ptr transport; + { + CSRWSharedLock entryLock(primary->lock); + transport = primary->transport; + } + Expose(*primary); + IInputTransport * input = transport ? transport->Input() : nullptr; + EndCall(*primary, transport); + EndPhase(); + return input; } diff --git a/idd/LGIdd/transport/CTransportManager.h b/idd/LGIdd/transport/CTransportManager.h index b0ea3e5d..e3dcc406 100644 --- a/idd/LGIdd/transport/CTransportManager.h +++ b/idd/LGIdd/transport/CTransportManager.h @@ -25,7 +25,6 @@ #include "transport/ITransport.h" #include -#include class CTransportManager final { @@ -36,6 +35,19 @@ public: using Recovery = ITransport::Recovery; private: + mutable CSRWLock m_lock; + + enum class Phase + { + IDLE, + OPEN, + INITIALIZE, + SETUP, + PROCESS, + ACCESS, + STOP, + }; + enum class State { CLOSED, @@ -47,41 +59,97 @@ private: STOPPED, }; + enum class Call + { + IDLE, + LIFECYCLE, + PROCESS, + RECOVERY, + ACCESS, + }; + + struct RecoveryUpdate + { + uint64_t session = 0; + uint32_t serial = 0; + bool active = false; + Recovery state = Recovery::FAILED; + uint32_t error = 0; + }; + struct Entry { - BackendId id; - const char * name; - bool required; - bool primary; - CreateFn create; - std::unique_ptr transport; + Entry(); + ~Entry(); + + mutable CSRWLock lock; + HANDLE idleEvent = nullptr; + Call call = Call::IDLE; + DWORD callOwner = 0; + bool stopRequested = false; + BackendId id = 0; + const char * name = nullptr; + bool required = false; + bool primary = false; + CreateFn create = nullptr; + std::shared_ptr transport; State state = State::CLOSED; uint32_t epoch = 1; uint64_t retryAt = 0; bool controlAdded = false; bool frameAdded = false; + bool exposed = false; + bool setupDone = false; + bool syncPending = false; + bool recoveryPending = false; + RecoveryUpdate recovery; + FrameMemoryLimits limits; + bool limitsValid = false; + DirectFrameBufferMemory directMemory; + bool directMemoryValid = false; + std::shared_ptr exposedTransport; }; - std::vector> m_entries; + std::unique_ptr m_entries[FRAME_MAX_SINKS]; + unsigned m_entryCount = 0; CControlHub m_control; CFrameHub m_frames; Entry * m_primary = nullptr; bool m_initialized = false; bool m_setup = false; - bool m_exposed = false; size_t m_alignment = 0; + Phase m_phase = Phase::IDLE; + DWORD m_phaseOwner = 0; + HANDLE m_phaseIdle = nullptr; + HANDLE m_stoppedEvent = nullptr; + bool m_started = false; + bool m_stopping = false; + bool m_stopped = false; + + unsigned Entries(Entry * entries[FRAME_MAX_SINKS]) const; + Entry * Primary() const; + + bool BeginPhase(Phase phase, bool wait, bool stopping = false); + void EndPhase(); + bool BeginCall(Entry& entry, Call call, bool wait); + void EndCall(Entry& entry, + const std::shared_ptr& transport, bool drain = true); + void DrainRecovery(Entry& entry, + const std::shared_ptr& transport); OpenResult OpenEntry(Entry& entry); bool InitializeEntry(Entry& entry); - bool SetupEntry(Entry& entry); - void RetryEntry(Entry& entry, uint64_t now); + bool SetupEntry(Entry& entry, size_t alignment); + bool AddServices(Entry& entry); + void RetryEntry(Entry& entry, uint64_t now, bool initialized, + bool setup, size_t alignment); void HandleProcessResult(Entry& entry, ProcessResult result); void ScheduleRetry(Entry& entry); void RemoveServices(Entry& entry); - ITransport& Primary(); + void Expose(Entry& entry); public: - CTransportManager() = default; + CTransportManager(); ~CTransportManager(); CTransportManager(const CTransportManager&) = delete; diff --git a/idd/LGIdd/transport/IFrameSink.h b/idd/LGIdd/transport/IFrameSink.h index 4f53be75..d57a7c8d 100644 --- a/idd/LGIdd/transport/IFrameSink.h +++ b/idd/LGIdd/transport/IFrameSink.h @@ -81,5 +81,6 @@ public: uint64_t now, bool periodic) = 0; virtual void FrameSuperseded() = 0; virtual HANDLE GetFrameScheduleEvent() const = 0; + virtual void SetFrameScheduleEvent(HANDLE event) = 0; virtual void TryRecordFrameTiming(uint64_t duration) = 0; }; diff --git a/idd/LGIdd/transport/IFrameTransport.h b/idd/LGIdd/transport/IFrameTransport.h index 752a586e..19e88fd3 100644 --- a/idd/LGIdd/transport/IFrameTransport.h +++ b/idd/LGIdd/transport/IFrameTransport.h @@ -28,60 +28,69 @@ #include #include +struct FramePlanTarget +{ + uint32_t sink = 0; + uint32_t backend = 0; + uint32_t epoch = 0; + CFrameScheduler::Schedule schedule = {}; + CFrameScheduler::Schedule commitSchedule = {}; + bool periodic = false; + bool primary = false; +}; + +struct FramePlan +{ + FramePlanTarget targets[FRAME_MAX_SINKS] = {}; + unsigned count = 0; + uint64_t nextWake = 0; + bool progressed = false; +}; + class IFrameTransport { public: virtual ~IFrameTransport() = default; virtual size_t GetMaxFrameSize() const = 0; + virtual uint64_t NextContentSerial() = 0; + virtual void FrameProductReady(uint64_t contentSerial) = 0; + virtual bool NeedsFrame() const = 0; + virtual bool GetFramePlan( + uint64_t now, bool productReady, FramePlan& plan) = 0; + virtual bool GetImmediateFramePlan( + uint64_t now, FramePlan& plan) = 0; + virtual void MissFramePlan(const FramePlan& plan, uint64_t now) = 0; - virtual bool FrameBufferAvailable( - const CFrameScheduler::Schedule& schedule, - bool allowReadyReplacement = true) = 0; - virtual bool HasPublishedFrame() const = 0; - virtual void ProcessDeliveries() = 0; - virtual bool GetPendingDeliveryTarget( - uint64_t now, uint64_t& target) = 0; - virtual bool RetryPendingDelivery(uint64_t now, bool& retry) = 0; - virtual PreparedFrameBuffer PrepareFrameBuffer(unsigned pitch, + virtual bool PrepareFrameBatch(const FramePlan& plan, + uint64_t contentSerial, unsigned pitch, size_t frameSize, const D12FrameFormat& srcFormat, const D12FrameFormat& dstFormat, const RECT * dirtyRects, unsigned nbDirtyRects, - const CFrameScheduler::Schedule& schedule, - bool allowReadyReplacement = true) = 0; - virtual bool PublishFrameBuffer(const FrameToken& token, - const CFrameScheduler::Schedule& schedule, - bool& deliveredToOwner) = 0; - virtual bool RepublishFrameBuffer( - const CFrameScheduler::Schedule& schedule) = 0; - virtual bool TryFrameSubmitted(const FrameToken& token, - const CFrameScheduler::Schedule& schedule) = 0; - virtual void CommitFrameBuffer(const FrameToken& token, - const CFrameScheduler::Schedule& schedule, bool periodic, - bool deliveredToOwner) = 0; - virtual void AbortFrameBuffer(const FrameToken& token) = 0; - virtual void FailFrameBuffer(const FrameToken& token) = 0; - virtual void CompleteFrameBuffer( - const FrameToken& token, bool succeeded) = 0; - virtual void SetFrameTiming(const FrameToken& token, uint64_t captureTime, - uint64_t postProcessTime, uint64_t copyTime, uint64_t readyTime, - uint64_t holdTime, const CFrameScheduler::Schedule& schedule, + bool allowReadyReplacement, PreparedFrameBatch& batch) = 0; + virtual uint32_t PublishFrameBatch(const FrameBatchToken& token) = 0; + virtual void CommitFrameBatch(const FrameBatchToken& token) = 0; + virtual void AbortFrameBatch(const FrameBatchToken& token) = 0; + virtual void FailFrameBatch(const FrameBatchToken& token) = 0; + + virtual void WriteFrameTarget(const FrameBatchToken& token, + unsigned target, void * src, size_t offset, size_t len, + bool setWritePos) const = 0; + virtual void WriteFrameTargetRows(const FrameBatchToken& token, + unsigned target, void * src, size_t offset, size_t rowBytes, + size_t pitch, unsigned rows) const = 0; + virtual void FinalizeFrameTarget( + const FrameBatchToken& token, unsigned target) const = 0; + virtual void SetFrameTargetTiming(const FrameBatchToken& token, + unsigned target, uint64_t captureTime, uint64_t postProcessTime, + uint64_t copyTime, uint64_t readyTime, uint64_t holdTime, uint64_t completedAt) = 0; - virtual void WriteFrameBuffer(const FrameToken& token, void * src, - size_t offset, size_t len, bool setWritePos) const = 0; - virtual void WriteFrameBufferRows(const FrameToken& token, void * src, - size_t offset, size_t rowBytes, size_t pitch, - unsigned rows) const = 0; - virtual void FinalizeFrameBuffer(const FrameToken& token) const = 0; + virtual void TryRecordFrameTiming(const FrameBatchToken& token, + unsigned target, uint64_t duration) = 0; + virtual void CompleteFrameTarget(const FrameBatchToken& token, + unsigned target, bool succeeded) = 0; virtual void ObserveFrame(uint64_t now) = 0; virtual void ForceFrame() = 0; - virtual bool GetPublishTarget(uint64_t now, uint64_t& target, - CFrameScheduler::Schedule& schedule, bool& periodic, - bool& republish) = 0; - virtual void FrameMissed(const CFrameScheduler::Schedule& schedule, - uint64_t now, bool periodic) = 0; virtual void FrameSuperseded() = 0; virtual HANDLE GetFrameScheduleEvent() const = 0; - virtual void TryRecordFrameTiming( - const FrameToken& token, uint64_t duration) = 0; }; diff --git a/idd/LGIdd/transport/PreparedFrameBuffer.h b/idd/LGIdd/transport/PreparedFrameBuffer.h index a1d96fb7..29705fe1 100644 --- a/idd/LGIdd/transport/PreparedFrameBuffer.h +++ b/idd/LGIdd/transport/PreparedFrameBuffer.h @@ -20,8 +20,17 @@ #pragma once +#include #include +enum : unsigned +{ + FRAME_MAX_SINKS = 8, + FRAME_BATCHES = 3, + FRAME_SINK_BUFFERS = 3, + FRAME_BUFFER_RESOURCES = FRAME_MAX_SINKS * FRAME_SINK_BUFFERS, +}; + struct FrameToken { uint32_t sink = 0; @@ -33,16 +42,32 @@ struct FrameToken struct PreparedFrameBuffer { FrameToken token; - unsigned resourceSlot; - uint8_t * mem; - uint64_t heapOffset; - bool fullCopy; + unsigned resourceSlot = 0; + uint8_t * mem = nullptr; + uint64_t heapOffset = 0; + size_t capacity = 0; + bool direct = false; + bool fullCopy = false; +}; + +struct FrameBatchToken +{ + uint32_t slot = 0; + uint64_t serial = 0; +}; + +struct PreparedFrameBatch +{ + FrameBatchToken token; + PreparedFrameBuffer targets[FRAME_MAX_SINKS] = {}; + unsigned count = 0; }; struct SinkTarget { - unsigned slot; - uint8_t * mem; - uint64_t heapOffset; - bool fullCopy; + unsigned slot = 0; + uint8_t * mem = nullptr; + uint64_t heapOffset = 0; + size_t capacity = 0; + bool fullCopy = false; }; diff --git a/idd/LGIdd/transport/lgmp/CLGMPFrameTransport.cpp b/idd/LGIdd/transport/lgmp/CLGMPFrameTransport.cpp index 19737bbb..a81d148d 100644 --- a/idd/LGIdd/transport/lgmp/CLGMPFrameTransport.cpp +++ b/idd/LGIdd/transport/lgmp/CLGMPFrameTransport.cpp @@ -917,7 +917,10 @@ SinkTarget CLGMPFrameTransport::PrepareFrameBuffer( result.mem = fb->data; result.heapOffset = reinterpret_cast(fb->data) - reinterpret_cast(m_ivshmem.GetMem()); + result.capacity = m_maxFrameSize; result.fullCopy = fullCopy; + if (fullCopy) + fi->damageRectsCount = 0; return result; } diff --git a/idd/LGIdd/transport/lgmp/CLGMPFrameTransport.h b/idd/LGIdd/transport/lgmp/CLGMPFrameTransport.h index 2672cb1c..acbad46e 100644 --- a/idd/LGIdd/transport/lgmp/CLGMPFrameTransport.h +++ b/idd/LGIdd/transport/lgmp/CLGMPFrameTransport.h @@ -214,5 +214,9 @@ public: { return m_frameScheduler.GetWakeEvent(); } + void SetFrameScheduleEvent(HANDLE event) override + { + m_frameScheduler.SetSharedWakeEvent(event); + } void TryRecordFrameTiming(uint64_t duration) override; };