diff --git a/idd/LGIdd/CD3D12CommandQueue.cpp b/idd/LGIdd/CD3D12CommandQueue.cpp index 2ea294cc..556b7e99 100644 --- a/idd/LGIdd/CD3D12CommandQueue.cpp +++ b/idd/LGIdd/CD3D12CommandQueue.cpp @@ -33,22 +33,326 @@ static uint64_t TicksToNanoseconds(uint64_t ticks, uint64_t frequency) return ScaleTicks(ticks, 1000000000ULL, frequency); } -bool CD3D12CommandQueue::InitTiming(ID3D12Device3 * device, - D3D12_COMMAND_LIST_TYPE type) +static bool ConvertGPUTimestamp(UINT64 timestamp, UINT64 timestampFrequency, + UINT64 calibrationGPU, UINT64 calibrationCPU, UINT64 qpcFrequency, + uint64_t& result) { - if (type != D3D12_COMMAND_LIST_TYPE_COPY) + UINT64 cpuTimestamp; + if (timestamp < calibrationGPU) + { + const UINT64 delta = ScaleTicks( + calibrationGPU - timestamp, qpcFrequency, timestampFrequency); + if (delta > calibrationCPU) + return false; + cpuTimestamp = calibrationCPU - delta; + } + else + { + const UINT64 delta = ScaleTicks( + timestamp - calibrationGPU, qpcFrequency, timestampFrequency); + if (UINT64_MAX - calibrationCPU < delta) + return false; + cpuTimestamp = calibrationCPU + delta; + } + + result = TicksToNanoseconds(cpuTimestamp, qpcFrequency); + return true; +} + +bool CD3D12CommandSlot::Init(ID3D12Device3 * device, + CD3D12CommandQueue * queue, const WCHAR * name, UINT queryBase, + ULONG waitFlags) +{ + m_queue = queue; + m_name = name; + m_queryBase = queryBase; + + HRESULT hr = device->CreateCommandAllocator( + queue->m_queue->GetDesc().Type, IID_PPV_ARGS(&m_allocator)); + if (FAILED(hr)) + { + DEBUG_ERROR_HR(hr, "Failed to create the CommandAllocator (%ls)", name); + return false; + } + + hr = device->CreateCommandList(0, queue->m_queue->GetDesc().Type, + m_allocator.Get(), NULL, IID_PPV_ARGS(&m_gfxList)); + if (FAILED(hr)) + { + DEBUG_ERROR_HR(hr, "Failed to create the Graphics CommandList (%ls)", name); + return false; + } + m_gfxList->SetName(name); + + m_cmdList = m_gfxList; + if (!m_cmdList) + { + DEBUG_ERROR("Failed to get the CommandList (%ls)", name); + return false; + } + + m_event.Attach(CreateEvent(NULL, FALSE, FALSE, NULL)); + if (!m_event.Get()) + { + DEBUG_ERROR_HR(GetLastError(), + "Failed to create the completion event (%ls)", name); + return false; + } + + if (!RegisterWaitForSingleObject( + &m_waitHandle, + m_event.Get(), + [](PVOID param, BOOLEAN timeout){ + static_cast(param)->OnCompletion(!!timeout); + }, + this, + INFINITE, + waitFlags)) + { + DEBUG_ERROR_HR(GetLastError(), + "Failed to register the completion wait (%ls)", name); + m_waitHandle = INVALID_HANDLE_VALUE; + return false; + } + + DEBUG_INFO("Created CD3D12CommandSlot(%ls)", name); + return true; +} + +void CD3D12CommandSlot::DeInit() +{ + if (m_waitHandle != INVALID_HANDLE_VALUE) + { + if (!UnregisterWaitEx(m_waitHandle, INVALID_HANDLE_VALUE)) + DEBUG_WARN_HR(GetLastError(), + "Failed to unregister the completion wait (%ls)", m_name); + m_waitHandle = INVALID_HANDLE_VALUE; + } + + m_event.Close(); + m_cmdList.Reset(); + m_gfxList.Reset(); + m_allocator.Reset(); + m_queue = nullptr; +} + +bool CD3D12CommandSlot::Acquire() +{ + if (!m_queue || m_queue->m_failed.load(std::memory_order_acquire)) return false; + State expected = STATE_FREE; + if (!m_state.compare_exchange_strong(expected, STATE_RECORDING, + std::memory_order_acq_rel)) + return false; + + m_completionCallback = nullptr; + m_completionParams[0] = nullptr; + m_completionParams[1] = nullptr; + m_completionResult = true; + m_fenceTarget = 0; + m_fenceWaitCount = 0; + m_timingActive = false; + m_timestampFrequency = 0; + m_calibrationGPU = 0; + m_calibrationCPU = 0; + m_submitted.store(false, std::memory_order_release); + + for (UINT i = 0; i < MAX_FENCE_WAITS; ++i) + { + m_fenceWaits[i].fence.Reset(); + m_fenceWaits[i].value = 0; + } + + if (!m_needsReset) + return true; + + HRESULT hr = m_allocator->Reset(); + if (FAILED(hr)) + { + DEBUG_ERROR_HR(hr, "Failed to reset the CommandAllocator (%ls)", m_name); + m_queue->m_failed.store(true, std::memory_order_release); + m_state.store(STATE_FAILED, std::memory_order_release); + return false; + } + + hr = m_gfxList->Reset(m_allocator.Get(), NULL); + if (FAILED(hr)) + { + DEBUG_ERROR_HR(hr, "Failed to reset the CommandList (%ls)", m_name); + m_queue->m_failed.store(true, std::memory_order_release); + m_state.store(STATE_FAILED, std::memory_order_release); + return false; + } + + m_needsReset = false; + return true; +} + +void CD3D12CommandSlot::Cancel() +{ + State expected = STATE_RECORDING; + if (!m_state.compare_exchange_strong(expected, STATE_CANCELLING, + std::memory_order_acq_rel)) + { + DEBUG_ERROR("Command slot cancelled while not recording (%ls)", m_name); + return; + } + + const HRESULT hr = m_gfxList->Close(); + m_needsReset = true; + m_timingActive = false; + + for (UINT i = 0; i < m_fenceWaitCount; ++i) + { + m_fenceWaits[i].fence.Reset(); + m_fenceWaits[i].value = 0; + } + m_fenceWaitCount = 0; + + if (FAILED(hr)) + { + DEBUG_ERROR_HR(hr, "Failed to close the cancelled CommandList (%ls)", + m_name); + m_queue->m_failed.store(true, std::memory_order_release); + m_state.store(STATE_FAILED, std::memory_order_release); + return; + } + + m_completionCallback = nullptr; + m_completionParams[0] = nullptr; + m_completionParams[1] = nullptr; + m_submitted.store(false, std::memory_order_release); + m_state.store(STATE_FREE, std::memory_order_release); +} + +bool CD3D12CommandSlot::Execute() +{ + State expected = STATE_RECORDING; + if (!m_state.compare_exchange_strong(expected, STATE_SUBMITTED, + std::memory_order_acq_rel)) + { + DEBUG_ERROR("Command slot executed while not recording (%ls)", m_name); + return false; + } + + m_needsReset = true; + HRESULT hr = m_gfxList->Close(); + if (FAILED(hr)) + { + DEBUG_ERROR_HR(hr, "Failed to close the CommandList (%ls)", m_name); + m_queue->m_failed.store(true, std::memory_order_release); + m_state.store(STATE_FAILED, std::memory_order_release); + return false; + } + + if (m_queue->Submit(*this)) + return true; + + if (!m_submitted.load(std::memory_order_acquire)) + m_state.store(STATE_FREE, std::memory_order_release); + return false; +} + +bool CD3D12CommandSlot::WaitFor(ID3D12Fence * fence, UINT64 value) +{ + if (!fence || !value || + m_state.load(std::memory_order_acquire) != STATE_RECORDING) + return false; + + if (m_fenceWaitCount == MAX_FENCE_WAITS) + { + DEBUG_ERROR("Too many fence waits for CommandSlot(%ls)", m_name); + return false; + } + + FenceWait& wait = m_fenceWaits[m_fenceWaitCount++]; + wait.fence = fence; + wait.value = value; + return true; +} + +bool CD3D12CommandSlot::WaitFor(const CD3D12CommandSlot& slot) +{ + if (!slot.m_queue || !slot.m_fenceTarget) + return false; + + return WaitFor(slot.m_queue->m_fence.Get(), slot.m_fenceTarget); +} + +bool CD3D12CommandSlot::BeginTiming() +{ + m_timingActive = m_queue && m_queue->m_timingSupported; + if (!m_timingActive) + return false; + + m_gfxList->EndQuery(m_queue->m_timestampHeap.Get(), + D3D12_QUERY_TYPE_TIMESTAMP, m_queryBase); + return true; +} + +void CD3D12CommandSlot::EndTiming() +{ + if (!m_timingActive) + return; + + if (!m_queue->SnapshotTiming(*this)) + { + m_timingActive = false; + return; + } + + m_gfxList->EndQuery(m_queue->m_timestampHeap.Get(), + D3D12_QUERY_TYPE_TIMESTAMP, m_queryBase + 1); + m_gfxList->ResolveQueryData(m_queue->m_timestampHeap.Get(), + D3D12_QUERY_TYPE_TIMESTAMP, m_queryBase, 2, + m_queue->m_timestampReadback.Get(), + (UINT64)m_queryBase * sizeof(UINT64)); +} + +bool CD3D12CommandSlot::GetGPUTimes( + uint64_t& start, uint64_t& end) const +{ + return m_queue && m_queue->GetGPUTimes(*this, start, end); +} + +void CD3D12CommandSlot::OnCompletion(bool timeout) +{ + if (!m_queue || !m_submitted.load(std::memory_order_acquire)) + return; + + const UINT64 completed = m_queue->m_fence->GetCompletedValue(); + if (completed != UINT64_MAX && completed < m_fenceTarget) + return; + + State expected = STATE_SUBMITTED; + if (!m_state.compare_exchange_strong(expected, STATE_COMPLETING, + std::memory_order_acq_rel)) + return; + + m_completionResult = !timeout && completed != UINT64_MAX; + if (!m_completionResult) + m_queue->m_failed.store(true, std::memory_order_release); + + if (m_completionCallback) + m_completionCallback(this, m_completionResult, + m_completionParams[0], m_completionParams[1]); + + m_completionCallback = nullptr; + m_completionParams[0] = nullptr; + m_completionParams[1] = nullptr; + m_submitted.store(false, std::memory_order_release); + m_state.store(STATE_FREE, std::memory_order_release); +} + +bool CD3D12CommandQueue::InitTiming(ID3D12Device3 * device, UINT slotCount) +{ D3D12_FEATURE_DATA_D3D12_OPTIONS3 options = {}; HRESULT hr = device->CheckFeatureSupport( D3D12_FEATURE_D3D12_OPTIONS3, &options, sizeof(options)); if (FAILED(hr) || !options.CopyQueueTimestampQueriesSupported) return false; - hr = m_queue->GetTimestampFrequency(&m_timestampFrequency); - if (FAILED(hr) || !m_timestampFrequency) - return false; - LARGE_INTEGER qpcFrequency; if (!QueryPerformanceFrequency(&qpcFrequency)) return false; @@ -56,9 +360,10 @@ bool CD3D12CommandQueue::InitTiming(ID3D12Device3 * device, D3D12_QUERY_HEAP_DESC queryDesc = {}; queryDesc.Type = D3D12_QUERY_HEAP_TYPE_COPY_QUEUE_TIMESTAMP; - queryDesc.Count = 2; + queryDesc.Count = slotCount * 2; - hr = device->CreateQueryHeap(&queryDesc, IID_PPV_ARGS(&m_timestampHeap)); + hr = device->CreateQueryHeap(&queryDesc, + IID_PPV_ARGS(&m_timestampHeap)); if (FAILED(hr)) return false; @@ -71,7 +376,7 @@ bool CD3D12CommandQueue::InitTiming(ID3D12Device3 * device, D3D12_RESOURCE_DESC resourceDesc = {}; resourceDesc.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER; - resourceDesc.Width = sizeof(UINT64) * 2; + resourceDesc.Width = sizeof(UINT64) * queryDesc.Count; resourceDesc.Height = 1; resourceDesc.DepthOrArraySize = 1; resourceDesc.MipLevels = 1; @@ -91,7 +396,8 @@ bool CD3D12CommandQueue::InitTiming(ID3D12Device3 * device, return false; } - D3D12_RANGE readRange = { 0, sizeof(UINT64) * 2 }; + const SIZE_T timingSize = sizeof(UINT64) * queryDesc.Count; + D3D12_RANGE readRange = { 0, timingSize }; void * timestampMap = nullptr; hr = m_timestampReadback->Map(0, &readRange, ×tampMap); if (FAILED(hr)) @@ -100,55 +406,31 @@ bool CD3D12CommandQueue::InitTiming(ID3D12Device3 * device, m_timestampHeap.Reset(); return false; } - m_timestampMap = static_cast(timestampMap); - - hr = m_queue->GetClockCalibration(&m_calibrationGPU, &m_calibrationCPU); - if (FAILED(hr)) - { - D3D12_RANGE writeRange = { 0, 0 }; - m_timestampReadback->Unmap(0, &writeRange); - m_timestampMap = nullptr; - m_timestampReadback.Reset(); - m_timestampHeap.Reset(); - return false; - } + m_timestampMap = static_cast(timestampMap); m_timingSupported = true; return true; } -void CD3D12CommandQueue::UpdateClockCalibration() -{ - if (!m_timingSupported) - return; - - LARGE_INTEGER now; - if (QueryPerformanceCounter(&now) && - (UINT64)now.QuadPart >= m_calibrationCPU && - (UINT64)now.QuadPart - m_calibrationCPU < m_qpcFrequency) - return; - - UINT64 gpu; - UINT64 cpu; - if (SUCCEEDED(m_queue->GetClockCalibration(&gpu, &cpu))) - { - m_calibrationGPU = gpu; - m_calibrationCPU = cpu; - } -} - bool CD3D12CommandQueue::Init(ID3D12Device3 * device, D3D12_COMMAND_LIST_TYPE type, const WCHAR * name, - CallbackMode callbackMode) + CD3D12CommandSlot::CallbackMode callbackMode, UINT slotCount, + bool enableTiming) { - HRESULT hr; - D3D12_COMMAND_QUEUE_DESC queueDesc = {}; + if (!slotCount || slotCount > MAX_SLOTS) + { + DEBUG_ERROR("Invalid slot count for CommandQueue(%ls): %u", + name, slotCount); + return false; + } + D3D12_COMMAND_QUEUE_DESC queueDesc = {}; queueDesc.Type = type; queueDesc.Priority = D3D12_COMMAND_QUEUE_PRIORITY_HIGH; queueDesc.Flags = D3D12_COMMAND_QUEUE_FLAG_NONE; - hr = device->CreateCommandQueue(&queueDesc, IID_PPV_ARGS(&m_queue)); + HRESULT hr = device->CreateCommandQueue( + &queueDesc, IID_PPV_ARGS(&m_queue)); if (FAILED(hr)) { DEBUG_ERROR_HR(hr, "Failed to create the CommandQueue (%ls)", name); @@ -156,86 +438,40 @@ bool CD3D12CommandQueue::Init(ID3D12Device3 * device, } m_queue->SetName(name); - hr = device->CreateCommandAllocator(type, IID_PPV_ARGS(&m_allocator)); + hr = device->CreateFence(0, D3D12_FENCE_FLAG_NONE, + IID_PPV_ARGS(&m_fence)); if (FAILED(hr)) { - DEBUG_ERROR_HR(hr, "Failed to create the CommandAllocator (%ls)", name); + DEBUG_ERROR_HR(hr, "Failed to create the CommandQueue fence (%ls)", name); return false; } - hr = device->CreateCommandList(0, type, m_allocator.Get(), NULL, IID_PPV_ARGS(&m_gfxList)); - if (FAILED(hr)) - { - DEBUG_ERROR_HR(hr, "Failed to create the Graphics CommandList (%ls)", name); - return false; - } - m_gfxList->SetName(name); - - m_cmdList = m_gfxList; - if (!m_cmdList) - { - DEBUG_ERROR_HR(hr, "Failed to get the CommandList (%ls)", name); - return false; - } - - hr = device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(&m_fence)); - if (FAILED(hr)) - { - DEBUG_ERROR_HR(hr, "Failed to create the Fence (%ls)", name); - return false; - } - - m_event.Attach(CreateEvent(NULL, FALSE, FALSE, NULL)); - if (m_event.Get() == INVALID_HANDLE_VALUE) - { - DEBUG_ERROR_HR(GetLastError(), "Failed to create the completion event (%ls)", name); - return false; - } - - if (callbackMode != DISABLED) - { - ULONG flags = (callbackMode == FAST) ? - WT_EXECUTEINWAITTHREAD : WT_EXECUTEINPERSISTENTTHREAD; - - if (!RegisterWaitForSingleObject( - &m_waitHandle, - m_event.Get(), - [](PVOID param, BOOLEAN timeout){ - CD3D12CommandQueue * queue = (CD3D12CommandQueue*)param; - if (timeout) - queue->m_completionResult = false; - queue->OnCompletion(); - }, - this, - INFINITE, - flags)) - { - DEBUG_ERROR_HR(GetLastError(), - "Failed to register the completion wait (%ls)", name); - m_waitHandle = INVALID_HANDLE_VALUE; - return false; - } - } - - if (callbackMode != DISABLED && - type == D3D12_COMMAND_LIST_TYPE_COPY && !InitTiming(device, type)) + if (enableTiming && !InitTiming(device, slotCount)) DEBUG_WARN("GPU timing is unavailable for CommandQueue(%ls)", name); - m_name = name; - m_fenceValue = 0; - DEBUG_INFO("Created CD3D12CommandQueue(%ls)", name); + const ULONG waitFlags = callbackMode == CD3D12CommandSlot::FAST ? + WT_EXECUTEINWAITTHREAD : WT_EXECUTEINPERSISTENTTHREAD; + + m_name = name; + for (UINT i = 0; i < slotCount; ++i) + { + if (!m_slots[i].Init(device, this, name, i * 2, waitFlags)) + return false; + ++m_slotCount; + } + + DEBUG_INFO("Created CD3D12CommandQueue(%ls) with %u slots", + name, slotCount); return true; } void CD3D12CommandQueue::DeInit() { - if (m_waitHandle != INVALID_HANDLE_VALUE) - { - // Queue owners drain callbacks before destruction. Keep this unregister - // non-blocking so a removed or hung device cannot stall teardown. - UnregisterWait(m_waitHandle); - m_waitHandle = INVALID_HANDLE_VALUE; - } + WaitForIdle(); + + for (UINT i = 0; i < MAX_SLOTS; ++i) + m_slots[i].DeInit(); + m_slotCount = 0; if (m_timestampMap) { @@ -243,142 +479,156 @@ void CD3D12CommandQueue::DeInit() m_timestampReadback->Unmap(0, &writeRange); m_timestampMap = nullptr; } + + m_timestampReadback.Reset(); + m_timestampHeap.Reset(); + m_fence.Reset(); + m_queue.Reset(); + m_timingSupported = false; + m_qpcFrequency = 0; } -bool CD3D12CommandQueue::Execute() +CD3D12CommandSlot * CD3D12CommandQueue::Acquire(UINT slotIndex) { - m_needsReset = true; - m_completionResult = true; - m_pending = m_waitHandle != INVALID_HANDLE_VALUE; + if (slotIndex >= m_slotCount) + return nullptr; - HRESULT hr = m_gfxList->Close(); - if (FAILED(hr)) + for (int i = 0; i < 100; ++i) { - m_completionResult = false; - SetEvent(m_event.Get()); - - DEBUG_ERROR_HR(hr, "Failed to close the command list (%ls)", m_name); - return false; + if (m_slots[slotIndex].Acquire()) + return &m_slots[slotIndex]; + if (m_failed.load(std::memory_order_acquire)) + break; + Sleep(1); } - ID3D12CommandList * lists[] = { m_cmdList.Get() }; - m_queue->ExecuteCommandLists(1, lists); - ++m_fenceValue; + DEBUG_ERROR("Failed to acquire CommandSlot(%ls:%u)", m_name, slotIndex); + return nullptr; +} - hr = m_fence->SetEventOnCompletion(m_fenceValue, m_event.Get()); - if (FAILED(hr)) +CD3D12CommandSlot * CD3D12CommandQueue::Acquire() +{ + return Acquire(0); +} + +void CD3D12CommandQueue::WaitForIdle() +{ + for (UINT slot = 0; slot < m_slotCount; ++slot) { - m_completionResult = false; - SetEvent(m_event.Get()); - - DEBUG_ERROR_HR(hr, "Failed to set the fence signal (%ls)", m_name); - return false; + while (!m_slots[slot].IsIdle()) + { + // The callback may be delayed behind other thread-pool work. If its + // fence has completed, finish it here before releasing callback state. + m_slots[slot].OnCompletion(false); + if (m_slots[slot].IsIdle()) + break; + Sleep(1); + } } +} - m_queue->Signal(m_fence.Get(), m_fenceValue); +bool CD3D12CommandQueue::Submit(CD3D12CommandSlot& slot) +{ + bool result = false; + AcquireSRWLockExclusive(&m_submitLock); + + do + { + if (m_failed.load(std::memory_order_relaxed)) + break; + + for (UINT i = 0; i < slot.m_fenceWaitCount; ++i) + { + const CD3D12CommandSlot::FenceWait& wait = slot.m_fenceWaits[i]; + const HRESULT hr = m_queue->Wait(wait.fence.Get(), wait.value); + if (FAILED(hr)) + { + DEBUG_ERROR_HR(hr, "Failed to queue a fence wait (%ls)", m_name); + m_failed.store(true, std::memory_order_release); + break; + } + } + if (m_failed.load(std::memory_order_relaxed)) + break; + + const UINT64 fenceTarget = ++m_fenceValue; + slot.m_fenceTarget = fenceTarget; + slot.m_submitted.store(true, std::memory_order_release); + + ID3D12CommandList * lists[] = { slot.m_cmdList.Get() }; + m_queue->ExecuteCommandLists(1, lists); + + HRESULT hr = m_queue->Signal(m_fence.Get(), fenceTarget); + if (FAILED(hr)) + { + DEBUG_ERROR_HR(hr, "Failed to signal the CommandQueue (%ls)", m_name); + m_failed.store(true, std::memory_order_release); + slot.OnCompletion(false); + break; + } + + hr = m_fence->SetEventOnCompletion(fenceTarget, slot.m_event.Get()); + if (FAILED(hr)) + { + DEBUG_ERROR_HR(hr, + "Failed to register CommandSlot completion (%ls)", m_name); + + // The work is already submitted and fenced. Poll only on this rare + // error path so allocator, callback, and framebuffer ownership remain + // valid until completion or confirmed device removal. + while (slot.m_submitted.load(std::memory_order_acquire)) + { + slot.OnCompletion(false); + if (slot.m_submitted.load(std::memory_order_acquire)) + Sleep(1); + } + + result = slot.m_completionResult; + break; + } + + result = true; + } + while (false); + + ReleaseSRWLockExclusive(&m_submitLock); + return result; +} + +bool CD3D12CommandQueue::SnapshotTiming( + CD3D12CommandSlot& slot) const +{ + UINT64 frequency; + UINT64 gpu; + UINT64 cpu; + if (FAILED(m_queue->GetTimestampFrequency(&frequency)) || !frequency || + FAILED(m_queue->GetClockCalibration(&gpu, &cpu))) + return false; + + slot.m_timestampFrequency = frequency; + slot.m_calibrationGPU = gpu; + slot.m_calibrationCPU = cpu; return true; } -bool CD3D12CommandQueue::BeginTiming() -{ - m_timingActive = m_timingSupported; - if (!m_timingActive) - return false; - - // Refresh after an idle period; active queues are calibrated by their - // completion path without adding work to frame submission. - UpdateClockCalibration(); - m_gfxList->EndQuery( - m_timestampHeap.Get(), D3D12_QUERY_TYPE_TIMESTAMP, 0); - return true; -} - -void CD3D12CommandQueue::EndTiming() -{ - if (!m_timingActive) - return; - - m_gfxList->EndQuery( - m_timestampHeap.Get(), D3D12_QUERY_TYPE_TIMESTAMP, 1); - m_gfxList->ResolveQueryData( - m_timestampHeap.Get(), D3D12_QUERY_TYPE_TIMESTAMP, - 0, 2, m_timestampReadback.Get(), 0); -} - -bool CD3D12CommandQueue::ConvertGPUTimestamp( - UINT64 timestamp, uint64_t& result) const -{ - UINT64 cpuTimestamp; - if (timestamp < m_calibrationGPU) - { - const UINT64 delta = ScaleTicks( - m_calibrationGPU - timestamp, m_qpcFrequency, m_timestampFrequency); - if (delta > m_calibrationCPU) - return false; - cpuTimestamp = m_calibrationCPU - delta; - } - else - { - const UINT64 delta = ScaleTicks( - timestamp - m_calibrationGPU, m_qpcFrequency, m_timestampFrequency); - if (UINT64_MAX - m_calibrationCPU < delta) - return false; - cpuTimestamp = m_calibrationCPU + delta; - } - - result = TicksToNanoseconds(cpuTimestamp, m_qpcFrequency); - return true; -} - -bool CD3D12CommandQueue::GetGPUTimes( +bool CD3D12CommandQueue::GetGPUTimes(const CD3D12CommandSlot& slot, uint64_t& start, uint64_t& end) const { - if (!m_timingActive || - !ConvertGPUTimestamp(m_timestampMap[0], start) || - !ConvertGPUTimestamp(m_timestampMap[1], end) || + if (!slot.m_timingActive || !m_timestampMap || + !slot.m_timestampFrequency || !m_qpcFrequency) + return false; + + const UINT64 gpuStart = m_timestampMap[slot.m_queryBase]; + const UINT64 gpuEnd = m_timestampMap[slot.m_queryBase + 1]; + if (gpuEnd < gpuStart || + !ConvertGPUTimestamp(gpuStart, slot.m_timestampFrequency, + slot.m_calibrationGPU, slot.m_calibrationCPU, + m_qpcFrequency, start) || + !ConvertGPUTimestamp(gpuEnd, slot.m_timestampFrequency, + slot.m_calibrationGPU, slot.m_calibrationCPU, + m_qpcFrequency, end) || end < start) return false; return true; } - -#if 0 -void CD3D12CommandQueue::Wait() -{ - if (m_fence->GetCompletedValue() >= m_fenceValue) - { - m_pending = false; - return; - } - - m_fence->SetEventOnCompletion(m_fenceValue, m_event.Get()); - WaitForSingleObject(m_event.Get(), INFINITE); - m_pending = false; -} -#endif - -bool CD3D12CommandQueue::Reset() -{ - m_timingActive = false; - if (!m_needsReset) - return true; - - HRESULT hr; - - hr = m_allocator->Reset(); - if (FAILED(hr)) - { - DEBUG_ERROR_HR(hr, "Failed to reset the command allocator (%ls)", m_name); - return false; - } - - hr = m_gfxList->Reset(m_allocator.Get(), NULL); - if (FAILED(hr)) - { - DEBUG_ERROR_HR(hr, "Failed to reset the graphics command list (%ls)", m_name); - return false; - } - - m_needsReset = false; - return true; -} diff --git a/idd/LGIdd/CD3D12CommandQueue.h b/idd/LGIdd/CD3D12CommandQueue.h index fa347968..41286f08 100644 --- a/idd/LGIdd/CD3D12CommandQueue.h +++ b/idd/LGIdd/CD3D12CommandQueue.h @@ -31,96 +31,148 @@ using namespace Microsoft::WRL; using namespace Microsoft::WRL::Wrappers; using namespace Microsoft::WRL::Wrappers::HandleTraits; -class CD3D12CommandQueue -{ - private: - const WCHAR * m_name = nullptr; +class CD3D12CommandQueue; - ComPtr m_queue; +class CD3D12CommandSlot +{ + friend class CD3D12CommandQueue; + + private: + enum State + { + STATE_FREE, + STATE_RECORDING, + STATE_CANCELLING, + STATE_SUBMITTED, + STATE_COMPLETING, + STATE_FAILED, + }; + + struct FenceWait + { + ComPtr fence; + UINT64 value = 0; + }; + + static const UINT MAX_FENCE_WAITS = 2; + + const WCHAR * m_name = nullptr; + CD3D12CommandQueue * m_queue = nullptr; ComPtr m_allocator; ComPtr m_gfxList; ComPtr m_cmdList; - ComPtr m_fence; - ComPtr m_timestampHeap; - ComPtr m_timestampReadback; - UINT64 * m_timestampMap = nullptr; - UINT64 m_timestampFrequency = 0; - UINT64 m_calibrationGPU = 0; - UINT64 m_calibrationCPU = 0; - UINT64 m_qpcFrequency = 0; - bool m_timingSupported = false; - bool m_timingActive = false; - - std::atomic m_pending = false; + std::atomic m_state = STATE_FREE; + std::atomic m_submitted = false; HandleT m_event; - HANDLE m_waitHandle = INVALID_HANDLE_VALUE; - UINT64 m_fenceValue = 0; - bool m_needsReset = false; + HANDLE m_waitHandle = INVALID_HANDLE_VALUE; + bool m_needsReset = false; + UINT64 m_fenceTarget = 0; - typedef void (*CompletionFunction)(CD3D12CommandQueue * queue, + FenceWait m_fenceWaits[MAX_FENCE_WAITS]; + UINT m_fenceWaitCount = 0; + + typedef void (*CompletionFunction)(CD3D12CommandSlot * slot, bool result, void * param1, void * param2); - CompletionFunction m_completionCallback = nullptr; - void * m_completionParams[2]; - bool m_completionResult = true; + CompletionFunction m_completionCallback = nullptr; + void * m_completionParams[2] = {}; + bool m_completionResult = true; - bool InitTiming(ID3D12Device3 * device, D3D12_COMMAND_LIST_TYPE type); - void UpdateClockCalibration(); - bool ConvertGPUTimestamp(UINT64 timestamp, uint64_t& result) const; + UINT m_queryBase = 0; + bool m_timingActive = false; + UINT64 m_timestampFrequency = 0; + UINT64 m_calibrationGPU = 0; + UINT64 m_calibrationCPU = 0; - void OnCompletion() - { - if (m_completionCallback) - m_completionCallback( - this, - m_completionResult, - m_completionParams[0], - m_completionParams[1]); - UpdateClockCalibration(); - m_pending = false; - } + bool Init(ID3D12Device3 * device, CD3D12CommandQueue * queue, + const WCHAR * name, UINT queryBase, ULONG waitFlags); + void DeInit(); + void OnCompletion(bool timeout); public: - ~CD3D12CommandQueue() { DeInit(); } - enum CallbackMode { - DISABLED, // no callbacks - FAST, // callback is expected to return almost immediately - NORMAL // normal callback + FAST, + NORMAL, }; - bool Init(ID3D12Device3 * device, D3D12_COMMAND_LIST_TYPE type, const WCHAR * name, - CallbackMode callbackMode = DISABLED); + ~CD3D12CommandSlot() { DeInit(); } - void DeInit(); + bool Acquire(); + void Cancel(); + bool Execute(); - void SetCompletionCallback(CompletionFunction fn, void * param1, void * param2) + bool BeginTiming(); + void EndTiming(); + bool GetGPUTimes(uint64_t& start, uint64_t& end) const; + + bool WaitFor(ID3D12Fence * fence, UINT64 value); + bool WaitFor(const CD3D12CommandSlot& slot); + + void SetCompletionCallback(CompletionFunction fn, + void * param1, void * param2) { m_completionCallback = fn; m_completionParams[0] = param1; m_completionParams[1] = param2; } - bool Reset(); - bool Execute(); - - bool BeginTiming(); - void EndTiming(); - - // Return the copy boundaries in QueryPerformanceCounter-domain ns. - bool GetGPUTimes(uint64_t& start, uint64_t& end) const; - - //void Wait(); - bool IsReady () const { return !m_pending ; } - HANDLE GetEvent() const { return m_event.Get(); } - - void WaitFor(CD3D12CommandQueue& queue) + bool IsIdle() const { - m_queue->Wait(queue.m_fence.Get(), queue.m_fenceValue); + const State state = m_state.load(std::memory_order_acquire); + return state == STATE_FREE || + (state == STATE_FAILED && + !m_submitted.load(std::memory_order_acquire)); } - ComPtr GetCmdQueue() { return m_queue; } - ComPtr GetGfxList() { return m_gfxList; } + bool HasSubmittedWork() const + { + return m_submitted.load(std::memory_order_acquire); + } + + ComPtr GetGfxList() { return m_gfxList; } +}; + +class CD3D12CommandQueue +{ + friend class CD3D12CommandSlot; + + private: + static const UINT MAX_SLOTS = 2; + + const WCHAR * m_name = nullptr; + + ComPtr m_queue; + ComPtr m_fence; + UINT64 m_fenceValue = 0; + SRWLOCK m_submitLock = SRWLOCK_INIT; + std::atomic m_failed = false; + + CD3D12CommandSlot m_slots[MAX_SLOTS]; + UINT m_slotCount = 0; + + ComPtr m_timestampHeap; + ComPtr m_timestampReadback; + UINT64 * m_timestampMap = nullptr; + UINT64 m_qpcFrequency = 0; + bool m_timingSupported = false; + + bool InitTiming(ID3D12Device3 * device, UINT slotCount); + bool Submit(CD3D12CommandSlot& slot); + bool SnapshotTiming(CD3D12CommandSlot& slot) const; + bool GetGPUTimes(const CD3D12CommandSlot& slot, + uint64_t& start, uint64_t& end) const; + + public: + ~CD3D12CommandQueue() { DeInit(); } + + bool Init(ID3D12Device3 * device, D3D12_COMMAND_LIST_TYPE type, + const WCHAR * name, CD3D12CommandSlot::CallbackMode callbackMode, + UINT slotCount, bool enableTiming = false); + void DeInit(); + + CD3D12CommandSlot * Acquire(UINT slotIndex); + CD3D12CommandSlot * Acquire(); + void WaitForIdle(); }; diff --git a/idd/LGIdd/CD3D12Device.cpp b/idd/LGIdd/CD3D12Device.cpp index 692ec214..ef9fff9c 100644 --- a/idd/LGIdd/CD3D12Device.cpp +++ b/idd/LGIdd/CD3D12Device.cpp @@ -130,14 +130,14 @@ CD3D12Device::InitResult CD3D12Device::Init(CIVSHMEM &ivshmem, DEBUG_INFO("Using IVSHMEM as a D3D12Heap"); } - for(int i = 0; i < ARRAYSIZE(m_copyQueue); ++i) - if (!m_copyQueue[i].Init(m_device.Get(), D3D12_COMMAND_LIST_TYPE_COPY, L"Copy", - m_indirectCopy ? CD3D12CommandQueue::NORMAL : CD3D12CommandQueue::FAST)) - return InitResult::FAILURE; + if (!m_copyQueue.Init(m_device.Get(), D3D12_COMMAND_LIST_TYPE_COPY, + L"Copy", m_indirectCopy ? CD3D12CommandSlot::NORMAL : + CD3D12CommandSlot::FAST, 2, true)) + return InitResult::FAILURE; if (m_computeEnabled && !m_computeQueue.Init(m_device.Get(), D3D12_COMMAND_LIST_TYPE_COMPUTE, - L"Compute", CD3D12CommandQueue::FAST)) + L"Compute", CD3D12CommandSlot::FAST, 1)) return InitResult::FAILURE; DEBUG_INFO("Created CD3D12Device"); @@ -153,19 +153,9 @@ void CD3D12Device::DeInit() void CD3D12Device::WaitForIdle() { - // A queue is ready once its GPU work has signalled and its completion - // callback has run (clearing the pending flag). Bound the wait so a - // removed/hung device cannot stall teardown indefinitely. - auto drain = [](CD3D12CommandQueue& queue) - { - for (int i = 0; i < 1000 && !queue.IsReady(); ++i) - Sleep(1); - }; - - for (int i = 0; i < ARRAYSIZE(m_copyQueue); ++i) - drain(m_copyQueue[i]); + m_copyQueue.WaitForIdle(); if (m_computeEnabled) - drain(m_computeQueue); + m_computeQueue.WaitForIdle(); } bool CD3D12Device::HeapTest() @@ -212,31 +202,12 @@ bool CD3D12Device::HeapTest() return true; } -CD3D12CommandQueue * CD3D12Device::GetCopyQueue() +CD3D12CommandSlot * CD3D12Device::GetCopySlot(unsigned frameIndex) { - // try for up to a maximum of 100ms to find a free copy queue - for (int c = 0; c < 100; ++c) - { - for (int i = 0; i < ARRAYSIZE(m_copyQueue); ++i) - { - auto& queue = m_copyQueue[m_copyQueueIndex++]; - if (m_copyQueueIndex == ARRAYSIZE(m_copyQueue)) - m_copyQueueIndex = 0; - - if (queue.IsReady()) - { - queue.Reset(); - return &queue; - } - } - Sleep(1); - } - - DEBUG_ERROR("Failed to get a copy queue"); - return nullptr; + return m_copyQueue.Acquire(frameIndex); } -CD3D12CommandQueue * CD3D12Device::GetComputeQueue() +CD3D12CommandSlot * CD3D12Device::GetComputeSlot() { if (!m_computeEnabled) { @@ -244,16 +215,5 @@ CD3D12CommandQueue * CD3D12Device::GetComputeQueue() return nullptr; } - for (int c = 0; c < 100; ++c) - { - if (m_computeQueue.IsReady()) - { - m_computeQueue.Reset(); - return &m_computeQueue; - } - Sleep(1); - } - - DEBUG_ERROR("Failed to get a compute queue"); - return nullptr; + return m_computeQueue.Acquire(); } diff --git a/idd/LGIdd/CD3D12Device.h b/idd/LGIdd/CD3D12Device.h index 2aa62e3f..60ec44fb 100644 --- a/idd/LGIdd/CD3D12Device.h +++ b/idd/LGIdd/CD3D12Device.h @@ -49,10 +49,9 @@ struct CD3D12Device ComPtr m_device; ComPtr m_ivshmemHeap; - CD3D12CommandQueue m_copyQueue[4]; - unsigned m_copyQueueIndex = 0; + CD3D12CommandQueue m_copyQueue; CD3D12CommandQueue m_computeQueue; - bool m_computeEnabled = false; + bool m_computeEnabled = false; bool HeapTest(); @@ -80,6 +79,6 @@ struct CD3D12Device ComPtr GetHeap() { return m_ivshmemHeap; } bool IsIndirectCopy() { return m_indirectCopy; } - CD3D12CommandQueue * GetCopyQueue(); - CD3D12CommandQueue * GetComputeQueue(); + CD3D12CommandSlot * GetCopySlot(unsigned frameIndex); + CD3D12CommandSlot * GetComputeSlot(); }; diff --git a/idd/LGIdd/CIndirectDeviceContext.cpp b/idd/LGIdd/CIndirectDeviceContext.cpp index b555f66c..a83cdcde 100644 --- a/idd/LGIdd/CIndirectDeviceContext.cpp +++ b/idd/LGIdd/CIndirectDeviceContext.cpp @@ -1066,8 +1066,12 @@ bool CIndirectDeviceContext::SetupLGMP(size_t alignSize) const size_t alignOffset = alignSize - sizeof(FrameBuffer); m_frame[i]->offset = (uint32_t)alignOffset; m_frameBuffer[i] = (FrameBuffer*)(((uint8_t*)m_frame[i]) + alignOffset); + m_frameInFlight[i].store(false, std::memory_order_release); } + m_publishedFrameIndex.store(-1, std::memory_order_release); + m_frameResendPending = false; + WDF_TIMER_CONFIG config; WDF_TIMER_CONFIG_INIT_PERIODIC(&config, [](WDFTIMER timer) -> void @@ -1101,8 +1105,6 @@ bool CIndirectDeviceContext::SetupLGMP(size_t alignSize) void CIndirectDeviceContext::DeInitLGMP() { - m_publishedFrameIndex.store(-1); - // The retry timer callback dereferences this context, so make sure it is // stopped and drained before we tear anything down. Wait for any in-flight // callback to complete. @@ -1113,7 +1115,11 @@ void CIndirectDeviceContext::DeInitLGMP() } if (m_lgmp == nullptr) + { + m_publishedFrameIndex.store(-1, std::memory_order_release); + m_frameResendPending = false; return; + } if (m_lgmpTimer) { @@ -1121,6 +1127,14 @@ void CIndirectDeviceContext::DeInitLGMP() m_lgmpTimer = nullptr; } + AcquireSRWLockExclusive(&m_framePublishLock); + m_publishedFrameIndex.store(-1, std::memory_order_release); + m_frameResendPending = false; + ReleaseSRWLockExclusive(&m_framePublishLock); + + for (int i = 0; i < LGMP_Q_FRAME_LEN; ++i) + m_frameInFlight[i].store(false, std::memory_order_release); + for (int i = 0; i < LGMP_Q_FRAME_LEN; ++i) lgmpHostMemFree(&m_frameMemory[i]); for (int i = 0; i < LGMP_Q_POINTER_LEN; ++i) @@ -1187,12 +1201,26 @@ void CIndirectDeviceContext::LGMPTimer() lgmpHostAckData(m_pointerQueue); } - if (lgmpHostQueueNewSubs(m_frameQueue) && m_monitor) + AcquireSRWLockExclusive(&m_framePublishLock); + if (lgmpHostQueueNewSubs(m_frameQueue)) + m_frameResendPending = true; + + if (m_frameResendPending && m_monitor && + lgmpHostQueuePending(m_frameQueue) == 0) { - const LONG frameIndex = m_publishedFrameIndex.load(); + const LONG frameIndex = + m_publishedFrameIndex.load(std::memory_order_acquire); if (frameIndex >= 0) - lgmpHostQueuePost(m_frameQueue, 0, m_frameMemory[frameIndex]); + { + status = lgmpHostQueuePost( + m_frameQueue, 0, m_frameMemory[frameIndex]); + if (status == LGMP_OK) + m_frameResendPending = false; + else if (status != LGMP_ERR_QUEUE_FULL) + DEBUG_ERROR("Failed to resend frame: %s", lgmpStatusString(status)); + } } + ReleaseSRWLockExclusive(&m_framePublishLock); if (lgmpHostQueueNewSubs(m_pointerQueue)) { @@ -1203,8 +1231,16 @@ void CIndirectDeviceContext::LGMPTimer() bool CIndirectDeviceContext::FrameBufferAvailable() const { - return m_lgmp && m_frameQueue && - lgmpHostQueuePending(m_frameQueue) < LGMP_Q_FRAME_LEN; + if (!m_lgmp || !m_frameQueue || + lgmpHostQueuePending(m_frameQueue) >= LGMP_Q_FRAME_LEN) + return false; + + const LONG publishedFrameIndex = + m_publishedFrameIndex.load(std::memory_order_acquire); + const unsigned frameIndex = + static_cast(publishedFrameIndex + 1) % LGMP_Q_FRAME_LEN; + + return !m_frameInFlight[frameIndex].load(std::memory_order_acquire); } CIndirectDeviceContext::PreparedFrameBuffer CIndirectDeviceContext::PrepareFrameBuffer( @@ -1216,6 +1252,22 @@ CIndirectDeviceContext::PreparedFrameBuffer CIndirectDeviceContext::PrepareFrame if (!FrameBufferAvailable()) return result; + if (dstFormat.format == FRAME_TYPE_INVALID) + { + DEBUG_ERROR("Unsupported frame format, skipping frame"); + return result; + } + + const LONG publishedFrameIndex = + m_publishedFrameIndex.load(std::memory_order_acquire); + const unsigned frameIndex = + static_cast(publishedFrameIndex + 1) % LGMP_Q_FRAME_LEN; + + bool expected = false; + if (!m_frameInFlight[frameIndex].compare_exchange_strong( + expected, true, std::memory_order_acq_rel)) + return result; + if (m_width != dstFormat.desc.Width || m_height != dstFormat.desc.Height || m_pitch != pitch || @@ -1265,16 +1317,7 @@ CIndirectDeviceContext::PreparedFrameBuffer CIndirectDeviceContext::PrepareFrame m_lastHDRMaxFrameAverageLightLevel = dstFormat.maxFrameAverageLightLevel; m_lastSDRWhiteLevel = dstFormat.sdrWhiteLevel; - if (++m_frameIndex == LGMP_Q_FRAME_LEN) - m_frameIndex = 0; - - KVMFRFrame * fi = m_frame[m_frameIndex]; - - if (dstFormat.format == FRAME_TYPE_INVALID) - { - DEBUG_ERROR("Unsupported frame format, skipping frame"); - return result; - } + KVMFRFrame * fi = m_frame[frameIndex]; const unsigned maxRows = (unsigned)(m_maxFrameSize / pitch); const int bpp = dstFormat.format == FRAME_TYPE_RGBA16F ? 8 : 4; @@ -1340,10 +1383,10 @@ CIndirectDeviceContext::PreparedFrameBuffer CIndirectDeviceContext::PrepareFrame } } - FrameBuffer* fb = m_frameBuffer[m_frameIndex]; + FrameBuffer* fb = m_frameBuffer[frameIndex]; fb->wp = 0; - result.frameIndex = m_frameIndex; + result.frameIndex = frameIndex; result.mem = fb->data; return result; @@ -1354,13 +1397,17 @@ bool CIndirectDeviceContext::PublishFrameBuffer(unsigned frameIndex) if (!m_frameQueue || frameIndex >= LGMP_Q_FRAME_LEN) return false; - /* Make resends select this submitted frame before posting it. This prevents - * a new subscriber racing publication from receiving the previous frame - * after the new one. */ - m_publishedFrameIndex.store(static_cast(frameIndex)); - + AcquireSRWLockExclusive(&m_framePublishLock); const LGMP_STATUS status = lgmpHostQueuePost(m_frameQueue, 0, m_frameMemory[frameIndex]); + if (status == LGMP_OK) + { + m_publishedFrameIndex.store( + static_cast(frameIndex), std::memory_order_release); + m_frameResendPending = false; + } + ReleaseSRWLockExclusive(&m_framePublishLock); + if (status != LGMP_OK) { DEBUG_ERROR("Failed to publish frame: %s", lgmpStatusString(status)); @@ -1370,6 +1417,32 @@ bool CIndirectDeviceContext::PublishFrameBuffer(unsigned frameIndex) return true; } +void CIndirectDeviceContext::AbortFrameBuffer(unsigned frameIndex) +{ + if (frameIndex >= LGMP_Q_FRAME_LEN) + return; + + m_frameBuffer[frameIndex]->wp = 0; + InterlockedExchange((volatile LONG *)&m_frame[frameIndex]->timingValid, 0); + m_frameInFlight[frameIndex].store(false, std::memory_order_release); +} + +void CIndirectDeviceContext::FailFrameBuffer(unsigned frameIndex) +{ + if (frameIndex >= LGMP_Q_FRAME_LEN) + return; + + InterlockedExchange((volatile LONG *)&m_frame[frameIndex]->timingValid, 0); + FinalizeFrameBuffer(frameIndex); + CompleteFrameBuffer(frameIndex); +} + +void CIndirectDeviceContext::CompleteFrameBuffer(unsigned frameIndex) +{ + if (frameIndex < LGMP_Q_FRAME_LEN) + m_frameInFlight[frameIndex].store(false, std::memory_order_release); +} + void CIndirectDeviceContext::SetFrameTiming(unsigned frameIndex, uint64_t captureTime, uint64_t postProcessTime, uint64_t copyTime, uint64_t readyTime) @@ -1401,8 +1474,9 @@ void CIndirectDeviceContext::WriteFrameBuffer(unsigned frameIndex, void* src, si void CIndirectDeviceContext::FinalizeFrameBuffer(unsigned frameIndex) const { - FrameBuffer * fb = m_frameBuffer[frameIndex]; - fb->wp = m_height * m_pitch; + const KVMFRFrame * frame = m_frame[frameIndex]; + FrameBuffer * fb = m_frameBuffer[frameIndex]; + fb->wp = frame->dataHeight * frame->pitch; } void CIndirectDeviceContext::SendCursor(const IDARG_OUT_QUERY_HWCURSOR& info, diff --git a/idd/LGIdd/CIndirectDeviceContext.h b/idd/LGIdd/CIndirectDeviceContext.h index 5f700713..a0924fcf 100644 --- a/idd/LGIdd/CIndirectDeviceContext.h +++ b/idd/LGIdd/CIndirectDeviceContext.h @@ -103,8 +103,10 @@ private: size_t m_alignSize = 0; size_t m_frameMemoryOffset = 0; size_t m_maxFrameSize = 0; - int m_frameIndex = 0; std::atomic m_publishedFrameIndex = -1; + std::atomic m_frameInFlight[LGMP_Q_FRAME_LEN] = {}; + SRWLOCK m_framePublishLock = SRWLOCK_INIT; + bool m_frameResendPending = false; uint32_t m_formatVer = 0; uint32_t m_frameSerial = 0; PLGMPMemory m_frameMemory[LGMP_Q_FRAME_LEN] = {}; @@ -217,6 +219,9 @@ public: bool FrameBufferAvailable() const; PreparedFrameBuffer PrepareFrameBuffer(unsigned pitch, const D12FrameFormat& srcFormat, const D12FrameFormat& dstFormat, const RECT * dirtyRects, unsigned nbDirtyRects); bool PublishFrameBuffer(unsigned frameIndex); + void AbortFrameBuffer(unsigned frameIndex); + void FailFrameBuffer(unsigned frameIndex); + void CompleteFrameBuffer(unsigned frameIndex); void SetFrameTiming(unsigned frameIndex, uint64_t captureTime, uint64_t postProcessTime, uint64_t copyTime, uint64_t readyTime); void WriteFrameBuffer(unsigned frameIndex, void* src, size_t offset, size_t len, bool setWritePos) const; diff --git a/idd/LGIdd/CInteropResource.cpp b/idd/LGIdd/CInteropResource.cpp index 699e7199..cc2792c0 100644 --- a/idd/LGIdd/CInteropResource.cpp +++ b/idd/LGIdd/CInteropResource.cpp @@ -126,16 +126,33 @@ bool CInteropResource::Compare(const ComPtr& srcTex) m_format.Format == format.Format; } -void CInteropResource::Signal() +bool CInteropResource::Signal() { - ++m_fenceValue; - m_dx11Device->GetContext()->Signal(m_d11Fence.Get(), m_fenceValue); + const UINT64 fenceValue = m_fenceValue + 1; + const HRESULT hr = + m_dx11Device->GetContext()->Signal(m_d11Fence.Get(), fenceValue); + if (FAILED(hr)) + { + DEBUG_ERROR_HR(hr, "Failed to signal the D3D11 source fence"); + return false; + } + + m_fenceValue = fenceValue; + return true; } -void CInteropResource::Sync(CD3D12CommandQueue& queue) +bool CInteropResource::Sync(CD3D12CommandSlot& slot) { - if (m_d11Fence->GetCompletedValue() < m_fenceValue) - queue.GetCmdQueue()->Wait(m_d12Fence.Get(), m_fenceValue); + if (m_d11Fence->GetCompletedValue() >= m_fenceValue) + return true; + + if (!slot.WaitFor(m_d12Fence.Get(), m_fenceValue)) + { + DEBUG_ERROR("Failed to queue a wait for the D3D11 source fence"); + return false; + } + + return true; } void CInteropResource::SetFullDamage() diff --git a/idd/LGIdd/CInteropResource.h b/idd/LGIdd/CInteropResource.h index a67c0409..07b6f16f 100644 --- a/idd/LGIdd/CInteropResource.h +++ b/idd/LGIdd/CInteropResource.h @@ -59,8 +59,8 @@ class CInteropResource bool IsReady() { return m_ready; } bool Compare(const ComPtr& srcTex); - void Signal(); - void Sync(CD3D12CommandQueue& queue); + bool Signal(); + bool Sync(CD3D12CommandSlot& slot); void SetFullDamage(); void SetDirtyRects(const RECT * dirtyRects, unsigned nbDirtyRects); diff --git a/idd/LGIdd/CSwapChainProcessor.cpp b/idd/LGIdd/CSwapChainProcessor.cpp index fbcd4949..827635bf 100644 --- a/idd/LGIdd/CSwapChainProcessor.cpp +++ b/idd/LGIdd/CSwapChainProcessor.cpp @@ -316,23 +316,31 @@ bool CSwapChainProcessor::SwapChainThreadCore() } void CSwapChainProcessor::CompletionFunction( - CD3D12CommandQueue * queue, bool result, void * param1, void * param2) + CD3D12CommandSlot * slot, bool result, void * param1, void * param2) { auto sc = (CSwapChainProcessor *)param1; auto fbRes = (CFrameBufferResource *)param2; + if (!result) + { + // A submitted frame may already be in LGMP, or publication may race this + // callback. Make the message releasable even though its contents failed. + sc->m_devContext->FailFrameBuffer(fbRes->GetFrameIndex()); + return; + } + const uint64_t cpuCopyStart = fbRes->GetCopyStart(); uint64_t gpuCopyStart = 0; uint64_t gpuCopyEnd = 0; - if (result && sc->m_dx12Device->IsIndirectCopy()) + if (sc->m_dx12Device->IsIndirectCopy()) sc->m_devContext->WriteFrameBuffer( fbRes->GetFrameIndex(), fbRes->GetMap(), 0, fbRes->GetFrameSize(), false); // Queue waits execute before the start timestamp. The end timestamp follows // the last CopyTextureRegion, separating GPU work from readiness dispatch. - const bool gpuTimingValid = result && - queue->GetGPUTimes(gpuCopyStart, gpuCopyEnd); + const bool gpuTimingValid = + slot->GetGPUTimes(gpuCopyStart, gpuCopyEnd); // Publish readiness before sampling the endpoint. Timing has its own valid // flag and is published immediately afterwards. @@ -353,6 +361,7 @@ void CSwapChainProcessor::CompletionFunction( sc->m_devContext->SetFrameTiming(fbRes->GetFrameIndex(), fbRes->GetCaptureTime(), postProcessTime, copyTime, readyTime); + sc->m_devContext->CompleteFrameBuffer(fbRes->GetFrameIndex()); } @@ -652,7 +661,11 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer * use a fence. Because we share this texture with DirectX12 it is able to * read from it before the desktop duplication API has finished updating it. */ - srcRes->Signal(); + if (!srcRes->Signal()) + { + SetFullPendingDamage(); + return false; + } RECT dirtyRects[LG_MAX_DIRTY_RECTS] = {0}; bool noImageUpdate = false; @@ -819,38 +832,34 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer unsigned frameDirtyRectCount = nbDirtyRects; m_postProcessor.AdjustFrameDamage(frameDirtyRects, &frameDirtyRectCount); - auto copyQueue = m_dx12Device->GetCopyQueue(); - if (!copyQueue) - { - DEBUG_ERROR("Failed to get a CopyQueue"); - return false; - } - ComPtr copySrcResource = srcRes->GetRes(); - CD3D12CommandQueue * computeQueue = nullptr; + CD3D12CommandSlot * computeSlot = nullptr; if (m_postProcessor.HasActiveEffects()) { - computeQueue = m_dx12Device->GetComputeQueue(); - if (!computeQueue) + computeSlot = m_dx12Device->GetComputeSlot(); + if (!computeSlot) { - DEBUG_ERROR("Failed to get a ComputeQueue"); + DEBUG_ERROR("Failed to get a compute CommandSlot"); + return false; + } + + if (!srcRes->Sync(*computeSlot)) + { + computeSlot->Cancel(); + SetFullPendingDamage(); return false; } - srcRes->Sync(*computeQueue); copySrcResource = m_postProcessor.Run( - computeQueue->GetGfxList(), copySrcResource, + computeSlot->GetGfxList(), copySrcResource, currentDirtyRects, &nbDirtyRects); - if (!computeQueue->Execute()) + if (!computeSlot->Execute()) { SetFullPendingDamage(); return false; } - copyQueue->WaitFor(*computeQueue); } - else - srcRes->Sync(*copyQueue); ClipDirtyRects(currentDirtyRects, &nbDirtyRects, dstFormat.desc); @@ -861,8 +870,8 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer frameDirtyRects, frameDirtyRectCount); - // The LGMP timer can fill the queue with a subscriber resend after the early - // availability check. Treat this as a dropped frame rather than an error. + // Queue or framebuffer ownership can change after the early availability + // check. Treat this as a dropped frame rather than an error. if (!buffer.mem) return true; @@ -871,15 +880,37 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer if (!fbRes) { + m_devContext->AbortFrameBuffer(buffer.frameIndex); DEBUG_ERROR("Failed to get a CFrameBufferResource from the pool"); SetFullPendingDamage(); return false; } + CD3D12CommandSlot * copySlot = + m_dx12Device->GetCopySlot(buffer.frameIndex); + if (!copySlot) + { + m_devContext->AbortFrameBuffer(buffer.frameIndex); + DEBUG_ERROR("Failed to get a copy CommandSlot"); + SetFullPendingDamage(); + return false; + } + + const bool syncResult = computeSlot ? + copySlot->WaitFor(*computeSlot) : srcRes->Sync(*copySlot); + if (!syncResult) + { + copySlot->Cancel(); + m_devContext->AbortFrameBuffer(buffer.frameIndex); + DEBUG_ERROR("Failed to queue copy synchronization"); + SetFullPendingDamage(); + return false; + } + const uint64_t copyStart = Nanotime(); fbRes->SetTiming(captureTime, postProcessStart, copyStart); - copyQueue->SetCompletionCallback(&CompletionFunction, this, fbRes); + copySlot->SetCompletionCallback(&CompletionFunction, this, fbRes); D3D12_TEXTURE_COPY_LOCATION srcLoc = {}; srcLoc.pResource = copySrcResource.Get(); @@ -924,24 +955,26 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr acquiredBuffer copyDirtyRects, nbCopyDirtyRects, dstFormat.desc); } - // The source/compute waits were inserted directly on the queue above. The - // command-list timestamp therefore marks the first actual copy operation. - copyQueue->BeginTiming(); + // Source/compute waits are submitted immediately before this command list. + // The timestamp therefore marks the first actual copy operation. + copySlot->BeginTiming(); if (fullCopy) { - copyQueue->GetGfxList()->CopyTextureRegion( + copySlot->GetGfxList()->CopyTextureRegion( &dstLoc, 0, 0, 0, &srcLoc, NULL); } else { for (const RECT * rect = copyDirtyRects; rect < copyDirtyRects + nbCopyDirtyRects; ++rect) - CopyDirtyRect(copyQueue->GetGfxList(), &dstLoc, &srcLoc, *rect); + CopyDirtyRect(copySlot->GetGfxList(), &dstLoc, &srcLoc, *rect); } - copyQueue->EndTiming(); + copySlot->EndTiming(); - if (!copyQueue->Execute()) + if (!copySlot->Execute()) { + if (!copySlot->HasSubmittedWork()) + m_devContext->AbortFrameBuffer(buffer.frameIndex); SetFullPendingDamage(); return false; } diff --git a/idd/LGIdd/CSwapChainProcessor.h b/idd/LGIdd/CSwapChainProcessor.h index 5974bbba..6f732bf2 100644 --- a/idd/LGIdd/CSwapChainProcessor.h +++ b/idd/LGIdd/CSwapChainProcessor.h @@ -94,7 +94,7 @@ private: void CursorThread(); static void CompletionFunction( - CD3D12CommandQueue * queue, bool result, void * param1, void * param2); + CD3D12CommandSlot * slot, bool result, void * param1, void * param2); void AccumulateFrameDamage(const RECT * dirtyRects, unsigned nbDirtyRects); void SetFullPendingDamage(); #ifdef HAS_IDDCX_110 diff --git a/idd/LGIdd/effect/CColorTransformEffect.cpp b/idd/LGIdd/effect/CColorTransformEffect.cpp index e90cf204..7c9b74ba 100644 --- a/idd/LGIdd/effect/CColorTransformEffect.cpp +++ b/idd/LGIdd/effect/CColorTransformEffect.cpp @@ -280,7 +280,7 @@ ComPtr CColorTransformEffect::Run( UNREFERENCED_PARAMETER(dirtyRects); UNREFERENCED_PARAMETER(nbDirtyRects); - // GetComputeQueue waits for the previous submission before Run is called, + // GetComputeSlot waits for the previous submission before Run is called, // so this is the first point where the shared upload buffers are guaranteed // not to be in use by the GPU. if (m_uploadPending) diff --git a/repos/LGMP b/repos/LGMP index 149c7310..37fb5d86 160000 --- a/repos/LGMP +++ b/repos/LGMP @@ -1 +1 @@ -Subproject commit 149c731050d28c93532ee5ab4c84dfe6c2342f49 +Subproject commit 37fb5d8612e2ec8ad705e11fb0adf69c18554144