[idd] serialize framebuffer copies on one queue

Replace the rotating D3D12 copy queues with two framebuffer-bound
recording slots on one physical COPY queue. Keep allocator, command
list, query range, callback state, and fence target independent per
slot.

Submit source waits, execution, and signaling on the shared timeline.
Track framebuffer ownership through completion and serialize LGMP
publication around the last successfully published frame.

idd/LGIdd/CD3D12CommandQueue.cpp

@@ -33,22 +33,326 @@ static uint64_t TicksToNanoseconds(uint64_t ticks, uint64_t frequency)
   return ScaleTicks(ticks, 1000000000ULL, frequency);
 }
 
-bool CD3D12CommandQueue::InitTiming(ID3D12Device3 * device,
+static bool ConvertGPUTimestamp(UINT64 timestamp, UINT64 timestampFrequency,
-  D3D12_COMMAND_LIST_TYPE type)
+  UINT64 calibrationGPU, UINT64 calibrationCPU, UINT64 qpcFrequency,
+  uint64_t& result)
 {
-  if (type != D3D12_COMMAND_LIST_TYPE_COPY)
+  UINT64 cpuTimestamp;
+  if (timestamp < calibrationGPU)
+  {
+    const UINT64 delta = ScaleTicks(
+      calibrationGPU - timestamp, qpcFrequency, timestampFrequency);
+    if (delta > calibrationCPU)
+      return false;
+    cpuTimestamp = calibrationCPU - delta;
+  }
+  else
+  {
+    const UINT64 delta = ScaleTicks(
+      timestamp - calibrationGPU, qpcFrequency, timestampFrequency);
+    if (UINT64_MAX - calibrationCPU < delta)
+      return false;
+    cpuTimestamp = calibrationCPU + delta;
+  }
+
+  result = TicksToNanoseconds(cpuTimestamp, qpcFrequency);
+  return true;
+}
+
+bool CD3D12CommandSlot::Init(ID3D12Device3 * device,
+  CD3D12CommandQueue * queue, const WCHAR * name, UINT queryBase,
+  ULONG waitFlags)
+{
+  m_queue     = queue;
+  m_name      = name;
+  m_queryBase = queryBase;
+
+  HRESULT hr = device->CreateCommandAllocator(
+    queue->m_queue->GetDesc().Type, IID_PPV_ARGS(&m_allocator));
+  if (FAILED(hr))
+  {
+    DEBUG_ERROR_HR(hr, "Failed to create the CommandAllocator (%ls)", name);
+    return false;
+  }
+
+  hr = device->CreateCommandList(0, queue->m_queue->GetDesc().Type,
+    m_allocator.Get(), NULL, IID_PPV_ARGS(&m_gfxList));
+  if (FAILED(hr))
+  {
+    DEBUG_ERROR_HR(hr, "Failed to create the Graphics CommandList (%ls)", name);
+    return false;
+  }
+  m_gfxList->SetName(name);
+
+  m_cmdList = m_gfxList;
+  if (!m_cmdList)
+  {
+    DEBUG_ERROR("Failed to get the CommandList (%ls)", name);
+    return false;
+  }
+
+  m_event.Attach(CreateEvent(NULL, FALSE, FALSE, NULL));
+  if (!m_event.Get())
+  {
+    DEBUG_ERROR_HR(GetLastError(),
+      "Failed to create the completion event (%ls)", name);
+    return false;
+  }
+
+  if (!RegisterWaitForSingleObject(
+    &m_waitHandle,
+    m_event.Get(),
+    [](PVOID param, BOOLEAN timeout){
+      static_cast<CD3D12CommandSlot *>(param)->OnCompletion(!!timeout);
+    },
+    this,
+    INFINITE,
+    waitFlags))
+  {
+    DEBUG_ERROR_HR(GetLastError(),
+      "Failed to register the completion wait (%ls)", name);
+    m_waitHandle = INVALID_HANDLE_VALUE;
+    return false;
+  }
+
+  DEBUG_INFO("Created CD3D12CommandSlot(%ls)", name);
+  return true;
+}
+
+void CD3D12CommandSlot::DeInit()
+{
+  if (m_waitHandle != INVALID_HANDLE_VALUE)
+  {
+    if (!UnregisterWaitEx(m_waitHandle, INVALID_HANDLE_VALUE))
+      DEBUG_WARN_HR(GetLastError(),
+        "Failed to unregister the completion wait (%ls)", m_name);
+    m_waitHandle = INVALID_HANDLE_VALUE;
+  }
+
+  m_event.Close();
+  m_cmdList.Reset();
+  m_gfxList.Reset();
+  m_allocator.Reset();
+  m_queue = nullptr;
+}
+
+bool CD3D12CommandSlot::Acquire()
+{
+  if (!m_queue || m_queue->m_failed.load(std::memory_order_acquire))
     return false;
 
+  State expected = STATE_FREE;
+  if (!m_state.compare_exchange_strong(expected, STATE_RECORDING,
+      std::memory_order_acq_rel))
+    return false;
+
+  m_completionCallback  = nullptr;
+  m_completionParams[0] = nullptr;
+  m_completionParams[1] = nullptr;
+  m_completionResult    = true;
+  m_fenceTarget         = 0;
+  m_fenceWaitCount      = 0;
+  m_timingActive        = false;
+  m_timestampFrequency  = 0;
+  m_calibrationGPU      = 0;
+  m_calibrationCPU      = 0;
+  m_submitted.store(false, std::memory_order_release);
+
+  for (UINT i = 0; i < MAX_FENCE_WAITS; ++i)
+  {
+    m_fenceWaits[i].fence.Reset();
+    m_fenceWaits[i].value = 0;
+  }
+
+  if (!m_needsReset)
+    return true;
+
+  HRESULT hr = m_allocator->Reset();
+  if (FAILED(hr))
+  {
+    DEBUG_ERROR_HR(hr, "Failed to reset the CommandAllocator (%ls)", m_name);
+    m_queue->m_failed.store(true, std::memory_order_release);
+    m_state.store(STATE_FAILED, std::memory_order_release);
+    return false;
+  }
+
+  hr = m_gfxList->Reset(m_allocator.Get(), NULL);
+  if (FAILED(hr))
+  {
+    DEBUG_ERROR_HR(hr, "Failed to reset the CommandList (%ls)", m_name);
+    m_queue->m_failed.store(true, std::memory_order_release);
+    m_state.store(STATE_FAILED, std::memory_order_release);
+    return false;
+  }
+
+  m_needsReset = false;
+  return true;
+}
+
+void CD3D12CommandSlot::Cancel()
+{
+  State expected = STATE_RECORDING;
+  if (!m_state.compare_exchange_strong(expected, STATE_CANCELLING,
+      std::memory_order_acq_rel))
+  {
+    DEBUG_ERROR("Command slot cancelled while not recording (%ls)", m_name);
+    return;
+  }
+
+  const HRESULT hr = m_gfxList->Close();
+  m_needsReset   = true;
+  m_timingActive = false;
+
+  for (UINT i = 0; i < m_fenceWaitCount; ++i)
+  {
+    m_fenceWaits[i].fence.Reset();
+    m_fenceWaits[i].value = 0;
+  }
+  m_fenceWaitCount = 0;
+
+  if (FAILED(hr))
+  {
+    DEBUG_ERROR_HR(hr, "Failed to close the cancelled CommandList (%ls)",
+      m_name);
+    m_queue->m_failed.store(true, std::memory_order_release);
+    m_state.store(STATE_FAILED, std::memory_order_release);
+    return;
+  }
+
+  m_completionCallback  = nullptr;
+  m_completionParams[0] = nullptr;
+  m_completionParams[1] = nullptr;
+  m_submitted.store(false, std::memory_order_release);
+  m_state.store(STATE_FREE, std::memory_order_release);
+}
+
+bool CD3D12CommandSlot::Execute()
+{
+  State expected = STATE_RECORDING;
+  if (!m_state.compare_exchange_strong(expected, STATE_SUBMITTED,
+      std::memory_order_acq_rel))
+  {
+    DEBUG_ERROR("Command slot executed while not recording (%ls)", m_name);
+    return false;
+  }
+
+  m_needsReset = true;
+  HRESULT hr = m_gfxList->Close();
+  if (FAILED(hr))
+  {
+    DEBUG_ERROR_HR(hr, "Failed to close the CommandList (%ls)", m_name);
+    m_queue->m_failed.store(true, std::memory_order_release);
+    m_state.store(STATE_FAILED, std::memory_order_release);
+    return false;
+  }
+
+  if (m_queue->Submit(*this))
+    return true;
+
+  if (!m_submitted.load(std::memory_order_acquire))
+    m_state.store(STATE_FREE, std::memory_order_release);
+  return false;
+}
+
+bool CD3D12CommandSlot::WaitFor(ID3D12Fence * fence, UINT64 value)
+{
+  if (!fence || !value ||
+      m_state.load(std::memory_order_acquire) != STATE_RECORDING)
+    return false;
+
+  if (m_fenceWaitCount == MAX_FENCE_WAITS)
+  {
+    DEBUG_ERROR("Too many fence waits for CommandSlot(%ls)", m_name);
+    return false;
+  }
+
+  FenceWait& wait = m_fenceWaits[m_fenceWaitCount++];
+  wait.fence = fence;
+  wait.value = value;
+  return true;
+}
+
+bool CD3D12CommandSlot::WaitFor(const CD3D12CommandSlot& slot)
+{
+  if (!slot.m_queue || !slot.m_fenceTarget)
+    return false;
+
+  return WaitFor(slot.m_queue->m_fence.Get(), slot.m_fenceTarget);
+}
+
+bool CD3D12CommandSlot::BeginTiming()
+{
+  m_timingActive = m_queue && m_queue->m_timingSupported;
+  if (!m_timingActive)
+    return false;
+
+  m_gfxList->EndQuery(m_queue->m_timestampHeap.Get(),
+    D3D12_QUERY_TYPE_TIMESTAMP, m_queryBase);
+  return true;
+}
+
+void CD3D12CommandSlot::EndTiming()
+{
+  if (!m_timingActive)
+    return;
+
+  if (!m_queue->SnapshotTiming(*this))
+  {
+    m_timingActive = false;
+    return;
+  }
+
+  m_gfxList->EndQuery(m_queue->m_timestampHeap.Get(),
+    D3D12_QUERY_TYPE_TIMESTAMP, m_queryBase + 1);
+  m_gfxList->ResolveQueryData(m_queue->m_timestampHeap.Get(),
+    D3D12_QUERY_TYPE_TIMESTAMP, m_queryBase, 2,
+    m_queue->m_timestampReadback.Get(),
+    (UINT64)m_queryBase * sizeof(UINT64));
+}
+
+bool CD3D12CommandSlot::GetGPUTimes(
+  uint64_t& start, uint64_t& end) const
+{
+  return m_queue && m_queue->GetGPUTimes(*this, start, end);
+}
+
+void CD3D12CommandSlot::OnCompletion(bool timeout)
+{
+  if (!m_queue || !m_submitted.load(std::memory_order_acquire))
+    return;
+
+  const UINT64 completed = m_queue->m_fence->GetCompletedValue();
+  if (completed != UINT64_MAX && completed < m_fenceTarget)
+    return;
+
+  State expected = STATE_SUBMITTED;
+  if (!m_state.compare_exchange_strong(expected, STATE_COMPLETING,
+      std::memory_order_acq_rel))
+    return;
+
+  m_completionResult = !timeout && completed != UINT64_MAX;
+  if (!m_completionResult)
+    m_queue->m_failed.store(true, std::memory_order_release);
+
+  if (m_completionCallback)
+    m_completionCallback(this, m_completionResult,
+      m_completionParams[0], m_completionParams[1]);
+
+  m_completionCallback  = nullptr;
+  m_completionParams[0] = nullptr;
+  m_completionParams[1] = nullptr;
+  m_submitted.store(false, std::memory_order_release);
+  m_state.store(STATE_FREE, std::memory_order_release);
+}
+
+bool CD3D12CommandQueue::InitTiming(ID3D12Device3 * device, UINT slotCount)
+{
   D3D12_FEATURE_DATA_D3D12_OPTIONS3 options = {};
   HRESULT hr = device->CheckFeatureSupport(
     D3D12_FEATURE_D3D12_OPTIONS3, &options, sizeof(options));
   if (FAILED(hr) || !options.CopyQueueTimestampQueriesSupported)
     return false;
 
-  hr = m_queue->GetTimestampFrequency(&m_timestampFrequency);
-  if (FAILED(hr) || !m_timestampFrequency)
-    return false;
-
   LARGE_INTEGER qpcFrequency;
   if (!QueryPerformanceFrequency(&qpcFrequency))
     return false;
@@ -56,9 +360,10 @@ bool CD3D12CommandQueue::InitTiming(ID3D12Device3 * device,
 
   D3D12_QUERY_HEAP_DESC queryDesc = {};
   queryDesc.Type  = D3D12_QUERY_HEAP_TYPE_COPY_QUEUE_TIMESTAMP;
-  queryDesc.Count = 2;
+  queryDesc.Count = slotCount * 2;
 
-  hr = device->CreateQueryHeap(&queryDesc, IID_PPV_ARGS(&m_timestampHeap));
+  hr = device->CreateQueryHeap(&queryDesc,
+    IID_PPV_ARGS(&m_timestampHeap));
   if (FAILED(hr))
     return false;
 
@@ -71,7 +376,7 @@ bool CD3D12CommandQueue::InitTiming(ID3D12Device3 * device,
 
   D3D12_RESOURCE_DESC resourceDesc = {};
   resourceDesc.Dimension          = D3D12_RESOURCE_DIMENSION_BUFFER;
-  resourceDesc.Width              = sizeof(UINT64) * 2;
+  resourceDesc.Width              = sizeof(UINT64) * queryDesc.Count;
   resourceDesc.Height             = 1;
   resourceDesc.DepthOrArraySize   = 1;
   resourceDesc.MipLevels          = 1;
@@ -91,7 +396,8 @@ bool CD3D12CommandQueue::InitTiming(ID3D12Device3 * device,
     return false;
   }
 
-  D3D12_RANGE readRange    = { 0, sizeof(UINT64) * 2 };
+  const SIZE_T timingSize  = sizeof(UINT64) * queryDesc.Count;
+  D3D12_RANGE readRange    = { 0, timingSize };
   void      * timestampMap = nullptr;
   hr = m_timestampReadback->Map(0, &readRange, &timestampMap);
   if (FAILED(hr))
@@ -100,55 +406,31 @@ bool CD3D12CommandQueue::InitTiming(ID3D12Device3 * device,
     m_timestampHeap.Reset();
     return false;
   }
+
   m_timestampMap    = static_cast<UINT64 *>(timestampMap);
-
-  hr = m_queue->GetClockCalibration(&m_calibrationGPU, &m_calibrationCPU);
-  if (FAILED(hr))
-  {
-    D3D12_RANGE writeRange = { 0, 0 };
-    m_timestampReadback->Unmap(0, &writeRange);
-    m_timestampMap = nullptr;
-    m_timestampReadback.Reset();
-    m_timestampHeap.Reset();
-    return false;
-  }
-
   m_timingSupported = true;
   return true;
 }
 
-void CD3D12CommandQueue::UpdateClockCalibration()
-{
-  if (!m_timingSupported)
-    return;
-
-  LARGE_INTEGER now;
-  if (QueryPerformanceCounter(&now) &&
-      (UINT64)now.QuadPart >= m_calibrationCPU &&
-      (UINT64)now.QuadPart - m_calibrationCPU < m_qpcFrequency)
-    return;
-
-  UINT64 gpu;
-  UINT64 cpu;
-  if (SUCCEEDED(m_queue->GetClockCalibration(&gpu, &cpu)))
-  {
-    m_calibrationGPU = gpu;
-    m_calibrationCPU = cpu;
-  }
-}
-
 bool CD3D12CommandQueue::Init(ID3D12Device3 * device,
   D3D12_COMMAND_LIST_TYPE type, const WCHAR * name,
-  CallbackMode callbackMode)
+  CD3D12CommandSlot::CallbackMode callbackMode, UINT slotCount,
+  bool enableTiming)
 {
-  HRESULT hr;
+  if (!slotCount || slotCount > MAX_SLOTS)
-  D3D12_COMMAND_QUEUE_DESC queueDesc = {};
+  {
+    DEBUG_ERROR("Invalid slot count for CommandQueue(%ls): %u",
+      name, slotCount);
+    return false;
+  }
 
+  D3D12_COMMAND_QUEUE_DESC queueDesc = {};
   queueDesc.Type     = type;
   queueDesc.Priority = D3D12_COMMAND_QUEUE_PRIORITY_HIGH;
   queueDesc.Flags    = D3D12_COMMAND_QUEUE_FLAG_NONE;
 
-  hr = device->CreateCommandQueue(&queueDesc, IID_PPV_ARGS(&m_queue));
+  HRESULT hr = device->CreateCommandQueue(
+    &queueDesc, IID_PPV_ARGS(&m_queue));
   if (FAILED(hr))
   {
     DEBUG_ERROR_HR(hr, "Failed to create the CommandQueue (%ls)", name);
@@ -156,86 +438,40 @@ bool CD3D12CommandQueue::Init(ID3D12Device3 * device,
   }
   m_queue->SetName(name);
 
-  hr = device->CreateCommandAllocator(type, IID_PPV_ARGS(&m_allocator));
+  hr = device->CreateFence(0, D3D12_FENCE_FLAG_NONE,
+    IID_PPV_ARGS(&m_fence));
   if (FAILED(hr))
   {
-    DEBUG_ERROR_HR(hr, "Failed to create the CommandAllocator (%ls)", name);
+    DEBUG_ERROR_HR(hr, "Failed to create the CommandQueue fence (%ls)", name);
     return false;
   }
 
-  hr = device->CreateCommandList(0, type, m_allocator.Get(), NULL, IID_PPV_ARGS(&m_gfxList));
+  if (enableTiming && !InitTiming(device, slotCount))
-  if (FAILED(hr))
-  {
-    DEBUG_ERROR_HR(hr, "Failed to create the Graphics CommandList (%ls)", name);
-    return false;
-  }
-  m_gfxList->SetName(name);
-
-  m_cmdList = m_gfxList;
-  if (!m_cmdList)
-  {
-    DEBUG_ERROR_HR(hr, "Failed to get the CommandList (%ls)", name);
-    return false;
-  }
-
-  hr = device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(&m_fence));
-  if (FAILED(hr))
-  {
-    DEBUG_ERROR_HR(hr, "Failed to create the Fence (%ls)", name);
-    return false;
-  }
-
-  m_event.Attach(CreateEvent(NULL, FALSE, FALSE, NULL));
-  if (m_event.Get() == INVALID_HANDLE_VALUE)
-  {
-    DEBUG_ERROR_HR(GetLastError(), "Failed to create the completion event (%ls)", name);
-    return false;
-  }
-
-  if (callbackMode != DISABLED)
-  {
-    ULONG flags = (callbackMode == FAST) ?
-      WT_EXECUTEINWAITTHREAD : WT_EXECUTEINPERSISTENTTHREAD;
-
-    if (!RegisterWaitForSingleObject(
-      &m_waitHandle,
-      m_event.Get(),
-      [](PVOID param, BOOLEAN timeout){
-        CD3D12CommandQueue * queue = (CD3D12CommandQueue*)param;
-        if (timeout)
-          queue->m_completionResult = false;
-        queue->OnCompletion();
-      },
-      this,
-      INFINITE,
-      flags))
-    {
-      DEBUG_ERROR_HR(GetLastError(),
-        "Failed to register the completion wait (%ls)", name);
-      m_waitHandle = INVALID_HANDLE_VALUE;
-      return false;
-    }
-  }
-
-  if (callbackMode != DISABLED &&
-      type == D3D12_COMMAND_LIST_TYPE_COPY && !InitTiming(device, type))
     DEBUG_WARN("GPU timing is unavailable for CommandQueue(%ls)", name);
 
+  const ULONG waitFlags = callbackMode == CD3D12CommandSlot::FAST ?
+    WT_EXECUTEINWAITTHREAD : WT_EXECUTEINPERSISTENTTHREAD;
+
   m_name = name;
-  m_fenceValue = 0;
+  for (UINT i = 0; i < slotCount; ++i)
-  DEBUG_INFO("Created CD3D12CommandQueue(%ls)", name);
+  {
+    if (!m_slots[i].Init(device, this, name, i * 2, waitFlags))
+      return false;
+    ++m_slotCount;
+  }
+
+  DEBUG_INFO("Created CD3D12CommandQueue(%ls) with %u slots",
+    name, slotCount);
   return true;
 }
 
 void CD3D12CommandQueue::DeInit()
 {
-  if (m_waitHandle != INVALID_HANDLE_VALUE)
+  WaitForIdle();
-  {
+
-    // Queue owners drain callbacks before destruction. Keep this unregister
+  for (UINT i = 0; i < MAX_SLOTS; ++i)
-    // non-blocking so a removed or hung device cannot stall teardown.
+    m_slots[i].DeInit();
-    UnregisterWait(m_waitHandle);
+  m_slotCount = 0;
-    m_waitHandle = INVALID_HANDLE_VALUE;
-  }
 
   if (m_timestampMap)
   {
@@ -243,142 +479,156 @@ void CD3D12CommandQueue::DeInit()
     m_timestampReadback->Unmap(0, &writeRange);
     m_timestampMap = nullptr;
   }
+
+  m_timestampReadback.Reset();
+  m_timestampHeap.Reset();
+  m_fence.Reset();
+  m_queue.Reset();
+  m_timingSupported = false;
+  m_qpcFrequency    = 0;
 }
 
-bool CD3D12CommandQueue::Execute()
+CD3D12CommandSlot * CD3D12CommandQueue::Acquire(UINT slotIndex)
 {
-  m_needsReset       = true;
+  if (slotIndex >= m_slotCount)
-  m_completionResult = true;
+    return nullptr;
-  m_pending          = m_waitHandle != INVALID_HANDLE_VALUE;
 
-  HRESULT hr = m_gfxList->Close();
+  for (int i = 0; i < 100; ++i)
+  {
+    if (m_slots[slotIndex].Acquire())
+      return &m_slots[slotIndex];
+    if (m_failed.load(std::memory_order_acquire))
+      break;
+    Sleep(1);
+  }
+
+  DEBUG_ERROR("Failed to acquire CommandSlot(%ls:%u)", m_name, slotIndex);
+  return nullptr;
+}
+
+CD3D12CommandSlot * CD3D12CommandQueue::Acquire()
+{
+  return Acquire(0);
+}
+
+void CD3D12CommandQueue::WaitForIdle()
+{
+  for (UINT slot = 0; slot < m_slotCount; ++slot)
+  {
+    while (!m_slots[slot].IsIdle())
+    {
+      // The callback may be delayed behind other thread-pool work. If its
+      // fence has completed, finish it here before releasing callback state.
+      m_slots[slot].OnCompletion(false);
+      if (m_slots[slot].IsIdle())
+        break;
+      Sleep(1);
+    }
+  }
+}
+
+bool CD3D12CommandQueue::Submit(CD3D12CommandSlot& slot)
+{
+  bool result = false;
+  AcquireSRWLockExclusive(&m_submitLock);
+
+  do
+  {
+    if (m_failed.load(std::memory_order_relaxed))
+      break;
+
+    for (UINT i = 0; i < slot.m_fenceWaitCount; ++i)
+    {
+      const CD3D12CommandSlot::FenceWait& wait = slot.m_fenceWaits[i];
+      const HRESULT hr = m_queue->Wait(wait.fence.Get(), wait.value);
       if (FAILED(hr))
       {
-    m_completionResult = false;
+        DEBUG_ERROR_HR(hr, "Failed to queue a fence wait (%ls)", m_name);
-    SetEvent(m_event.Get());
+        m_failed.store(true, std::memory_order_release);
-
+        break;
-    DEBUG_ERROR_HR(hr, "Failed to close the command list (%ls)", m_name);
-    return false;
       }
+    }
+    if (m_failed.load(std::memory_order_relaxed))
+      break;
 
-  ID3D12CommandList * lists[] = { m_cmdList.Get() };
+    const UINT64 fenceTarget = ++m_fenceValue;
+    slot.m_fenceTarget       = fenceTarget;
+    slot.m_submitted.store(true, std::memory_order_release);
+
+    ID3D12CommandList * lists[] = { slot.m_cmdList.Get() };
     m_queue->ExecuteCommandLists(1, lists);
-  ++m_fenceValue;
 
-  hr = m_fence->SetEventOnCompletion(m_fenceValue, m_event.Get());
+    HRESULT hr = m_queue->Signal(m_fence.Get(), fenceTarget);
     if (FAILED(hr))
     {
-    m_completionResult = false;
+      DEBUG_ERROR_HR(hr, "Failed to signal the CommandQueue (%ls)", m_name);
-    SetEvent(m_event.Get());
+      m_failed.store(true, std::memory_order_release);
-
+      slot.OnCompletion(false);
-    DEBUG_ERROR_HR(hr, "Failed to set the fence signal (%ls)", m_name);
+      break;
-    return false;
     }
 
-  m_queue->Signal(m_fence.Get(), m_fenceValue);
+    hr = m_fence->SetEventOnCompletion(fenceTarget, slot.m_event.Get());
+    if (FAILED(hr))
+    {
+      DEBUG_ERROR_HR(hr,
+        "Failed to register CommandSlot completion (%ls)", m_name);
+
+      // The work is already submitted and fenced. Poll only on this rare
+      // error path so allocator, callback, and framebuffer ownership remain
+      // valid until completion or confirmed device removal.
+      while (slot.m_submitted.load(std::memory_order_acquire))
+      {
+        slot.OnCompletion(false);
+        if (slot.m_submitted.load(std::memory_order_acquire))
+          Sleep(1);
+      }
+
+      result = slot.m_completionResult;
+      break;
+    }
+
+    result = true;
+  }
+  while (false);
+
+  ReleaseSRWLockExclusive(&m_submitLock);
+  return result;
+}
+
+bool CD3D12CommandQueue::SnapshotTiming(
+  CD3D12CommandSlot& slot) const
+{
+  UINT64 frequency;
+  UINT64 gpu;
+  UINT64 cpu;
+  if (FAILED(m_queue->GetTimestampFrequency(&frequency)) || !frequency ||
+      FAILED(m_queue->GetClockCalibration(&gpu, &cpu)))
+    return false;
+
+  slot.m_timestampFrequency = frequency;
+  slot.m_calibrationGPU     = gpu;
+  slot.m_calibrationCPU     = cpu;
   return true;
 }
 
-bool CD3D12CommandQueue::BeginTiming()
+bool CD3D12CommandQueue::GetGPUTimes(const CD3D12CommandSlot& slot,
-{
-  m_timingActive = m_timingSupported;
-  if (!m_timingActive)
-    return false;
-
-  // Refresh after an idle period; active queues are calibrated by their
-  // completion path without adding work to frame submission.
-  UpdateClockCalibration();
-  m_gfxList->EndQuery(
-    m_timestampHeap.Get(), D3D12_QUERY_TYPE_TIMESTAMP, 0);
-  return true;
-}
-
-void CD3D12CommandQueue::EndTiming()
-{
-  if (!m_timingActive)
-    return;
-
-  m_gfxList->EndQuery(
-    m_timestampHeap.Get(), D3D12_QUERY_TYPE_TIMESTAMP, 1);
-  m_gfxList->ResolveQueryData(
-    m_timestampHeap.Get(), D3D12_QUERY_TYPE_TIMESTAMP,
-    0, 2, m_timestampReadback.Get(), 0);
-}
-
-bool CD3D12CommandQueue::ConvertGPUTimestamp(
-  UINT64 timestamp, uint64_t& result) const
-{
-  UINT64 cpuTimestamp;
-  if (timestamp < m_calibrationGPU)
-  {
-    const UINT64 delta = ScaleTicks(
-      m_calibrationGPU - timestamp, m_qpcFrequency, m_timestampFrequency);
-    if (delta > m_calibrationCPU)
-      return false;
-    cpuTimestamp = m_calibrationCPU - delta;
-  }
-  else
-  {
-    const UINT64 delta = ScaleTicks(
-      timestamp - m_calibrationGPU, m_qpcFrequency, m_timestampFrequency);
-    if (UINT64_MAX - m_calibrationCPU < delta)
-      return false;
-    cpuTimestamp = m_calibrationCPU + delta;
-  }
-
-  result = TicksToNanoseconds(cpuTimestamp, m_qpcFrequency);
-  return true;
-}
-
-bool CD3D12CommandQueue::GetGPUTimes(
   uint64_t& start, uint64_t& end) const
 {
-  if (!m_timingActive ||
+  if (!slot.m_timingActive || !m_timestampMap ||
-      !ConvertGPUTimestamp(m_timestampMap[0], start) ||
+      !slot.m_timestampFrequency || !m_qpcFrequency)
-      !ConvertGPUTimestamp(m_timestampMap[1], end) ||
+    return false;
+
+  const UINT64 gpuStart = m_timestampMap[slot.m_queryBase];
+  const UINT64 gpuEnd   = m_timestampMap[slot.m_queryBase + 1];
+  if (gpuEnd < gpuStart ||
+      !ConvertGPUTimestamp(gpuStart, slot.m_timestampFrequency,
+        slot.m_calibrationGPU, slot.m_calibrationCPU,
+        m_qpcFrequency, start) ||
+      !ConvertGPUTimestamp(gpuEnd, slot.m_timestampFrequency,
+        slot.m_calibrationGPU, slot.m_calibrationCPU,
+        m_qpcFrequency, end) ||
       end < start)
     return false;
 
   return true;
 }
-
-#if 0
-void CD3D12CommandQueue::Wait()
-{
-  if (m_fence->GetCompletedValue() >= m_fenceValue)
-  {
-    m_pending = false;
-    return;
-  }
-
-  m_fence->SetEventOnCompletion(m_fenceValue, m_event.Get());
-  WaitForSingleObject(m_event.Get(), INFINITE);
-  m_pending = false;
-}
-#endif
-
-bool CD3D12CommandQueue::Reset()
-{
-  m_timingActive = false;
-  if (!m_needsReset)
-    return true;
-
-  HRESULT hr;
-
-  hr = m_allocator->Reset();
-  if (FAILED(hr))
-  {
-    DEBUG_ERROR_HR(hr, "Failed to reset the command allocator (%ls)", m_name);
-    return false;
-  }
-
-  hr = m_gfxList->Reset(m_allocator.Get(), NULL);
-  if (FAILED(hr))
-  {
-    DEBUG_ERROR_HR(hr, "Failed to reset the graphics command list (%ls)", m_name);
-    return false;
-  }
-
-  m_needsReset = false;
-  return true;
-}

idd/LGIdd/CD3D12CommandQueue.h

@@ -31,96 +31,148 @@ using namespace Microsoft::WRL;
 using namespace Microsoft::WRL::Wrappers;
 using namespace Microsoft::WRL::Wrappers::HandleTraits;
 
-class CD3D12CommandQueue
+class CD3D12CommandQueue;
-{
-  private:
-    const WCHAR * m_name = nullptr;
 
-    ComPtr<ID3D12CommandQueue       > m_queue;
+class CD3D12CommandSlot
+{
+  friend class CD3D12CommandQueue;
+
+  private:
+    enum State
+    {
+      STATE_FREE,
+      STATE_RECORDING,
+      STATE_CANCELLING,
+      STATE_SUBMITTED,
+      STATE_COMPLETING,
+      STATE_FAILED,
+    };
+
+    struct FenceWait
+    {
+      ComPtr<ID3D12Fence> fence;
+      UINT64              value = 0;
+    };
+
+    static const UINT MAX_FENCE_WAITS = 2;
+
+    const WCHAR          * m_name  = nullptr;
+    CD3D12CommandQueue   * m_queue = nullptr;
     ComPtr<ID3D12CommandAllocator   > m_allocator;
     ComPtr<ID3D12GraphicsCommandList> m_gfxList;
     ComPtr<ID3D12CommandList        > m_cmdList;
-    ComPtr<ID3D12Fence              > m_fence;
 
-    ComPtr<ID3D12QueryHeap>  m_timestampHeap;
+    std::atomic<State>        m_state       = STATE_FREE;
-    ComPtr<ID3D12Resource >  m_timestampReadback;
+    std::atomic<bool>         m_submitted   = false;
-    UINT64                 * m_timestampMap       = nullptr;
-    UINT64                   m_timestampFrequency = 0;
-    UINT64                   m_calibrationGPU     = 0;
-    UINT64                   m_calibrationCPU     = 0;
-    UINT64                   m_qpcFrequency       = 0;
-    bool                     m_timingSupported    = false;
-    bool                     m_timingActive       = false;
-
-    std::atomic<bool>         m_pending    = false;
     HandleT<HANDLENullTraits> m_event;
     HANDLE                    m_waitHandle  = INVALID_HANDLE_VALUE;
-    UINT64                    m_fenceValue = 0;
     bool                      m_needsReset  = false;
+    UINT64                    m_fenceTarget = 0;
 
-    typedef void (*CompletionFunction)(CD3D12CommandQueue * queue,
+    FenceWait m_fenceWaits[MAX_FENCE_WAITS];
+    UINT      m_fenceWaitCount = 0;
+
+    typedef void (*CompletionFunction)(CD3D12CommandSlot * slot,
       bool result, void * param1, void * param2);
 
     CompletionFunction m_completionCallback  = nullptr;
-    void               * m_completionParams[2];
+    void             * m_completionParams[2] = {};
     bool               m_completionResult    = true;
 
-    bool InitTiming(ID3D12Device3 * device, D3D12_COMMAND_LIST_TYPE type);
+    UINT   m_queryBase          = 0;
-    void UpdateClockCalibration();
+    bool   m_timingActive       = false;
-    bool ConvertGPUTimestamp(UINT64 timestamp, uint64_t& result) const;
+    UINT64 m_timestampFrequency = 0;
+    UINT64 m_calibrationGPU     = 0;
+    UINT64 m_calibrationCPU     = 0;
 
-    void OnCompletion()
+    bool Init(ID3D12Device3 * device, CD3D12CommandQueue * queue,
-    {
+      const WCHAR * name, UINT queryBase, ULONG waitFlags);
-      if (m_completionCallback)
+    void DeInit();
-        m_completionCallback(
+    void OnCompletion(bool timeout);
-          this,
-          m_completionResult,
-          m_completionParams[0],
-          m_completionParams[1]);
-      UpdateClockCalibration();
-      m_pending = false;
-    }
 
   public:
-    ~CD3D12CommandQueue() { DeInit(); }
-
     enum CallbackMode
     {
-      DISABLED, // no callbacks
+      FAST,
-      FAST,     // callback is expected to return almost immediately
+      NORMAL,
-      NORMAL    // normal callback
     };
 
-    bool Init(ID3D12Device3 * device, D3D12_COMMAND_LIST_TYPE type, const WCHAR * name,
+    ~CD3D12CommandSlot() { DeInit(); }
-      CallbackMode callbackMode = DISABLED);
 
-    void DeInit();
+    bool Acquire();
+    void Cancel();
+    bool Execute();
 
-    void SetCompletionCallback(CompletionFunction fn, void * param1, void * param2)
+    bool BeginTiming();
+    void EndTiming();
+    bool GetGPUTimes(uint64_t& start, uint64_t& end) const;
+
+    bool WaitFor(ID3D12Fence * fence, UINT64 value);
+    bool WaitFor(const CD3D12CommandSlot& slot);
+
+    void SetCompletionCallback(CompletionFunction fn,
+      void * param1, void * param2)
     {
       m_completionCallback  = fn;
       m_completionParams[0] = param1;
       m_completionParams[1] = param2;
     }
 
-    bool Reset();
+    bool IsIdle() const
-    bool Execute();
-
-    bool BeginTiming();
-    void EndTiming();
-
-    // Return the copy boundaries in QueryPerformanceCounter-domain ns.
-    bool GetGPUTimes(uint64_t& start, uint64_t& end) const;
-
-    //void Wait();
-    bool   IsReady () const { return !m_pending   ; }
-    HANDLE GetEvent() const { return m_event.Get(); }
-
-    void WaitFor(CD3D12CommandQueue& queue)
     {
-      m_queue->Wait(queue.m_fence.Get(), queue.m_fenceValue);
+      const State state = m_state.load(std::memory_order_acquire);
+      return state == STATE_FREE ||
+        (state == STATE_FAILED &&
+         !m_submitted.load(std::memory_order_acquire));
+    }
+
+    bool HasSubmittedWork() const
+    {
+      return m_submitted.load(std::memory_order_acquire);
     }
 
-    ComPtr<ID3D12CommandQueue       > GetCmdQueue() { return m_queue;   }
     ComPtr<ID3D12GraphicsCommandList> GetGfxList() { return m_gfxList; }
 };
+
+class CD3D12CommandQueue
+{
+  friend class CD3D12CommandSlot;
+
+  private:
+    static const UINT MAX_SLOTS = 2;
+
+    const WCHAR * m_name = nullptr;
+
+    ComPtr<ID3D12CommandQueue> m_queue;
+    ComPtr<ID3D12Fence       > m_fence;
+    UINT64                     m_fenceValue     = 0;
+    SRWLOCK                    m_submitLock     = SRWLOCK_INIT;
+    std::atomic<bool>          m_failed         = false;
+
+    CD3D12CommandSlot m_slots[MAX_SLOTS];
+    UINT              m_slotCount = 0;
+
+    ComPtr<ID3D12QueryHeap> m_timestampHeap;
+    ComPtr<ID3D12Resource > m_timestampReadback;
+    UINT64                * m_timestampMap    = nullptr;
+    UINT64                  m_qpcFrequency    = 0;
+    bool                    m_timingSupported = false;
+
+    bool InitTiming(ID3D12Device3 * device, UINT slotCount);
+    bool Submit(CD3D12CommandSlot& slot);
+    bool SnapshotTiming(CD3D12CommandSlot& slot) const;
+    bool GetGPUTimes(const CD3D12CommandSlot& slot,
+      uint64_t& start, uint64_t& end) const;
+
+  public:
+    ~CD3D12CommandQueue() { DeInit(); }
+
+    bool Init(ID3D12Device3 * device, D3D12_COMMAND_LIST_TYPE type,
+      const WCHAR * name, CD3D12CommandSlot::CallbackMode callbackMode,
+      UINT slotCount, bool enableTiming = false);
+    void DeInit();
+
+    CD3D12CommandSlot * Acquire(UINT slotIndex);
+    CD3D12CommandSlot * Acquire();
+    void WaitForIdle();
+};

idd/LGIdd/CD3D12Device.cpp

@@ -130,14 +130,14 @@ CD3D12Device::InitResult CD3D12Device::Init(CIVSHMEM &ivshmem,
     DEBUG_INFO("Using IVSHMEM as a D3D12Heap");
   }
 
-  for(int i = 0; i < ARRAYSIZE(m_copyQueue); ++i)
+  if (!m_copyQueue.Init(m_device.Get(), D3D12_COMMAND_LIST_TYPE_COPY,
-    if (!m_copyQueue[i].Init(m_device.Get(), D3D12_COMMAND_LIST_TYPE_COPY, L"Copy",
+      L"Copy", m_indirectCopy ? CD3D12CommandSlot::NORMAL :
-        m_indirectCopy ? CD3D12CommandQueue::NORMAL : CD3D12CommandQueue::FAST))
+                               CD3D12CommandSlot::FAST, 2, true))
     return InitResult::FAILURE;
 
   if (m_computeEnabled &&
       !m_computeQueue.Init(m_device.Get(), D3D12_COMMAND_LIST_TYPE_COMPUTE,
-        L"Compute", CD3D12CommandQueue::FAST))
+        L"Compute", CD3D12CommandSlot::FAST, 1))
     return InitResult::FAILURE;
 
   DEBUG_INFO("Created CD3D12Device");
@@ -153,19 +153,9 @@ void CD3D12Device::DeInit()
 
 void CD3D12Device::WaitForIdle()
 {
-  // A queue is ready once its GPU work has signalled and its completion
+  m_copyQueue.WaitForIdle();
-  // callback has run (clearing the pending flag). Bound the wait so a
-  // removed/hung device cannot stall teardown indefinitely.
-  auto drain = [](CD3D12CommandQueue& queue)
-  {
-    for (int i = 0; i < 1000 && !queue.IsReady(); ++i)
-      Sleep(1);
-  };
-
-  for (int i = 0; i < ARRAYSIZE(m_copyQueue); ++i)
-    drain(m_copyQueue[i]);
   if (m_computeEnabled)
-    drain(m_computeQueue);
+    m_computeQueue.WaitForIdle();
 }
 
 bool CD3D12Device::HeapTest()
@@ -212,31 +202,12 @@ bool CD3D12Device::HeapTest()
   return true;
 }
 
-CD3D12CommandQueue * CD3D12Device::GetCopyQueue()
+CD3D12CommandSlot * CD3D12Device::GetCopySlot(unsigned frameIndex)
 {
-  // try for up to a maximum of 100ms to find a free copy queue
+  return m_copyQueue.Acquire(frameIndex);
-  for (int c = 0; c < 100; ++c)
-  {
-    for (int i = 0; i < ARRAYSIZE(m_copyQueue); ++i)
-    {
-      auto& queue = m_copyQueue[m_copyQueueIndex++];
-      if (m_copyQueueIndex == ARRAYSIZE(m_copyQueue))
-        m_copyQueueIndex = 0;
-
-      if (queue.IsReady())
-      {
-        queue.Reset();
-        return &queue;
-      }
-    }
-    Sleep(1);
 }
 
-  DEBUG_ERROR("Failed to get a copy queue");
+CD3D12CommandSlot * CD3D12Device::GetComputeSlot()
-  return nullptr;
-}
-
-CD3D12CommandQueue * CD3D12Device::GetComputeQueue()
 {
   if (!m_computeEnabled)
   {
@@ -244,16 +215,5 @@ CD3D12CommandQueue * CD3D12Device::GetComputeQueue()
     return nullptr;
   }
 
-  for (int c = 0; c < 100; ++c)
+  return m_computeQueue.Acquire();
-  {
-    if (m_computeQueue.IsReady())
-    {
-      m_computeQueue.Reset();
-      return &m_computeQueue;
-    }
-    Sleep(1);
-  }
-
-  DEBUG_ERROR("Failed to get a compute queue");
-  return nullptr;
 }

idd/LGIdd/CD3D12Device.h

@@ -49,8 +49,7 @@ struct CD3D12Device
     ComPtr<ID3D12Device3> m_device;
     ComPtr<ID3D12Heap   > m_ivshmemHeap;
 
-    CD3D12CommandQueue m_copyQueue[4];
+    CD3D12CommandQueue m_copyQueue;
-    unsigned           m_copyQueueIndex = 0;
     CD3D12CommandQueue m_computeQueue;
     bool                m_computeEnabled = false;
 
@@ -80,6 +79,6 @@ struct CD3D12Device
     ComPtr<ID3D12Heap   > GetHeap() { return m_ivshmemHeap; }
     bool IsIndirectCopy() { return m_indirectCopy; }
 
-    CD3D12CommandQueue * GetCopyQueue();
+    CD3D12CommandSlot * GetCopySlot(unsigned frameIndex);
-    CD3D12CommandQueue * GetComputeQueue();
+    CD3D12CommandSlot * GetComputeSlot();
 };

idd/LGIdd/CIndirectDeviceContext.cpp

@@ -1066,8 +1066,12 @@ bool CIndirectDeviceContext::SetupLGMP(size_t alignSize)
     const size_t alignOffset = alignSize - sizeof(FrameBuffer);
     m_frame[i]->offset = (uint32_t)alignOffset;
     m_frameBuffer[i] = (FrameBuffer*)(((uint8_t*)m_frame[i]) + alignOffset);
+    m_frameInFlight[i].store(false, std::memory_order_release);
   }
 
+  m_publishedFrameIndex.store(-1, std::memory_order_release);
+  m_frameResendPending = false;
+
   WDF_TIMER_CONFIG config;
   WDF_TIMER_CONFIG_INIT_PERIODIC(&config,
     [](WDFTIMER timer) -> void
@@ -1101,8 +1105,6 @@ bool CIndirectDeviceContext::SetupLGMP(size_t alignSize)
 
 void CIndirectDeviceContext::DeInitLGMP()
 {
-  m_publishedFrameIndex.store(-1);
-
   // The retry timer callback dereferences this context, so make sure it is
   // stopped and drained before we tear anything down. Wait for any in-flight
   // callback to complete.
@@ -1113,7 +1115,11 @@ void CIndirectDeviceContext::DeInitLGMP()
   }
 
   if (m_lgmp == nullptr)
+  {
+    m_publishedFrameIndex.store(-1, std::memory_order_release);
+    m_frameResendPending = false;
     return;
+  }
 
   if (m_lgmpTimer)
   {
@@ -1121,6 +1127,14 @@ void CIndirectDeviceContext::DeInitLGMP()
     m_lgmpTimer = nullptr;
   }
 
+  AcquireSRWLockExclusive(&m_framePublishLock);
+  m_publishedFrameIndex.store(-1, std::memory_order_release);
+  m_frameResendPending = false;
+  ReleaseSRWLockExclusive(&m_framePublishLock);
+
+  for (int i = 0; i < LGMP_Q_FRAME_LEN; ++i)
+    m_frameInFlight[i].store(false, std::memory_order_release);
+
   for (int i = 0; i < LGMP_Q_FRAME_LEN; ++i)
     lgmpHostMemFree(&m_frameMemory[i]);
   for (int i = 0; i < LGMP_Q_POINTER_LEN; ++i)
@@ -1187,12 +1201,26 @@ void CIndirectDeviceContext::LGMPTimer()
     lgmpHostAckData(m_pointerQueue);
   }
 
-  if (lgmpHostQueueNewSubs(m_frameQueue) && m_monitor)
+  AcquireSRWLockExclusive(&m_framePublishLock);
+  if (lgmpHostQueueNewSubs(m_frameQueue))
+    m_frameResendPending = true;
+
+  if (m_frameResendPending && m_monitor &&
+      lgmpHostQueuePending(m_frameQueue) == 0)
   {
-    const LONG frameIndex = m_publishedFrameIndex.load();
+    const LONG frameIndex =
+      m_publishedFrameIndex.load(std::memory_order_acquire);
     if (frameIndex >= 0)
-      lgmpHostQueuePost(m_frameQueue, 0, m_frameMemory[frameIndex]);
+    {
+      status = lgmpHostQueuePost(
+        m_frameQueue, 0, m_frameMemory[frameIndex]);
+      if (status == LGMP_OK)
+        m_frameResendPending = false;
+      else if (status != LGMP_ERR_QUEUE_FULL)
+        DEBUG_ERROR("Failed to resend frame: %s", lgmpStatusString(status));
     }
+  }
+  ReleaseSRWLockExclusive(&m_framePublishLock);
 
   if (lgmpHostQueueNewSubs(m_pointerQueue))
   {
@@ -1203,8 +1231,16 @@ void CIndirectDeviceContext::LGMPTimer()
 
 bool CIndirectDeviceContext::FrameBufferAvailable() const
 {
-  return m_lgmp && m_frameQueue &&
+  if (!m_lgmp || !m_frameQueue ||
-    lgmpHostQueuePending(m_frameQueue) < LGMP_Q_FRAME_LEN;
+      lgmpHostQueuePending(m_frameQueue) >= LGMP_Q_FRAME_LEN)
+    return false;
+
+  const LONG publishedFrameIndex =
+    m_publishedFrameIndex.load(std::memory_order_acquire);
+  const unsigned frameIndex =
+    static_cast<unsigned>(publishedFrameIndex + 1) % LGMP_Q_FRAME_LEN;
+
+  return !m_frameInFlight[frameIndex].load(std::memory_order_acquire);
 }
 
 CIndirectDeviceContext::PreparedFrameBuffer CIndirectDeviceContext::PrepareFrameBuffer(
@@ -1216,6 +1252,22 @@ CIndirectDeviceContext::PreparedFrameBuffer CIndirectDeviceContext::PrepareFrame
   if (!FrameBufferAvailable())
     return result;
 
+  if (dstFormat.format == FRAME_TYPE_INVALID)
+  {
+    DEBUG_ERROR("Unsupported frame format, skipping frame");
+    return result;
+  }
+
+  const LONG publishedFrameIndex =
+    m_publishedFrameIndex.load(std::memory_order_acquire);
+  const unsigned frameIndex =
+    static_cast<unsigned>(publishedFrameIndex + 1) % LGMP_Q_FRAME_LEN;
+
+  bool expected = false;
+  if (!m_frameInFlight[frameIndex].compare_exchange_strong(
+      expected, true, std::memory_order_acq_rel))
+    return result;
+
   if (m_width     != dstFormat.desc.Width  ||
       m_height    != dstFormat.desc.Height ||
       m_pitch     != pitch                 ||
@@ -1265,16 +1317,7 @@ CIndirectDeviceContext::PreparedFrameBuffer CIndirectDeviceContext::PrepareFrame
   m_lastHDRMaxFrameAverageLightLevel = dstFormat.maxFrameAverageLightLevel;
   m_lastSDRWhiteLevel                = dstFormat.sdrWhiteLevel;
 
-  if (++m_frameIndex == LGMP_Q_FRAME_LEN)
+  KVMFRFrame * fi = m_frame[frameIndex];
-    m_frameIndex = 0;
-
-  KVMFRFrame * fi = m_frame[m_frameIndex];
-
-  if (dstFormat.format == FRAME_TYPE_INVALID)
-  {
-    DEBUG_ERROR("Unsupported frame format, skipping frame");
-    return result;
-  }
 
   const unsigned maxRows = (unsigned)(m_maxFrameSize / pitch);
   const int bpp = dstFormat.format == FRAME_TYPE_RGBA16F ? 8 : 4;
@@ -1340,10 +1383,10 @@ CIndirectDeviceContext::PreparedFrameBuffer CIndirectDeviceContext::PrepareFrame
     }
   }
 
-  FrameBuffer* fb = m_frameBuffer[m_frameIndex];
+  FrameBuffer* fb = m_frameBuffer[frameIndex];
   fb->wp = 0;
 
-  result.frameIndex = m_frameIndex;
+  result.frameIndex = frameIndex;
   result.mem        = fb->data;
 
   return result;
@@ -1354,13 +1397,17 @@ bool CIndirectDeviceContext::PublishFrameBuffer(unsigned frameIndex)
   if (!m_frameQueue || frameIndex >= LGMP_Q_FRAME_LEN)
     return false;
 
-  /* Make resends select this submitted frame before posting it. This prevents
+  AcquireSRWLockExclusive(&m_framePublishLock);
-   * a new subscriber racing publication from receiving the previous frame
-   * after the new one. */
-  m_publishedFrameIndex.store(static_cast<LONG>(frameIndex));
-
   const LGMP_STATUS status =
     lgmpHostQueuePost(m_frameQueue, 0, m_frameMemory[frameIndex]);
+  if (status == LGMP_OK)
+  {
+    m_publishedFrameIndex.store(
+      static_cast<LONG>(frameIndex), std::memory_order_release);
+    m_frameResendPending = false;
+  }
+  ReleaseSRWLockExclusive(&m_framePublishLock);
+
   if (status != LGMP_OK)
   {
     DEBUG_ERROR("Failed to publish frame: %s", lgmpStatusString(status));
@@ -1370,6 +1417,32 @@ bool CIndirectDeviceContext::PublishFrameBuffer(unsigned frameIndex)
   return true;
 }
 
+void CIndirectDeviceContext::AbortFrameBuffer(unsigned frameIndex)
+{
+  if (frameIndex >= LGMP_Q_FRAME_LEN)
+    return;
+
+  m_frameBuffer[frameIndex]->wp = 0;
+  InterlockedExchange((volatile LONG *)&m_frame[frameIndex]->timingValid, 0);
+  m_frameInFlight[frameIndex].store(false, std::memory_order_release);
+}
+
+void CIndirectDeviceContext::FailFrameBuffer(unsigned frameIndex)
+{
+  if (frameIndex >= LGMP_Q_FRAME_LEN)
+    return;
+
+  InterlockedExchange((volatile LONG *)&m_frame[frameIndex]->timingValid, 0);
+  FinalizeFrameBuffer(frameIndex);
+  CompleteFrameBuffer(frameIndex);
+}
+
+void CIndirectDeviceContext::CompleteFrameBuffer(unsigned frameIndex)
+{
+  if (frameIndex < LGMP_Q_FRAME_LEN)
+    m_frameInFlight[frameIndex].store(false, std::memory_order_release);
+}
+
 void CIndirectDeviceContext::SetFrameTiming(unsigned frameIndex,
   uint64_t captureTime, uint64_t postProcessTime, uint64_t copyTime,
   uint64_t readyTime)
@@ -1401,8 +1474,9 @@ void CIndirectDeviceContext::WriteFrameBuffer(unsigned frameIndex, void* src, si
 
 void CIndirectDeviceContext::FinalizeFrameBuffer(unsigned frameIndex) const
 {
+  const KVMFRFrame * frame = m_frame[frameIndex];
   FrameBuffer      * fb    = m_frameBuffer[frameIndex];
-  fb->wp = m_height * m_pitch;
+  fb->wp = frame->dataHeight * frame->pitch;
 }
 
 void CIndirectDeviceContext::SendCursor(const IDARG_OUT_QUERY_HWCURSOR& info,

idd/LGIdd/CIndirectDeviceContext.h

@@ -103,8 +103,10 @@ private:
   size_t            m_alignSize           = 0;
   size_t            m_frameMemoryOffset   = 0;
   size_t            m_maxFrameSize        = 0;
-  int               m_frameIndex          = 0;
   std::atomic<LONG> m_publishedFrameIndex = -1;
+  std::atomic<bool> m_frameInFlight[LGMP_Q_FRAME_LEN] = {};
+  SRWLOCK           m_framePublishLock    = SRWLOCK_INIT;
+  bool              m_frameResendPending  = false;
   uint32_t          m_formatVer           = 0;
   uint32_t          m_frameSerial         = 0;
   PLGMPMemory       m_frameMemory[LGMP_Q_FRAME_LEN] = {};
@@ -217,6 +219,9 @@ public:
   bool FrameBufferAvailable() const;
   PreparedFrameBuffer PrepareFrameBuffer(unsigned pitch, const D12FrameFormat& srcFormat, const D12FrameFormat& dstFormat, const RECT * dirtyRects, unsigned nbDirtyRects);
   bool PublishFrameBuffer(unsigned frameIndex);
+  void AbortFrameBuffer(unsigned frameIndex);
+  void FailFrameBuffer(unsigned frameIndex);
+  void CompleteFrameBuffer(unsigned frameIndex);
   void SetFrameTiming(unsigned frameIndex, uint64_t captureTime,
     uint64_t postProcessTime, uint64_t copyTime, uint64_t readyTime);
   void WriteFrameBuffer(unsigned frameIndex, void* src, size_t offset, size_t len, bool setWritePos) const;

idd/LGIdd/CInteropResource.cpp

@@ -126,16 +126,33 @@ bool CInteropResource::Compare(const ComPtr<ID3D11Texture2D>& srcTex)
     m_format.Format == format.Format;
 }
 
-void CInteropResource::Signal()
+bool CInteropResource::Signal()
 {
-  ++m_fenceValue;
+  const UINT64 fenceValue = m_fenceValue + 1;
-  m_dx11Device->GetContext()->Signal(m_d11Fence.Get(), m_fenceValue);
+  const HRESULT hr =
+    m_dx11Device->GetContext()->Signal(m_d11Fence.Get(), fenceValue);
+  if (FAILED(hr))
+  {
+    DEBUG_ERROR_HR(hr, "Failed to signal the D3D11 source fence");
+    return false;
   }
 
-void CInteropResource::Sync(CD3D12CommandQueue& queue)
+  m_fenceValue = fenceValue;
+  return true;
+}
+
+bool CInteropResource::Sync(CD3D12CommandSlot& slot)
 {
-  if (m_d11Fence->GetCompletedValue() < m_fenceValue)
+  if (m_d11Fence->GetCompletedValue() >= m_fenceValue)
-    queue.GetCmdQueue()->Wait(m_d12Fence.Get(), m_fenceValue);
+    return true;
+
+  if (!slot.WaitFor(m_d12Fence.Get(), m_fenceValue))
+  {
+    DEBUG_ERROR("Failed to queue a wait for the D3D11 source fence");
+    return false;
+  }
+
+  return true;
 }
 
 void CInteropResource::SetFullDamage()

idd/LGIdd/CInteropResource.h

@@ -59,8 +59,8 @@ class CInteropResource
 
     bool IsReady() { return m_ready; }
     bool Compare(const ComPtr<ID3D11Texture2D>& srcTex);
-    void Signal();
+    bool Signal();
-    void Sync(CD3D12CommandQueue& queue);
+    bool Sync(CD3D12CommandSlot& slot);
     void SetFullDamage();
     void SetDirtyRects(const RECT * dirtyRects, unsigned nbDirtyRects);
 

idd/LGIdd/CSwapChainProcessor.cpp

@@ -316,23 +316,31 @@ bool CSwapChainProcessor::SwapChainThreadCore()
 }
 
 void CSwapChainProcessor::CompletionFunction(
-  CD3D12CommandQueue * queue, bool result, void * param1, void * param2)
+  CD3D12CommandSlot * slot, bool result, void * param1, void * param2)
 {
   auto sc    = (CSwapChainProcessor   *)param1;
   auto fbRes = (CFrameBufferResource *)param2;
 
+  if (!result)
+  {
+    // A submitted frame may already be in LGMP, or publication may race this
+    // callback. Make the message releasable even though its contents failed.
+    sc->m_devContext->FailFrameBuffer(fbRes->GetFrameIndex());
+    return;
+  }
+
   const uint64_t cpuCopyStart = fbRes->GetCopyStart();
   uint64_t       gpuCopyStart = 0;
   uint64_t       gpuCopyEnd   = 0;
 
-  if (result && sc->m_dx12Device->IsIndirectCopy())
+  if (sc->m_dx12Device->IsIndirectCopy())
     sc->m_devContext->WriteFrameBuffer(
       fbRes->GetFrameIndex(), fbRes->GetMap(), 0, fbRes->GetFrameSize(), false);
 
   // Queue waits execute before the start timestamp. The end timestamp follows
   // the last CopyTextureRegion, separating GPU work from readiness dispatch.
-  const bool gpuTimingValid = result &&
+  const bool gpuTimingValid =
-    queue->GetGPUTimes(gpuCopyStart, gpuCopyEnd);
+    slot->GetGPUTimes(gpuCopyStart, gpuCopyEnd);
 
   // Publish readiness before sampling the endpoint. Timing has its own valid
   // flag and is published immediately afterwards.
@@ -353,6 +361,7 @@ void CSwapChainProcessor::CompletionFunction(
 
   sc->m_devContext->SetFrameTiming(fbRes->GetFrameIndex(),
     fbRes->GetCaptureTime(), postProcessTime, copyTime, readyTime);
+  sc->m_devContext->CompleteFrameBuffer(fbRes->GetFrameIndex());
 }
 
 
@@ -652,7 +661,11 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr<IDXGIResource> acquiredBuffer
    * use a fence. Because we share this texture with DirectX12 it is able to
    * read from it before the desktop duplication API has finished updating it.
    */
-  srcRes->Signal();
+  if (!srcRes->Signal())
+  {
+    SetFullPendingDamage();
+    return false;
+  }
 
   RECT dirtyRects[LG_MAX_DIRTY_RECTS] = {0};
   bool noImageUpdate = false;
@@ -819,38 +832,34 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr<IDXGIResource> acquiredBuffer
   unsigned frameDirtyRectCount = nbDirtyRects;
   m_postProcessor.AdjustFrameDamage(frameDirtyRects, &frameDirtyRectCount);
 
-  auto copyQueue = m_dx12Device->GetCopyQueue();
-  if (!copyQueue)
-  {
-    DEBUG_ERROR("Failed to get a CopyQueue");
-    return false;
-  }
-
   ComPtr<ID3D12Resource> copySrcResource = srcRes->GetRes();
-  CD3D12CommandQueue * computeQueue = nullptr;
+  CD3D12CommandSlot * computeSlot = nullptr;
   if (m_postProcessor.HasActiveEffects())
   {
-    computeQueue = m_dx12Device->GetComputeQueue();
+    computeSlot = m_dx12Device->GetComputeSlot();
-    if (!computeQueue)
+    if (!computeSlot)
     {
-      DEBUG_ERROR("Failed to get a ComputeQueue");
+      DEBUG_ERROR("Failed to get a compute CommandSlot");
+      return false;
+    }
+
+    if (!srcRes->Sync(*computeSlot))
+    {
+      computeSlot->Cancel();
+      SetFullPendingDamage();
       return false;
     }
 
-    srcRes->Sync(*computeQueue);
     copySrcResource = m_postProcessor.Run(
-      computeQueue->GetGfxList(), copySrcResource,
+      computeSlot->GetGfxList(), copySrcResource,
       currentDirtyRects, &nbDirtyRects);
 
-    if (!computeQueue->Execute())
+    if (!computeSlot->Execute())
     {
       SetFullPendingDamage();
       return false;
     }
-    copyQueue->WaitFor(*computeQueue);
   }
-  else
-    srcRes->Sync(*copyQueue);
 
   ClipDirtyRects(currentDirtyRects, &nbDirtyRects, dstFormat.desc);
 
@@ -861,8 +870,8 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr<IDXGIResource> acquiredBuffer
     frameDirtyRects,
     frameDirtyRectCount);
 
-  // The LGMP timer can fill the queue with a subscriber resend after the early
+  // Queue or framebuffer ownership can change after the early availability
-  // availability check. Treat this as a dropped frame rather than an error.
+  // check. Treat this as a dropped frame rather than an error.
   if (!buffer.mem)
     return true;
 
@@ -871,15 +880,37 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr<IDXGIResource> acquiredBuffer
 
   if (!fbRes)
   {
+    m_devContext->AbortFrameBuffer(buffer.frameIndex);
     DEBUG_ERROR("Failed to get a CFrameBufferResource from the pool");
     SetFullPendingDamage();
     return false;
   }
 
+  CD3D12CommandSlot * copySlot =
+    m_dx12Device->GetCopySlot(buffer.frameIndex);
+  if (!copySlot)
+  {
+    m_devContext->AbortFrameBuffer(buffer.frameIndex);
+    DEBUG_ERROR("Failed to get a copy CommandSlot");
+    SetFullPendingDamage();
+    return false;
+  }
+
+  const bool syncResult = computeSlot ?
+    copySlot->WaitFor(*computeSlot) : srcRes->Sync(*copySlot);
+  if (!syncResult)
+  {
+    copySlot->Cancel();
+    m_devContext->AbortFrameBuffer(buffer.frameIndex);
+    DEBUG_ERROR("Failed to queue copy synchronization");
+    SetFullPendingDamage();
+    return false;
+  }
+
   const uint64_t copyStart = Nanotime();
   fbRes->SetTiming(captureTime, postProcessStart, copyStart);
 
-  copyQueue->SetCompletionCallback(&CompletionFunction, this, fbRes);
+  copySlot->SetCompletionCallback(&CompletionFunction, this, fbRes);
 
   D3D12_TEXTURE_COPY_LOCATION srcLoc = {};
   srcLoc.pResource        = copySrcResource.Get();
@@ -924,24 +955,26 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr<IDXGIResource> acquiredBuffer
           copyDirtyRects, nbCopyDirtyRects, dstFormat.desc);
   }
 
-  // The source/compute waits were inserted directly on the queue above. The
+  // Source/compute waits are submitted immediately before this command list.
-  // command-list timestamp therefore marks the first actual copy operation.
+  // The timestamp therefore marks the first actual copy operation.
-  copyQueue->BeginTiming();
+  copySlot->BeginTiming();
   if (fullCopy)
   {
-    copyQueue->GetGfxList()->CopyTextureRegion(
+    copySlot->GetGfxList()->CopyTextureRegion(
       &dstLoc, 0, 0, 0, &srcLoc, NULL);
   }
   else
   {
     for (const RECT * rect = copyDirtyRects;
          rect < copyDirtyRects + nbCopyDirtyRects; ++rect)
-      CopyDirtyRect(copyQueue->GetGfxList(), &dstLoc, &srcLoc, *rect);
+      CopyDirtyRect(copySlot->GetGfxList(), &dstLoc, &srcLoc, *rect);
   }
-  copyQueue->EndTiming();
+  copySlot->EndTiming();
 
-  if (!copyQueue->Execute())
+  if (!copySlot->Execute())
   {
+    if (!copySlot->HasSubmittedWork())
+      m_devContext->AbortFrameBuffer(buffer.frameIndex);
     SetFullPendingDamage();
     return false;
   }

idd/LGIdd/CSwapChainProcessor.h

@@ -94,7 +94,7 @@ private:
   void CursorThread();
 
   static void CompletionFunction(
-    CD3D12CommandQueue * queue, bool result, void * param1, void * param2);
+    CD3D12CommandSlot * slot, bool result, void * param1, void * param2);
   void AccumulateFrameDamage(const RECT * dirtyRects, unsigned nbDirtyRects);
   void SetFullPendingDamage();
 #ifdef HAS_IDDCX_110

idd/LGIdd/effect/CColorTransformEffect.cpp

@@ -280,7 +280,7 @@ ComPtr<ID3D12Resource> CColorTransformEffect::Run(
   UNREFERENCED_PARAMETER(dirtyRects);
   UNREFERENCED_PARAMETER(nbDirtyRects);
 
-  // GetComputeQueue waits for the previous submission before Run is called,
+  // GetComputeSlot waits for the previous submission before Run is called,
   // so this is the first point where the shared upload buffers are guaranteed
   // not to be in use by the GPU.
   if (m_uploadPending)