[idd] driver: very experimental change to syncronization

This may get reverted, at this point it's an experiment for the
testers to trial.

idd/LGIdd/CD3D12CommandQueue.cpp

@@ -54,12 +54,32 @@ bool CD3D12CommandQueue::Init(ID3D12Device3 * device, D3D12_COMMAND_LIST_TYPE ty
     return false;
   }
 
+  RegisterWaitForSingleObject(
+    &m_waitHandle,
+    m_event.Get(),
+    [](PVOID param, BOOLEAN timeout){
+      if (!timeout)
+        ((CD3D12CommandQueue*)param)->OnCompletion();
+    },
+    this,
+    INFINITE,
+    WT_EXECUTEINPERSISTENTTHREAD);
+
   m_name       = name;
   m_fenceValue = 0;
   DEBUG_INFO("Created CD3D12CommandQueue(%ls)", name);
   return true;
 }
 
+void CD3D12CommandQueue::DeInit()
+{
+  if (m_waitHandle != INVALID_HANDLE_VALUE)
+  {
+    UnregisterWait(m_waitHandle);
+    m_waitHandle = INVALID_HANDLE_VALUE;
+  }
+}
+
 bool CD3D12CommandQueue::Execute()
 {
   HRESULT hr = m_gfxList->Close();
@@ -79,17 +99,25 @@ bool CD3D12CommandQueue::Execute()
     return false;
   }
 
+  m_fence->SetEventOnCompletion(m_fenceValue, m_event.Get());
+  m_pending = true;
   return true;
 }
 
+#if 0
 void CD3D12CommandQueue::Wait()
 {
   if (m_fence->GetCompletedValue() >= m_fenceValue)
+  {
+    m_pending = false;
     return;
+  }
 
   m_fence->SetEventOnCompletion(m_fenceValue, m_event.Get());
   WaitForSingleObject(m_event.Get(), INFINITE);
+  m_pending = false;
 }
+#endif
 
 bool CD3D12CommandQueue::Reset()
 {

idd/LGIdd/CD3D12CommandQueue.h

@@ -4,8 +4,11 @@
 #include <wdf.h>
 #include <wrl.h>
 #include <d3d12.h>
+#include <functional>
 
 using namespace Microsoft::WRL;
+using namespace Microsoft::WRL::Wrappers;
+using namespace Microsoft::WRL::Wrappers::HandleTraits;
 
 class CD3D12CommandQueue
 {
@@ -18,19 +21,46 @@ class CD3D12CommandQueue
     ComPtr<ID3D12CommandList        > m_cmdList;
     ComPtr<ID3D12Fence              > m_fence;
 
-    Wrappers::HandleT<Wrappers::HandleTraits::HANDLENullTraits> m_event;
+    bool m_pending;
+    HandleT<HANDLENullTraits> m_event;
+    HANDLE m_waitHandle = INVALID_HANDLE_VALUE;
     UINT64 m_fenceValue = 0;
 
+    typedef std::function<void(CD3D12CommandQueue * queue, void * param1, void * param2)> CompletionFunction;
+    CompletionFunction m_completionCallback = nullptr;
+    void * m_completionParams[2];
+
+    bool Reset();
+    void OnCompletion()
+    {
+      Reset();
+      m_pending = false;
+      if (m_completionCallback)
+        m_completionCallback(
+          this,
+          m_completionParams[0],
+          m_completionParams[1]);
+    }
+
   public:
+    ~CD3D12CommandQueue() { DeInit(); }
+
     bool Init(ID3D12Device3 * device, D3D12_COMMAND_LIST_TYPE type, const WCHAR * name);
+    void DeInit();
+
+    void SetCompletionCallback(CompletionFunction fn, void * param1, void * param2)
+    {
+      m_completionCallback  = fn;
+      m_completionParams[0] = param1;
+      m_completionParams[1] = param2;
+    }
 
     bool Execute();
 
-    void Wait();
+    //void Wait();
-
+    bool IsReady() { return !m_pending; }
-    bool Reset();
+    HANDLE GetEvent() { return m_event.Get(); }
 
     ComPtr<ID3D12CommandQueue       > GetCmdQueue() { return m_queue;   }
     ComPtr<ID3D12GraphicsCommandList> GetGfxList()  { return m_gfxList; }
-};
+};
-

idd/LGIdd/CD3D12Device.cpp

@@ -107,8 +107,9 @@ CD3D12Device::InitResult CD3D12Device::Init(CIVSHMEM &ivshmem, UINT64 &alignSize
     DEBUG_INFO("Using IVSHMEM as a D3D12Heap");
   }
 
-  if (!m_copyQueue.Init(m_device.Get(), D3D12_COMMAND_LIST_TYPE_COPY, L"Copy"))
+  for(int i = 0; i < ARRAYSIZE(m_copyQueue); ++i)
-    return InitResult::FAILURE;
+    if (!m_copyQueue[i].Init(m_device.Get(), D3D12_COMMAND_LIST_TYPE_COPY, L"Copy"))
+      return InitResult::FAILURE;
 
   //if (!m_computeQueue.Init(m_device.Get(), D3D12_COMMAND_LIST_TYPE_COMPUTE, L"Compute"))
     //return InitResult::FAILURE;
@@ -167,3 +168,24 @@ bool CD3D12Device::HeapTest()
 
   return true;
 }
+
+CD3D12CommandQueue * CD3D12Device::GetCopyQueue()
+{
+  HANDLE waitOn[ARRAYSIZE(m_copyQueue)];
+
+  for (int i = 0; i < ARRAYSIZE(m_copyQueue); ++i)
+  {
+    auto& queue = m_copyQueue[i];
+    if (queue.IsReady())
+      return &queue;
+
+    waitOn[i] = queue.GetEvent();
+  }
+
+  DEBUG_TRACE("Wait");
+  DWORD ready = WaitForMultipleObjects(ARRAYSIZE(waitOn), waitOn, FALSE, INFINITE);
+  if (ready < ARRAYSIZE(waitOn))
+    return &m_copyQueue[ready];
+
+  return nullptr;
+}

idd/LGIdd/CD3D12Device.h

@@ -29,7 +29,7 @@ struct CD3D12Device
     ComPtr<ID3D12Device3> m_device;
     ComPtr<ID3D12Heap   > m_ivshmemHeap;
 
-    CD3D12CommandQueue m_copyQueue;
+    CD3D12CommandQueue m_copyQueue[2];
     CD3D12CommandQueue m_computeQueue;
 
     bool HeapTest();
@@ -52,6 +52,6 @@ struct CD3D12Device
     ComPtr<ID3D12Heap   > GetHeap() { return m_ivshmemHeap; }
     bool IsIndirectCopy() { return m_indirectCopy; }
 
-    CD3D12CommandQueue& GetCopyQueue() { return m_copyQueue; }
+    CD3D12CommandQueue * GetCopyQueue();
-    CD3D12CommandQueue& GetComputeQueue() { return m_computeQueue; }
+    CD3D12CommandQueue & GetComputeQueue() { return m_computeQueue; }
 };

idd/LGIdd/CFrameBufferPool.cpp

@@ -25,7 +25,7 @@ CFrameBufferResource * CFrameBufferPool::Get(
   if (!fbr->IsValid() || fbr->GetBase() != buffer.mem || fbr->GetSize() < minSize)
   {
     fbr->Reset();
-    if (!fbr->Init(m_swapChain, buffer.mem, minSize))
+    if (!fbr->Init(m_swapChain, buffer.frameIndex, buffer.mem, minSize))
       return nullptr;
   }
 

idd/LGIdd/CFrameBufferResource.cpp

@@ -2,8 +2,10 @@
 #include "CSwapChainProcessor.h"
 #include "CDebug.h"
 
-bool CFrameBufferResource::Init(CSwapChainProcessor * swapChain, uint8_t * base, size_t size)
+bool CFrameBufferResource::Init(CSwapChainProcessor * swapChain, unsigned frameIndex, uint8_t * base, size_t size)
 {
+  m_frameIndex = frameIndex;
+
   if (size > swapChain->GetDevice()->GetMaxFrameSize())
   {
     DEBUG_ERROR("Frame size of %lu is too large to fit in available shared ram");

idd/LGIdd/CFrameBufferResource.h

@@ -14,6 +14,7 @@ class CFrameBufferResource
 {
   private:
     bool                   m_valid;
+    unsigned               m_frameIndex;
     uint8_t              * m_base;
     size_t                 m_size;
     size_t                 m_frameSize;
@@ -21,14 +22,15 @@ class CFrameBufferResource
     void                 * m_map;
 
   public:
-    bool Init(CSwapChainProcessor * swapChain, uint8_t * base, size_t size);
+    bool Init(CSwapChainProcessor * swapChain, unsigned frameIndex, uint8_t * base, size_t size);
     void Reset();
 
-    bool      IsValid()      { return m_valid;     }
+    bool      IsValid()       { return m_valid;      }
-    uint8_t * GetBase()      { return m_base;      }
+    unsigned  GetFrameIndex() { return m_frameIndex; }
-    size_t    GetSize()      { return m_size;      }
+    uint8_t * GetBase()       { return m_base;       }
-    size_t    GetFrameSize() { return m_frameSize; }
+    size_t    GetSize()       { return m_size;       }
-    void *    GetMap()       { return m_map;       }
+    size_t    GetFrameSize()  { return m_frameSize;  }
+    void *    GetMap()        { return m_map;        }
 
     ComPtr<ID3D12Resource> Get() { return m_res; }
 };

idd/LGIdd/CIndirectDeviceContext.cpp

@@ -737,9 +737,9 @@ CIndirectDeviceContext::PreparedFrameBuffer CIndirectDeviceContext::PrepareFrame
   return result;
 }
 
-void CIndirectDeviceContext::WriteFrameBuffer(void* src, size_t offset, size_t len, bool setWritePos)
+void CIndirectDeviceContext::WriteFrameBuffer(unsigned frameIndex, void* src, size_t offset, size_t len, bool setWritePos)
 {
-  KVMFRFrame  * fi = (KVMFRFrame*)lgmpHostMemPtr(m_frameMemory[m_frameIndex]);
+  KVMFRFrame  * fi = (KVMFRFrame*)lgmpHostMemPtr(m_frameMemory[frameIndex]);
   FrameBuffer * fb = (FrameBuffer*)(((uint8_t*)fi) + fi->offset);
 
   memcpy(
@@ -751,9 +751,9 @@ void CIndirectDeviceContext::WriteFrameBuffer(void* src, size_t offset, size_t l
     fb->wp = (uint32_t)(offset + len);
 }
 
-void CIndirectDeviceContext::FinalizeFrameBuffer()
+void CIndirectDeviceContext::FinalizeFrameBuffer(unsigned frameIndex)
 {
-  KVMFRFrame  * fi = (KVMFRFrame*)lgmpHostMemPtr(m_frameMemory[m_frameIndex]);
+  KVMFRFrame  * fi = (KVMFRFrame*)lgmpHostMemPtr(m_frameMemory[frameIndex]);
   FrameBuffer * fb = (FrameBuffer *)(((uint8_t*)fi) + fi->offset);
   fb->wp = m_height * m_pitch;
 }

idd/LGIdd/CIndirectDeviceContext.h

@@ -121,8 +121,8 @@ public:
   };
 
   PreparedFrameBuffer PrepareFrameBuffer(int width, int height, int pitch, DXGI_FORMAT format);
-  void WriteFrameBuffer(void* src, size_t offset, size_t len, bool setWritePos);
+  void WriteFrameBuffer(unsigned frameIndex, void* src, size_t offset, size_t len, bool setWritePos);
-  void FinalizeFrameBuffer();
+  void FinalizeFrameBuffer(unsigned frameIndex);
 
   void SendCursor(const IDARG_OUT_QUERY_HWCURSOR & info, const BYTE * data);
 

idd/LGIdd/CSwapChainProcessor.cpp

@@ -205,11 +205,35 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr<IDXGIResource> acquiredBuffer
   dstLoc.PlacedFootprint.Footprint.Depth    = 1;
   dstLoc.PlacedFootprint.Footprint.RowPitch = srcRes->GetFormat().Width * 4; //FIXME
 
-  srcRes->Sync(m_dx12Device->GetCopyQueue());
+  auto copyQueue = m_dx12Device->GetCopyQueue();
-  m_dx12Device->GetCopyQueue().GetGfxList()->CopyTextureRegion(
+  if (!copyQueue)
-    &dstLoc, 0, 0, 0, &srcLoc, NULL);
+  {
+    DEBUG_ERROR("Failed to get a CopyQueue");
+    return false;
+  }
 
-  m_dx12Device->GetCopyQueue().Execute();
+  copyQueue->SetCompletionCallback(
+    [](CD3D12CommandQueue * copyQueue, void * param1, void * param2)
+    {
+      UNREFERENCED_PARAMETER(copyQueue);
+
+      auto sc    = (CSwapChainProcessor  *)param1;
+      auto fbRes = (CFrameBufferResource *)param2;
+
+      if (sc->m_dx12Device->IsIndirectCopy())
+        sc->m_devContext->WriteFrameBuffer(
+          fbRes->GetFrameIndex(),
+          fbRes->GetMap(), 0, fbRes->GetFrameSize(), true);
+      else
+        sc->m_devContext->FinalizeFrameBuffer(fbRes->GetFrameIndex());
+    },
+    this,
+    fbRes);
+
+  srcRes->Sync(*copyQueue);
+  copyQueue->GetGfxList()->CopyTextureRegion(
+    &dstLoc, 0, 0, 0, &srcLoc, NULL);
+  copyQueue->Execute();
 
   // report that all GPU processing for this frame has been queued
   hr = IddCxSwapChainFinishedProcessingFrame(m_hSwapChain);
@@ -219,16 +243,5 @@ bool CSwapChainProcessor::SwapChainNewFrame(ComPtr<IDXGIResource> acquiredBuffer
     return false;
   }
 
-  // wait for the completion
-  m_dx12Device->GetCopyQueue().Wait();
-  m_dx12Device->GetCopyQueue().Reset();
-
-  // copy/finialize
-  if (m_dx12Device->IsIndirectCopy())
-    m_devContext->WriteFrameBuffer(
-      fbRes->GetMap(), 0, fbRes->GetFrameSize(), true);
-  else
-    m_devContext->FinalizeFrameBuffer();
-
   return true;
 }