[idd] transport: fan out frame publication

idd/LGIdd/capture/CFrameBufferPool.cpp

@@ -19,14 +19,11 @@
  */
 
 #include "capture/CFrameBufferPool.h"
-#include "transport/IFrameTransport.h"
-
 #include <stdint.h>
 
 void CFrameBufferPool::Init(
-  IFrameTransport * transport, CD3D12Device * dx12)
+  IFrameTransport *, CD3D12Device * dx12)
 {
-  m_transport = transport;
   m_dx12 = dx12;
 }
 
@@ -44,7 +41,7 @@ CFrameBufferResource * CFrameBufferPool::Get(
 
   CFrameBufferResource * fbr = &m_buffers[buffer.resourceSlot];
   if (!fbr->Init(m_dx12, buffer.token, buffer.mem,
-      buffer.heapOffset, minSize, m_transport->GetMaxFrameSize()))
+      buffer.heapOffset, minSize, buffer.capacity, buffer.direct))
     return nullptr;
 
   return fbr;

idd/LGIdd/capture/CFrameBufferPool.h

@@ -30,13 +30,12 @@ class IFrameTransport;
 class CFrameBufferPool
 {
 private:
-  IFrameTransport * m_transport = nullptr;
   CD3D12Device * m_dx12 = nullptr;
 
-  CFrameBufferResource m_buffers[CAPTURE_FRAME_BUFFERS];
+  CFrameBufferResource m_buffers[FRAME_BUFFER_RESOURCES];
 
 public:
-  void Init(IFrameTransport * transport, CD3D12Device * dx12);
+  void Init(IFrameTransport *, CD3D12Device * dx12);
   void Reset();
 
   CFrameBufferResource * Get(

idd/LGIdd/capture/CFrameBufferResource.cpp

@@ -26,7 +26,7 @@
 
 bool CFrameBufferResource::Init(CD3D12Device * dx12,
   const FrameToken& token, uint8_t * base, uint64_t heapOffset, size_t size,
-  size_t maxFrameSize)
+  size_t maxFrameSize, bool direct)
 {
   if (size > maxFrameSize)
   {
@@ -35,7 +35,7 @@ bool CFrameBufferResource::Init(CD3D12Device * dx12,
     return false;
   }
 
-  const bool indirect = dx12->IsIndirectCopy();
+  const bool indirect = !direct || dx12->IsIndirectCopy();
 
   D3D12_RESOURCE_DESC desc = {};
   desc.Dimension          = D3D12_RESOURCE_DIMENSION_BUFFER;
@@ -55,7 +55,9 @@ bool CFrameBufferResource::Init(CD3D12Device * dx12,
   }
 
   // Nothing to do if the resource already represents this allocation.
-  if (m_base == base && m_size >= size)
+  if (m_base == base && m_size >= size &&
+      m_capacity == maxFrameSize && m_heapOffset == heapOffset &&
+      m_direct == direct)
   {
     m_token     = token;
     m_frameSize = size;
@@ -139,7 +141,10 @@ bool CFrameBufferResource::Init(CD3D12Device * dx12,
   m_token      = token;
   m_base       = base;
   m_size       = size;
+  m_capacity   = maxFrameSize;
+  m_heapOffset = heapOffset;
   m_frameSize  = size;
+  m_direct     = direct;
   return true;
 }
 
@@ -154,8 +159,11 @@ void CFrameBufferResource::Reset()
   m_token             = {};
   m_base              = nullptr;
   m_size              = 0;
+  m_capacity          = 0;
+  m_heapOffset        = 0;
   m_frameSize         = 0;
   m_fullCopy          = false;
+  m_direct            = false;
   m_nbCopyDirtyRects  = 0;
   m_copyPitch         = 0;
   m_copyBytesPerPixel = 0;

idd/LGIdd/capture/CFrameBufferResource.h

@@ -41,6 +41,8 @@ class CFrameBufferResource
     FrameToken                m_token             = {};
     uint8_t                 * m_base              = nullptr;
     size_t                    m_size              = 0;
+    size_t                    m_capacity          = 0;
+    uint64_t                  m_heapOffset        = 0;
     size_t                    m_frameSize         = 0;
     uint64_t                  m_captureTime       = 0;
     uint64_t                  m_postProcessStart  = 0;
@@ -51,6 +53,7 @@ class CFrameBufferResource
     unsigned                  m_timingEffectIndex = 0;
     uint64_t                  m_timingToken       = 0;
     bool                      m_fullCopy          = false;
+    bool                      m_direct            = false;
     RECT                      m_copyDirtyRects[LG_MAX_DIRTY_RECTS * 2] = {};
     unsigned                  m_nbCopyDirtyRects  = 0;
     unsigned                  m_copyPitch         = 0;
@@ -62,7 +65,7 @@ class CFrameBufferResource
 
   public:
     bool Init(CD3D12Device * dx12, const FrameToken& token, uint8_t * base,
-      uint64_t heapOffset, size_t size, size_t maxFrameSize);
+      uint64_t heapOffset, size_t size, size_t maxFrameSize, bool direct);
     void Reset();
 
     const FrameToken& GetToken() const { return m_token;     }

idd/LGIdd/capture/CFrameProcessor.h

@@ -34,6 +34,15 @@
 using namespace Microsoft::WRL;
 
 class IFrameTransport;
+struct FramePlan;
+
+struct FrameCopyBatch
+{
+  PreparedFrameBatch      prepared;
+  CFrameBufferResource  * resources[FRAME_MAX_SINKS] = {};
+  uint32_t                accepted = 0;
+  unsigned                candidateIndex = 0;
+};
 
 struct FrameSubmission
 {
@@ -84,8 +93,8 @@ public:
   virtual bool IsValid() const;
   virtual bool Submit(const FrameSubmission& submission) = 0;
   virtual bool HasReadyFrame() const                     = 0;
-  virtual bool Publish(const CFrameScheduler::Schedule& schedule,
+  virtual bool Publish(
-    bool periodic, uint64_t publishStart)                = 0;
+    const FramePlan& plan, uint64_t publishStart)         = 0;
   virtual bool UsesCadence() const                       = 0;
   virtual void Reset();
   virtual void Invalidate();

idd/LGIdd/capture/CFrameScheduler.cpp

@@ -54,6 +54,15 @@ void CFrameScheduler::WakePublisher() const
 {
   if (m_wakeEvent)
     SetEvent(m_wakeEvent);
+  CSRWSharedLock lock(m_wakeLock);
+  if (m_sharedWakeEvent)
+    SetEvent(m_sharedWakeEvent);
+}
+
+void CFrameScheduler::SetSharedWakeEvent(HANDLE event)
+{
+  CSRWExclusiveLock lock(m_wakeLock);
+  m_sharedWakeEvent = event;
 }
 
 uint64_t CFrameScheduler::Nanotime()

idd/LGIdd/capture/CFrameScheduler.h

@@ -103,7 +103,9 @@ private:
   static const unsigned WORK_TIMING_HISTORY_SIZE  = 32;
 
   mutable CSRWLock m_lock;
+  mutable CSRWLock m_wakeLock;
   HANDLE           m_wakeEvent            = nullptr;
+  HANDLE           m_sharedWakeEvent      = nullptr;
   Client           m_clients[MAX_CLIENTS] = {};
   Schedule         m_schedule             = {};
   bool             m_scheduling           = false;
@@ -162,6 +164,7 @@ public:
     const FrameScheduleUpdate& schedule, uint64_t now);
   bool GetSchedule(Schedule& schedule) const;
   HANDLE GetWakeEvent() const { return m_wakeEvent; }
+  void SetSharedWakeEvent(HANDLE event);
   void ObserveFrame(uint64_t now);
   void ForceFrame();
   bool GetPublishTarget(uint64_t now, uint64_t& target,

idd/LGIdd/capture/CHardwareFrameProcessor.cpp

@@ -73,9 +73,10 @@ public:
 CHardwareFrameProcessor::CHardwareFrameProcessor(
     IFrameTransport * transport, std::shared_ptr<CD3D12Device> dx12,
     CPostProcessor postProcessors[CAPTURE_PIPELINE_SLOTS],
-    CSRWLock * pipelineLock, HANDLE terminateEvent) :
+    CSRWLock * pipelineLock, HANDLE terminateEvent, bool useCadence) :
   CFrameProcessor(transport, std::move(dx12), postProcessors,
-    pipelineLock, terminateEvent)
+    pipelineLock, terminateEvent),
+  m_useCadence(useCadence)
 {
   m_candidateAvailableEvent.Attach(
     CreateEvent(nullptr, FALSE, FALSE, nullptr));
@@ -145,14 +146,18 @@ void CHardwareFrameProcessor::ResetPipeline()
 bool CHardwareFrameProcessor::HasReadyFrame() const
 {
   bool ready    = false;
+  bool retained = false;
+  {
     CSRWSharedLock lock(m_candidateLock);
     for (const FrameCandidate& candidate : m_candidates)
-    if (candidate.state == CANDIDATE_READY)
     {
+      if (candidate.state == CANDIDATE_READY)
         ready = true;
-      break;
+      else if (candidate.state == CANDIDATE_RETAINED)
+        retained = true;
     }
-  return ready;
+  }
+  return ready || (retained && m_transport->NeedsFrame());
 }
 
 int CHardwareFrameProcessor::AcquireCandidate(
@@ -163,12 +168,18 @@ int CHardwareFrameProcessor::AcquireCandidate(
   bool     superseded = false;
   bool     idle       = true;
   bool     publishing = false;
+  bool     retained   = false;
 
   {
     CSRWExclusiveLock lock(m_candidateLock);
     for (unsigned i = 0; i < ARRAYSIZE(m_candidates); ++i)
     {
-      if (m_candidates[i].state != CANDIDATE_FREE)
+      if (m_candidates[i].state == CANDIDATE_RETAINED)
+      {
+        retained = true;
+        continue;
+      }
+      else if (m_candidates[i].state != CANDIDATE_FREE)
       {
         idle = false;
         if (m_candidates[i].state == CANDIDATE_PUBLISHING)
@@ -187,7 +198,7 @@ int CHardwareFrameProcessor::AcquireCandidate(
         ++readyCount;
 
     if (allowSupersede && !exclusiveSample && selected < 0 &&
-        readyCount > (publishing ? 0U : 1U))
+        readyCount > ((publishing || retained) ? 0U : 1U))
       for (unsigned i = 0; i < ARRAYSIZE(m_candidates); ++i)
         if (m_candidates[i].state == CANDIDATE_READY &&
             m_candidates[i].sequence < oldest)
@@ -202,7 +213,7 @@ int CHardwareFrameProcessor::AcquireCandidate(
         m_candidates[static_cast<unsigned>(selected)];
       superseded         = candidate.state == CANDIDATE_READY;
       candidate.state    = CANDIDATE_PREPARING;
-      candidate.sequence = ++m_candidateSequence;
+      candidate.sequence = m_transport->NextContentSerial();
     }
   }
 
@@ -223,6 +234,44 @@ void CHardwareFrameProcessor::ReleaseCandidate(unsigned candidateIndex)
   SignalCandidateState();
 }
 
+void CHardwareFrameProcessor::RetainCandidate(unsigned candidateIndex)
+{
+  if (candidateIndex >= ARRAYSIZE(m_candidates))
+    return;
+
+  unsigned superseded = 0;
+  {
+    CSRWExclusiveLock lock(m_candidateLock);
+    FrameCandidate& candidate = m_candidates[candidateIndex];
+    if (candidate.state != CANDIDATE_PUBLISHING)
+      return;
+
+    bool newer = false;
+    for (unsigned i = 0; i < ARRAYSIZE(m_candidates); ++i)
+      if (i != candidateIndex)
+      {
+        FrameCandidate& current = m_candidates[i];
+        const bool complete = current.state == CANDIDATE_READY ||
+          current.state == CANDIDATE_PUBLISHING ||
+          current.state == CANDIDATE_RETAINED;
+        if (complete && current.sequence > candidate.sequence)
+          newer = true;
+        else if (current.sequence < candidate.sequence &&
+                 (current.state == CANDIDATE_READY ||
+                  current.state == CANDIDATE_RETAINED))
+        {
+          if (current.state == CANDIDATE_READY)
+            ++superseded;
+          current.state = CANDIDATE_FREE;
+        }
+      }
+    candidate.state = newer ? CANDIDATE_FREE : CANDIDATE_RETAINED;
+  }
+  for (unsigned i = 0; i < superseded; ++i)
+    m_transport->FrameSuperseded();
+  SignalCandidateState();
+}
+
 bool CHardwareFrameProcessor::EnsureCandidateResource(
   unsigned candidateIndex, size_t frameSize)
 {
@@ -312,6 +361,7 @@ void CHardwareFrameProcessor::CandidateCompletionFunction(
   uint64_t gpuEnd   = 0;
   const bool timingValid = result && slot->GetGPUTimes(gpuStart, gpuEnd);
 
+  uint64_t readySequence = 0;
   bool forceFrame        = false;
   {
     CSRWExclusiveLock lock(processor->m_candidateLock);
@@ -321,9 +371,28 @@ void CHardwareFrameProcessor::CandidateCompletionFunction(
       candidate->prepareGPUStart    = gpuStart;
       candidate->prepareGPUEnd      = gpuEnd;
       candidate->prepareTimingValid = timingValid;
-      candidate->state              =
+      if (result)
-        result ? CANDIDATE_READY : CANDIDATE_FREE;
+      {
-      forceFrame = result && candidate->timingToken != 0;
+        bool newer = false;
+        for (FrameCandidate& current : processor->m_candidates)
+          if (&current != candidate)
+          {
+            const bool complete = current.state == CANDIDATE_READY ||
+              current.state == CANDIDATE_PUBLISHING ||
+              current.state == CANDIDATE_RETAINED;
+            if (complete && current.sequence > candidate->sequence)
+              newer = true;
+            else if (current.state == CANDIDATE_RETAINED)
+              current.state = CANDIDATE_FREE;
+          }
+        candidate->state = newer ? CANDIDATE_FREE : CANDIDATE_READY;
+      }
+      else
+        candidate->state = CANDIDATE_FREE;
+      if (candidate->state == CANDIDATE_READY)
+        readySequence = candidate->sequence;
+      forceFrame = candidate->state == CANDIDATE_READY &&
+        candidate->timingToken != 0;
     }
   }
 
@@ -332,8 +401,12 @@ void CHardwareFrameProcessor::CandidateCompletionFunction(
     processor->SetFullDamage();
     processor->m_transport->ForceFrame();
   }
-  else if (forceFrame)
+  else if (readySequence)
+  {
+    processor->m_transport->FrameProductReady(readySequence);
+    if (forceFrame)
       processor->m_transport->ForceFrame();
+  }
   processor->SignalCandidateState();
 }
 
@@ -341,15 +414,16 @@ void CHardwareFrameProcessor::CompletionFunction(
   CD3D12CommandSlot * slot, bool result, void * param1, void * param2)
 {
   auto processor = static_cast<CHardwareFrameProcessor *>(param1);
-  auto fbRes     = static_cast<CFrameBufferResource *>(param2);
+  const FrameCopyBatch batch =
-  const unsigned candidateIndex = fbRes->GetCandidateIndex();
+    *static_cast<FrameCopyBatch *>(param2);
+  const unsigned candidateIndex = batch.candidateIndex;
 
   if (!result)
   {
-    processor->m_transport->FailFrameBuffer(fbRes->GetToken());
+    processor->m_transport->FailFrameBatch(batch.prepared.token);
     processor->SetFullDamage();
     processor->m_transport->ForceFrame();
-    processor->ReleaseCandidate(candidateIndex);
+    processor->RetainCandidate(candidateIndex);
     return;
   }
 
@@ -371,23 +445,55 @@ void CHardwareFrameProcessor::CompletionFunction(
     prepareTimingValid = candidate.prepareTimingValid;
   }
 
-  const uint64_t publishStart = fbRes->GetCopyStart();
+  uint64_t publishStart = 0;
+  for (unsigned i = 0; i < batch.prepared.count; ++i)
+    if ((batch.accepted & (1U << i)) && batch.resources[i])
+    {
+      publishStart = batch.resources[i]->GetCopyStart();
+      break;
+    }
   uint64_t       gpuCopyStart     = 0;
   uint64_t       gpuCopyEnd       = 0;
-  uint64_t       indirectCopyTime = 0;
-  if (processor->m_dx12->IsIndirectCopy())
-  {
-    const uint64_t indirectCopyStart = CFrameScheduler::Nanotime();
-    processor->m_transport->WriteFrameBuffer(
-      fbRes->GetToken(), fbRes->GetMap(), 0,
-      fbRes->GetFrameSize(), false);
-    indirectCopyTime = CFrameScheduler::Nanotime() - indirectCopyStart;
-  }
-
   const bool gpuTimingValid =
     slot->GetGPUTimes(gpuCopyStart, gpuCopyEnd);
-  const uint64_t copyReady = CFrameScheduler::Nanotime();
+  bool timingRecorded = false;
+  for (unsigned i = 0; i < batch.prepared.count; ++i)
+  {
+    if (!(batch.accepted & (1U << i)))
+      continue;
+    CFrameBufferResource * fbRes = batch.resources[i];
+    if (!fbRes)
+      continue;
 
+    uint64_t stagedCopyTime = 0;
+    if (fbRes->GetMap())
+    {
+      const uint64_t stagedCopyStart = CFrameScheduler::Nanotime();
+      if (fbRes->IsFullCopy())
+        processor->m_transport->WriteFrameTarget(batch.prepared.token,
+          i, fbRes->GetMap(), 0, fbRes->GetFrameSize(), false);
+      else
+      {
+        const unsigned pitch         = fbRes->GetCopyPitch();
+        const unsigned bytesPerPixel = fbRes->GetCopyBytesPerPixel();
+        const RECT * dirtyRects      = fbRes->GetCopyDirtyRects();
+        const unsigned count         = fbRes->GetCopyDirtyRectCount();
+        for (const RECT * rect = dirtyRects;
+             rect < dirtyRects + count; ++rect)
+        {
+          const size_t rowOffset =
+            (size_t)rect->top * pitch +
+            (size_t)rect->left * bytesPerPixel;
+          const size_t rowBytes =
+            (size_t)(rect->right - rect->left) * bytesPerPixel;
+          processor->m_transport->WriteFrameTargetRows(
+            batch.prepared.token, i, fbRes->GetMap(), rowOffset,
+            rowBytes, pitch, (unsigned)(rect->bottom - rect->top));
+        }
+      }
+      stagedCopyTime = CFrameScheduler::Nanotime() - stagedCopyStart;
+    }
+    const uint64_t copyReady = CFrameScheduler::Nanotime();
     const uint64_t postProcessStart = fbRes->GetPostProcessStart();
     uint64_t postProcessTime = prepareCopyStart - postProcessStart;
     uint64_t prepareCopyTime = prepareReady - prepareCopyStart;
@@ -401,11 +507,12 @@ void CHardwareFrameProcessor::CompletionFunction(
     uint64_t publishCopyTime = copyReady - publishStart;
     if (gpuTimingValid && gpuCopyStart >= publishStart &&
         gpuCopyEnd >= gpuCopyStart && gpuCopyEnd <= copyReady)
-    publishCopyTime = gpuCopyEnd - gpuCopyStart + indirectCopyTime;
+      publishCopyTime = gpuCopyEnd - gpuCopyStart + stagedCopyTime;
 
     const uint64_t copyTime = prepareCopyTime + publishCopyTime;
 
-  processor->m_transport->FinalizeFrameBuffer(fbRes->GetToken());
+    processor->m_transport->FinalizeFrameTarget(
+      batch.prepared.token, i);
     const uint64_t publishedAt = CFrameScheduler::Nanotime();
     const uint64_t prepareElapsed = prepareReady >= postProcessStart ?
       prepareReady - postProcessStart : 0;
@@ -420,30 +527,32 @@ void CHardwareFrameProcessor::CompletionFunction(
     const uint64_t holdTime = publishStart >= prepareReady ?
       publishStart - prepareReady : 0;
 
-  processor->m_transport->SetFrameTiming(fbRes->GetToken(),
+    processor->m_transport->SetFrameTargetTiming(batch.prepared.token, i,
-    fbRes->GetCaptureTime(), postProcessTime, copyTime, readyTime, holdTime,
+      fbRes->GetCaptureTime(), postProcessTime, copyTime, readyTime,
-    fbRes->GetSchedule(), publishedAt);
+      holdTime, publishedAt);
     processor->m_transport->TryRecordFrameTiming(
-    fbRes->GetToken(), publishedAt - publishStart);
+      batch.prepared.token, i, publishedAt - publishStart);
 
     const uint64_t timingToken = fbRes->GetTimingToken();
-  if (timingToken && timingStart && prepareReady >= timingStart &&
+    if (!timingRecorded && timingToken && timingStart &&
-      copyReady >= publishStart)
+        prepareReady >= timingStart && copyReady >= publishStart)
     {
       const uint64_t totalTime =
         (prepareReady - timingStart) + (copyReady - publishStart);
       processor->m_postProcessors[candidateIndex].RecordTiming(
         fbRes->GetTimingEffectIndex(), timingToken,
         fbRes->IsFullCopy(), totalTime);
+      timingRecorded = true;
     }
 
-  processor->m_transport->CompleteFrameBuffer(fbRes->GetToken(), true);
+    processor->m_transport->CompleteFrameTarget(
-  processor->ReleaseCandidate(candidateIndex);
+      batch.prepared.token, i, true);
+  }
+  processor->RetainCandidate(candidateIndex);
 }
 
 bool CHardwareFrameProcessor::Publish(
-  const CFrameScheduler::Schedule& schedule, bool periodic,
+  const FramePlan& plan, uint64_t publishStart)
-  uint64_t publishStart)
 {
   CPublishPending publishPending(
     m_copySubmitLock, &m_publishPending, m_copySubmitEvent.Get());
@@ -451,6 +560,7 @@ bool CHardwareFrameProcessor::Publish(
 
   int      selectedCandidate = -1;
   uint64_t newestSequence    = 0;
+  CandidateState selectedState = CANDIDATE_FREE;
 
   {
     CSRWExclusiveLock lock(m_candidateLock);
@@ -463,9 +573,23 @@ bool CHardwareFrameProcessor::Publish(
         newestSequence    = m_candidates[i].sequence;
       }
 
+    if (selectedCandidate < 0 && m_transport->NeedsFrame())
+      for (unsigned i = 0; i < ARRAYSIZE(m_candidates); ++i)
+        if (m_candidates[i].state == CANDIDATE_RETAINED &&
+            (selectedCandidate < 0 ||
+              m_candidates[i].sequence > newestSequence))
+        {
+          selectedCandidate = static_cast<int>(i);
+          newestSequence    = m_candidates[i].sequence;
+        }
+
     if (selectedCandidate >= 0)
-      m_candidates[static_cast<unsigned>(selectedCandidate)].state =
+    {
-        CANDIDATE_PUBLISHING;
+      FrameCandidate& selected =
+        m_candidates[static_cast<unsigned>(selectedCandidate)];
+      selectedState  = selected.state;
+      selected.state = CANDIDATE_PUBLISHING;
+    }
   }
 
   if (selectedCandidate < 0)
@@ -473,12 +597,13 @@ bool CHardwareFrameProcessor::Publish(
   const unsigned candidateIndex =
     static_cast<unsigned>(selectedCandidate);
 
-  const auto restoreCandidate = [this, candidateIndex]()
+  const auto restoreCandidate =
+    [this, candidateIndex, selectedState]()
   {
     {
       CSRWExclusiveLock lock(m_candidateLock);
       if (m_candidates[candidateIndex].state == CANDIDATE_PUBLISHING)
-        m_candidates[candidateIndex].state = CANDIDATE_READY;
+        m_candidates[candidateIndex].state = selectedState;
     }
     SignalCandidateState();
   };
@@ -500,30 +625,20 @@ bool CHardwareFrameProcessor::Publish(
   CPostProcessor& postProcessor    = m_postProcessors[candidateIndex];
   const uint64_t candidateSequence = candidate.sequence;
 
-  auto buffer = m_transport->PrepareFrameBuffer(
+  PreparedFrameBatch prepared = {};
-    candidate.pitch, candidate.srcFormat, candidate.dstFormat,
+  if (!m_transport->PrepareFrameBatch(plan, candidate.sequence,
-    candidate.dirtyRects, candidate.nbDirtyRects, schedule);
+        candidate.pitch, candidate.frameSize, candidate.srcFormat,
-  if (!buffer.mem)
+        candidate.dstFormat, candidate.dirtyRects,
+        candidate.nbDirtyRects, true, prepared))
   {
     restoreCandidate();
     return false;
   }
 
-  CFrameBufferResource * fbRes =
-    m_frameBuffers.Get(buffer, candidate.frameSize);
-  if (!fbRes)
-  {
-    m_transport->AbortFrameBuffer(buffer.token);
-    restoreCandidate();
-    DEBUG_ERROR("Failed to get a CFrameBufferResource from the pool");
-    SetFullDamage();
-    return false;
-  }
-
   CD3D12CommandSlot * copySlot = m_dx12->GetCopySlot(candidateIndex);
   if (!copySlot)
   {
-    m_transport->AbortFrameBuffer(buffer.token);
+    m_transport->AbortFrameBatch(prepared.token);
     restoreCandidate();
     DEBUG_ERROR("Failed to get a copy CommandSlot for publication");
     SetFullDamage();
@@ -534,42 +649,72 @@ bool CHardwareFrameProcessor::Publish(
   unsigned nbPreviousDirtyRects                   = 0;
   GetPreviousDamage(previousDirtyRects, &nbPreviousDirtyRects);
 
+  FrameCopyBatch& batch = m_publishBatches[candidateIndex];
+  batch = {};
+  batch.prepared       = prepared;
+  batch.candidateIndex = candidateIndex;
+  const unsigned bytesPerPixel =
+    candidate.dstFormat.format == FRAME_TYPE_RGBA16F ? 8 : 4;
+  bool resourcesReady = true;
+  for (unsigned i = 0; i < prepared.count; ++i)
+  {
+    CFrameBufferResource * fbRes =
+      m_frameBuffers.Get(prepared.targets[i], candidate.frameSize);
+    batch.resources[i] = fbRes;
+    if (!fbRes)
+    {
+      resourcesReady = false;
+      break;
+    }
+
     RECT copyDirtyRects[LG_MAX_DIRTY_RECTS * 2] = {};
     unsigned nbCopyDirtyRects = 0;
     const bool fullCopy = CFrameProcessorUtil::BuildCopyDamage(
-    postProcessor, buffer.fullCopy,
+      postProcessor, prepared.targets[i].fullCopy,
       previousDirtyRects, nbPreviousDirtyRects,
       candidate.dirtyRects, candidate.nbDirtyRects,
       candidate.dstFormat.width, candidate.dstFormat.height,
       copyDirtyRects, &nbCopyDirtyRects);
-
     fbRes->SetTiming(
       candidate.captureTime, candidate.postProcessStart, publishStart);
     fbRes->SetCandidateIndex(candidateIndex);
     fbRes->SetPostProcessSample(
       candidate.timingEffectIndex, candidate.timingToken, fullCopy);
-  copySlot->SetCompletionCallback(&CompletionFunction, this, fbRes);
+    fbRes->SetCopyDamage(copyDirtyRects, nbCopyDirtyRects,
-
+      fullCopy, candidate.pitch, bytesPerPixel);
-  copySlot->BeginTiming();
+  }
-  postProcessor.CopyFromCandidate(
+  if (!resourcesReady)
-    copySlot->GetGfxList(), fbRes->Get().Get(), candidate.resource.Get(),
+  {
-    copyDirtyRects, nbCopyDirtyRects, fullCopy);
+    copySlot->Cancel();
-  copySlot->EndTiming();
+    m_transport->FailFrameBatch(prepared.token);
-
+    restoreCandidate();
-  bool deliveredToOwner;
+    DEBUG_ERROR("Failed to get a CFrameBufferResource from the pool");
-  if (!m_transport->PublishFrameBuffer(
+    SetFullDamage();
-        buffer.token, schedule, deliveredToOwner))
+    return false;
+  }
+
+  const uint32_t accepted =
+    m_transport->PublishFrameBatch(prepared.token);
+  if (!accepted)
   {
     copySlot->Cancel();
-    m_transport->AbortFrameBuffer(buffer.token);
     restoreCandidate();
     return false;
   }
-  CFrameScheduler::Schedule frameSchedule = schedule;
+  batch.accepted = accepted;
-  if (!deliveredToOwner ||
+
-      !m_transport->TryFrameSubmitted(buffer.token, schedule))
+  copySlot->SetCompletionCallback(&CompletionFunction, this, &batch);
-    frameSchedule.phaseEligible = false;
+  copySlot->BeginTiming();
-  fbRes->SetSchedule(frameSchedule);
+  for (unsigned i = 0; i < prepared.count; ++i)
+    if (accepted & (1U << i))
+    {
+      CFrameBufferResource * fbRes = batch.resources[i];
+      postProcessor.CopyFromCandidate(copySlot->GetGfxList(),
+        fbRes->Get().Get(), candidate.resource.Get(),
+        fbRes->GetCopyDirtyRects(), fbRes->GetCopyDirtyRectCount(),
+        fbRes->IsFullCopy());
+    }
+  copySlot->EndTiming();
 
   {
     CSRWExclusiveLock lock(m_damageLock);
@@ -595,23 +740,25 @@ bool CHardwareFrameProcessor::Publish(
   if (!submitted)
   {
     SetFullDamage();
+    const bool submittedWork = copySlot->HasSubmittedWork();
     bool callbackPending;
     {
       CSRWSharedLock lock(m_candidateLock);
       callbackPending = candidate.state == CANDIDATE_PUBLISHING;
     }
-    if (callbackPending && !copySlot->HasSubmittedWork())
+    if (submittedWork || !callbackPending)
+      m_transport->CommitFrameBatch(prepared.token);
+    else
     {
-      m_transport->FailFrameBuffer(buffer.token);
+      m_transport->FailFrameBatch(prepared.token);
-      ReleaseCandidate(candidateIndex);
+      RetainCandidate(candidateIndex);
     }
     m_transport->ForceFrame();
     SignalCandidateState();
     return false;
   }
 
-  m_transport->CommitFrameBuffer(
+  m_transport->CommitFrameBatch(prepared.token);
-    buffer.token, schedule, periodic, deliveredToOwner);
 
   unsigned superseded = 0;
   {
@@ -780,6 +927,18 @@ bool CHardwareFrameProcessor::Submit(const FrameSubmission& submission)
     dstFormat.width, dstFormat.height);
 
   const size_t frameSize = postProcessor.GetOutputSize();
+  const unsigned pitch = postProcessor.GetOutputPitch();
+  if (!pitch || !frameSize || frameSize > m_transport->GetMaxFrameSize())
+  {
+    copySlot->Cancel();
+    if (computeSlot)
+      m_dx12->WaitForIdle();
+    ReleaseCandidate(candidateIndex);
+    DEBUG_ERROR("Processed frame does not fit in primary frame memory");
+    SetFullDamage();
+    return false;
+  }
+
   if (!EnsureCandidateResource(candidateIndex, frameSize))
   {
     copySlot->Cancel();
@@ -793,7 +952,7 @@ bool CHardwareFrameProcessor::Submit(const FrameSubmission& submission)
   candidate.srcFormat          = submission.sourceFormat;
   candidate.dstFormat          = dstFormat;
   candidate.nbDirtyRects       = nbDirtyRects;
-  candidate.pitch              = postProcessor.GetOutputPitch();
+  candidate.pitch              = pitch;
   candidate.frameSize          = frameSize;
   candidate.captureTime        = submission.captureTime;
   candidate.postProcessStart   = submission.postProcessStart;

idd/LGIdd/capture/CHardwareFrameProcessor.h

@@ -22,7 +22,7 @@
 
 #include "capture/CFrameProcessor.h"
 
-class CHardwareFrameProcessor final : public CFrameProcessor
+class CHardwareFrameProcessor : public CFrameProcessor
 {
 private:
   enum CandidateState
@@ -31,6 +31,7 @@ private:
     CANDIDATE_PREPARING,
     CANDIDATE_READY,
     CANDIDATE_PUBLISHING,
+    CANDIDATE_RETAINED,
   };
 
   struct FrameCandidate
@@ -67,10 +68,11 @@ private:
 
   FrameCandidate      m_candidates[CAPTURE_PIPELINE_SLOTS];
   CandidateDamageTail m_candidateDamageTail[CAPTURE_PIPELINE_SLOTS];
+  FrameCopyBatch      m_publishBatches[CAPTURE_PIPELINE_SLOTS];
   mutable CSRWLock    m_candidateLock;
   CSRWLock            m_copySubmitLock;
-  uint64_t            m_candidateSequence  = 0;
   bool                m_publishPending     = false;
+  bool                m_useCadence         = true;
   Wrappers::Event     m_candidateAvailableEvent;
   Wrappers::Event     m_copySubmitEvent;
 
@@ -80,6 +82,7 @@ private:
     CD3D12CommandSlot * slot, bool result, void * param1, void * param2);
   int AcquireCandidate(bool exclusiveSample, bool allowSupersede);
   void ReleaseCandidate(unsigned candidateIndex);
+  void RetainCandidate(unsigned candidateIndex);
   bool EnsureCandidateResource(unsigned candidateIndex, size_t frameSize);
   void ResetCandidates();
   void SignalCandidateState();
@@ -92,14 +95,14 @@ public:
   CHardwareFrameProcessor(IFrameTransport * transport,
     std::shared_ptr<CD3D12Device> dx12,
     CPostProcessor postProcessors[CAPTURE_PIPELINE_SLOTS],
-    CSRWLock * pipelineLock, HANDLE terminateEvent);
+    CSRWLock * pipelineLock, HANDLE terminateEvent,
+    bool useCadence = true);
 
   bool IsValid() const override;
   bool Submit(const FrameSubmission& submission) override;
   bool HasReadyFrame() const override;
-  bool Publish(const CFrameScheduler::Schedule& schedule,
+  bool Publish(const FramePlan& plan, uint64_t publishStart) override;
-    bool periodic, uint64_t publishStart) override;
+  bool UsesCadence() const override { return m_useCadence; }
-  bool UsesCadence() const override { return true; }
   void Reset() override;
   void ResetPipeline() override;
 };

idd/LGIdd/capture/CSoftwareFrameProcessor.h

@@ -25,8 +25,74 @@
 class CSoftwareFrameProcessor final : public CFrameProcessor
 {
 private:
+  enum ProductState
+  {
+    PRODUCT_FREE,
+    PRODUCT_PREPARING,
+    PRODUCT_READY,
+    PRODUCT_PUBLISHING,
+    PRODUCT_RETAINED,
+    PRODUCT_COMPLETING,
+  };
+
+  struct Product
+  {
+    ProductState           state        = PRODUCT_FREE;
+    ProductState           restoreState = PRODUCT_FREE;
+    ProductState           completedState = PRODUCT_FREE;
+    ComPtr<ID3D12Resource> resource;
+    D12FrameFormat         srcFormat    = {};
+    D12FrameFormat         dstFormat    = {};
+    RECT                   dirtyRects[LG_MAX_DIRTY_RECTS] = {};
+    unsigned               nbDirtyRects = 0;
+    RECT                   previousDirtyRects[LG_MAX_DIRTY_RECTS] = {};
+    unsigned               nbPreviousDirtyRects = 0;
+    unsigned               pitch        = 0;
+    size_t                 frameSize    = 0;
+    uint64_t               sequence     = 0;
+    uint64_t               captureTime  = 0;
+    uint64_t               postProcessStart = 0;
+    uint64_t               prepareCopyStart = 0;
+    uint64_t               prepareReady = 0;
+    uint64_t               prepareGPUStart = 0;
+    uint64_t               prepareGPUEnd = 0;
+    uint64_t               timingStart  = 0;
+    unsigned               timingEffectIndex = 0;
+    uint64_t               timingToken  = 0;
+    bool                   prepareTimingValid = false;
+    bool                   sourceReady = false;
+    bool                   batchCommitted = false;
+    bool                   productNotified = false;
+    bool                   executeActive = false;
+    bool                   completionDone = false;
+    bool                   completionSucceeded = false;
+    FrameCopyBatch         batch        = {};
+  };
+
+  Product          m_products[CAPTURE_PIPELINE_SLOTS];
+  mutable CSRWLock m_productLock;
+  Wrappers::Event  m_productAvailableEvent;
+
   static void CompletionFunction(
     CD3D12CommandSlot * slot, bool result, void * param1, void * param2);
+  int AcquireProduct();
+  int WaitForProduct();
+  void RestoreProduct(unsigned productIndex, ProductState state);
+  void FinishProduct(
+    unsigned productIndex, bool publishing, bool result);
+  void MarkSourceReady(unsigned productIndex, uint64_t sequence);
+  void MarkBatchCommitted(unsigned productIndex, uint64_t sequence);
+  bool EnsureProductResource(unsigned productIndex, size_t frameSize);
+  bool BeginExecute(unsigned productIndex, uint64_t sequence,
+    ProductState state);
+  void EndExecute(unsigned productIndex, uint64_t sequence,
+    bool& completed, bool& succeeded);
+  bool PrepareBatch(Product& product, unsigned productIndex,
+    const FramePlan& plan, uint64_t copyStart);
+  void CompleteBatch(CD3D12CommandSlot * slot, Product& product,
+    unsigned productIndex, bool publishing, bool result);
+  void ResetProducts();
+  void SignalProductState(bool available = false);
 
 public:
   CSoftwareFrameProcessor(IFrameTransport * transport,
@@ -35,10 +101,10 @@ public:
     CSRWLock * pipelineLock, HANDLE terminateEvent);
 
   bool Submit(const FrameSubmission& submission) override;
-  bool HasReadyFrame() const override { return false; }
+  bool HasReadyFrame() const override;
-  bool Publish(const CFrameScheduler::Schedule&, bool, uint64_t) override
+  bool Publish(const FramePlan& plan, uint64_t publishStart) override;
-  {
-    return false;
-  }
   bool UsesCadence() const override { return false; }
+  bool IsValid() const override;
+  void Reset() override;
+  void ResetPipeline() override;
 };

idd/LGIdd/capture/CSwapChainProcessor.cpp

@@ -786,194 +786,65 @@ void CSwapChainProcessor::PublisherThread()
   for (;;)
   {
     const uint64_t now = CFrameScheduler::Nanotime();
-    uint64_t                  target;
-    CFrameScheduler::Schedule schedule;
-    bool                      periodic;
-    bool                      republish;
-    m_transport.GetPublishTarget(
-      now, target, schedule, periodic, republish);
-
     const bool ready = m_frameProcessor->HasReadyFrame();
-    if (!ready)
+    FramePlan plan = {};
+    const bool due = m_transport.GetFramePlan(now, ready, plan);
+
+    if (plan.progressed)
+      continue;
+
+    if (due && ready)
     {
-      m_transport.ProcessDeliveries();
-      if (m_frameProcessor->HasReadyFrame())
-        continue;
-
-      uint64_t current       = CFrameScheduler::Nanotime();
-      uint64_t cadenceTarget = 0;
-      if (cadenceEnabled && schedule.deliveryDeadlineSerial && periodic)
-      {
-        if (schedule.deadline <= current)
-        {
-          m_transport.FrameMissed(schedule, current, periodic);
-          continue;
-        }
-        cadenceTarget = schedule.deadline;
-      }
-
-      if (republish && m_transport.HasPublishedFrame())
-      {
-        if (m_transport.RepublishFrameBuffer(schedule))
-          continue;
-
-        current = CFrameScheduler::Nanotime();
-        if (cadenceTarget && cadenceTarget <= current)
-        {
-          m_transport.FrameMissed(schedule, current, periodic);
-          continue;
-        }
-
-        uint64_t retryTarget = current + PUBLISH_RETRY_NS;
-        if (cadenceTarget)
-          retryTarget = min(retryTarget, cadenceTarget);
-        ArmPublishTimer(m_publishTimer.Get(), retryTarget - current);
-        if (WaitForMultipleObjects(
-              ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) ==
-            WAIT_OBJECT_0)
-          break;
-        continue;
-      }
-
-      uint64_t replayTarget;
-      if (m_transport.GetPendingDeliveryTarget(current, replayTarget))
-      {
-        bool retry = false;
-        if (replayTarget <= current)
-        {
-          if (m_transport.RetryPendingDelivery(current, retry))
-            continue;
-
-          current = CFrameScheduler::Nanotime();
-          if (cadenceTarget && cadenceTarget <= current)
-          {
-            m_transport.FrameMissed(schedule, current, periodic);
-            continue;
-          }
-
-          if (retry)
-            replayTarget = current + PUBLISH_RETRY_NS;
-          else
-          {
-            if (cadenceTarget)
-              replayTarget = cadenceTarget;
-            else
-            {
-              if (m_publishTimer.Get())
-                CancelWaitableTimer(m_publishTimer.Get());
-              if (WaitForMultipleObjects(
-                    ARRAYSIZE(idleHandles), idleHandles, FALSE, INFINITE) ==
-                  WAIT_OBJECT_0)
-                break;
-              continue;
-            }
-          }
-        }
-
-        if (cadenceTarget)
-          replayTarget = min(replayTarget, cadenceTarget);
-
-        current = CFrameScheduler::Nanotime();
-        if (cadenceTarget && cadenceTarget <= current)
-        {
-          m_transport.FrameMissed(schedule, current, periodic);
-          continue;
-        }
-        if (replayTarget <= current)
-          continue;
-
-        ArmPublishTimer(m_publishTimer.Get(), replayTarget - current);
-        if (WaitForMultipleObjects(
-              ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) ==
-            WAIT_OBJECT_0)
-          break;
-        continue;
-      }
-
-      if (cadenceTarget)
-      {
-        current = CFrameScheduler::Nanotime();
-        if (cadenceTarget <= current)
-        {
-          m_transport.FrameMissed(schedule, current, periodic);
-          continue;
-        }
-
-        ArmPublishTimer(m_publishTimer.Get(), cadenceTarget - current);
-        if (WaitForMultipleObjects(
-              ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) ==
-            WAIT_OBJECT_0)
-          break;
-        continue;
-      }
-
-      if (m_publishTimer.Get())
-        CancelWaitableTimer(m_publishTimer.Get());
-      if (WaitForMultipleObjects(
-            ARRAYSIZE(idleHandles), idleHandles, FALSE, INFINITE) ==
-          WAIT_OBJECT_0)
-        break;
-      continue;
-    }
-
-    uint64_t current = CFrameScheduler::Nanotime();
-    uint64_t replayTarget;
-    if (m_transport.GetPendingDeliveryTarget(current, replayTarget) &&
-        replayTarget < target)
-    {
-      if (replayTarget <= current)
-      {
-        m_transport.ProcessDeliveries();
-        current = CFrameScheduler::Nanotime();
-        bool retry = false;
-        if (m_transport.RetryPendingDelivery(current, retry))
-          continue;
-
-        current = CFrameScheduler::Nanotime();
-        if (retry)
-          replayTarget = current + PUBLISH_RETRY_NS;
-        else
-          replayTarget = target;
-      }
-
-      replayTarget = min(replayTarget, target);
-      current      = CFrameScheduler::Nanotime();
-      if (target > current)
-      {
-        if (replayTarget <= current)
-          continue;
-
-        ArmPublishTimer(m_publishTimer.Get(), replayTarget - current);
-        if (WaitForMultipleObjects(
-              ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) ==
-            WAIT_OBJECT_0)
-          break;
-        continue;
-      }
-    }
-
-    current = CFrameScheduler::Nanotime();
-    if (target > current)
-    {
-      ArmPublishTimer(m_publishTimer.Get(), target - current);
-      if (WaitForMultipleObjects(
-            ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) ==
-          WAIT_OBJECT_0)
-        break;
-      continue;
-    }
-
       const uint64_t publishStart = CFrameScheduler::Nanotime();
-    m_transport.ProcessDeliveries();
+      if (m_frameProcessor->Publish(plan, publishStart))
-    if (!m_transport.FrameBufferAvailable(schedule) ||
+        continue;
-        !m_frameProcessor->Publish(schedule, periodic, publishStart))
+      const uint64_t retry = publishStart + PUBLISH_RETRY_NS;
+      if (!plan.nextWake || retry < plan.nextWake)
+        plan.nextWake = retry;
+    }
+
+    uint64_t current = CFrameScheduler::Nanotime();
+    bool missed = false;
+    if (due && !ready && cadenceEnabled)
     {
-      ArmPublishTimer(m_publishTimer.Get(), PUBLISH_RETRY_NS);
+      for (unsigned i = 0; i < plan.count; ++i)
+      {
+        const FramePlanTarget& request = plan.targets[i];
+        if (!request.periodic ||
+            !request.commitSchedule.deliveryDeadlineSerial)
+          continue;
+        if (request.commitSchedule.deadline <= current)
+          missed = true;
+        else if (!plan.nextWake ||
+                 request.commitSchedule.deadline < plan.nextWake)
+          plan.nextWake = request.commitSchedule.deadline;
+      }
+    }
+    if (missed)
+    {
+      m_transport.MissFramePlan(plan, current);
+      continue;
+    }
+
+    current = CFrameScheduler::Nanotime();
+    if (plan.nextWake && plan.nextWake > current)
+    {
+      ArmPublishTimer(m_publishTimer.Get(), plan.nextWake - current);
       if (WaitForMultipleObjects(
             ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) ==
         WAIT_OBJECT_0)
         break;
+      continue;
     }
+
+    if (plan.nextWake)
+      continue;
+    if (m_publishTimer.Get())
+      CancelWaitableTimer(m_publishTimer.Get());
+    if (WaitForMultipleObjects(
+          ARRAYSIZE(idleHandles), idleHandles, FALSE, INFINITE) ==
+        WAIT_OBJECT_0)
+      break;
   }
 
   if (avTaskHandle)

idd/LGIdd/transport/CFrameHub.h

@@ -25,81 +25,147 @@
 #include "transport/IFrameTransport.h"
 #include "transport/ITransport.h"
 
-#include <vector>
+#include <atomic>
 
 class CFrameHub final : public IFrameTransport
 {
 private:
-  static const unsigned MAX_SLOTS = 1024;
+  struct Sink
-
-  struct Slot
   {
-    uint64_t serial             = 0;
+    struct ResourceLane
+    {
+      uint8_t  * mem        = nullptr;
+      uint64_t   heapOffset = 0;
+      unsigned   localSlot  = 0;
+      bool       direct     = false;
+      bool       valid      = false;
+      bool       busy       = false;
+    };
+
+    CSRWLock          callLock;
+    IFrameSink      * target        = nullptr;
+    HANDLE            drained       = nullptr;
+    std::atomic<unsigned> outstanding = 0;
+    std::atomic_bool   active        = false;
+    std::atomic<BackendId> backend   = 0;
+    std::atomic<uint32_t> epoch      = 0;
+    bool               reserved      = false;
+    bool               primary       = false;
+    bool               needsFullCopy = true;
+    uint64_t           lastContent   = 0;
+    uint64_t           blockedContent = 0;
+    size_t             blockedFrameSize = 0;
+    bool               blockedAllocation = false;
+    unsigned           pitch         = 0;
+    unsigned           width         = 0;
+    unsigned           height        = 0;
+    DXGI_FORMAT        format        = DXGI_FORMAT_UNKNOWN;
+    FrameType          frameType     = FRAME_TYPE_INVALID;
+    ResourceLane       lanes[FRAME_SINK_BUFFERS] = {};
+  };
+
+  struct BatchTarget
+  {
+    Sink                    * sink       = nullptr;
+    BackendId                 backend    = 0;
+    uint32_t                  epoch      = 0;
+    unsigned                  localSlot  = 0;
+    unsigned                  resourceLane = 0;
+    CFrameScheduler::Schedule schedule  = {};
+    CFrameScheduler::Schedule deliverySchedule = {};
+    uint64_t                  content    = 0;
+    unsigned                  pitch      = 0;
+    unsigned                  width      = 0;
+    unsigned                  height     = 0;
+    DXGI_FORMAT               format     = DXGI_FORMAT_UNKNOWN;
+    FrameType                 frameType  = FRAME_TYPE_INVALID;
+    bool                      periodic   = false;
+    bool                      published  = false;
+    bool                      delivered  = false;
+    bool                      submitted  = false;
     bool                      committed  = false;
     bool                      completionPending   = false;
     bool                      completionSucceeded = false;
+    bool                      releasePending = false;
+    bool                      active      = false;
   };
 
-  mutable CSRWLock m_lock;
+  struct Batch
-  IFrameSink     * m_sink       = nullptr;
+  {
-  BackendId        m_backend    = 0;
+    CSRWLock    lock;
-  uint32_t         m_epoch      = 0;
+    uint64_t    serial = 0;
-  uint64_t         m_nextSerial = 0;
+    unsigned    count  = 0;
-  std::vector<Slot> m_slots;
+    bool        active = false;
+    BatchTarget targets[FRAME_MAX_SINKS] = {};
+  };
 
-  bool Valid(const FrameToken& token) const;
+  struct SinkRef
-  void Invalidate(const FrameToken& token);
+  {
+    Sink     * sink;
+    unsigned   index;
+    bool       primary;
+  };
+
+  mutable CSRWLock m_listLock;
+  Sink             m_sinks[FRAME_MAX_SINKS];
+  Batch            m_batches[FRAME_BATCHES];
+  std::atomic<uint64_t> m_nextSerial = 0;
+  std::atomic<uint64_t> m_nextContent = 0;
+  std::atomic<uint64_t> m_newestContent = 0;
+  HANDLE           m_wakeEvent      = nullptr;
+
+  unsigned Snapshot(SinkRef refs[FRAME_MAX_SINKS]) const;
+  bool BatchValid(const Batch& batch, const FrameBatchToken& token) const;
+  void ReleaseTarget(Batch& batch, BatchTarget& target);
+  void CompleteTarget(Batch& batch, BatchTarget& target, bool succeeded);
 
 public:
-  bool Bind(BackendId backend, uint32_t epoch, IFrameSink& sink);
+  CFrameHub();
+  ~CFrameHub() override;
+
+  CFrameHub(const CFrameHub&) = delete;
+  CFrameHub& operator=(const CFrameHub&) = delete;
+
+  bool Bind(BackendId backend, uint32_t epoch, bool primary,
+    IFrameSink& sink);
   void Unbind(BackendId backend, uint32_t epoch);
 
   size_t GetMaxFrameSize() const override;
-  bool FrameBufferAvailable(const CFrameScheduler::Schedule& schedule,
+  uint64_t NextContentSerial() override;
-    bool allowReadyReplacement = true) override;
+  void FrameProductReady(uint64_t contentSerial) override;
-  bool HasPublishedFrame() const override;
+  bool NeedsFrame() const override;
-  void ProcessDeliveries() override;
+  bool GetFramePlan(
-  bool GetPendingDeliveryTarget(uint64_t now, uint64_t& target) override;
+    uint64_t now, bool productReady, FramePlan& plan) override;
-  bool RetryPendingDelivery(uint64_t now, bool& retry) override;
+  bool GetImmediateFramePlan(
-  PreparedFrameBuffer PrepareFrameBuffer(unsigned pitch,
+    uint64_t now, FramePlan& plan) override;
+  void MissFramePlan(const FramePlan& plan, uint64_t now) override;
+  bool PrepareFrameBatch(const FramePlan& plan,
+    uint64_t contentSerial, unsigned pitch, size_t frameSize,
     const D12FrameFormat& srcFormat, const D12FrameFormat& dstFormat,
     const RECT * dirtyRects, unsigned nbDirtyRects,
-    const CFrameScheduler::Schedule& schedule,
+    bool allowReadyReplacement, PreparedFrameBatch& batch) override;
-    bool allowReadyReplacement = true) override;
+  uint32_t PublishFrameBatch(const FrameBatchToken& token) override;
-  bool PublishFrameBuffer(const FrameToken& token,
+  void CommitFrameBatch(const FrameBatchToken& token) override;
-    const CFrameScheduler::Schedule& schedule,
+  void AbortFrameBatch(const FrameBatchToken& token) override;
-    bool& deliveredToOwner) override;
+  void FailFrameBatch(const FrameBatchToken& token) override;
-  bool RepublishFrameBuffer(
+  void WriteFrameTarget(const FrameBatchToken& token,
-    const CFrameScheduler::Schedule& schedule) override;
+    unsigned target, void * src, size_t offset, size_t len,
-  bool TryFrameSubmitted(const FrameToken& token,
+    bool setWritePos) const override;
-    const CFrameScheduler::Schedule& schedule) override;
+  void WriteFrameTargetRows(const FrameBatchToken& token,
-  void CommitFrameBuffer(const FrameToken& token,
+    unsigned target, void * src, size_t offset, size_t rowBytes,
-    const CFrameScheduler::Schedule& schedule, bool periodic,
+    size_t pitch, unsigned rows) const override;
-    bool deliveredToOwner) override;
+  void FinalizeFrameTarget(const FrameBatchToken& token,
-  void AbortFrameBuffer(const FrameToken& token) override;
+    unsigned target) const override;
-  void FailFrameBuffer(const FrameToken& token) override;
+  void SetFrameTargetTiming(const FrameBatchToken& token,
-  void CompleteFrameBuffer(
+    unsigned target, uint64_t captureTime, uint64_t postProcessTime,
-    const FrameToken& token, bool succeeded) override;
+    uint64_t copyTime, uint64_t readyTime, uint64_t holdTime,
-  void SetFrameTiming(const FrameToken& token, uint64_t captureTime,
-    uint64_t postProcessTime, uint64_t copyTime, uint64_t readyTime,
-    uint64_t holdTime, const CFrameScheduler::Schedule& schedule,
     uint64_t completedAt) override;
-  void WriteFrameBuffer(const FrameToken& token, void * src,
+  void TryRecordFrameTiming(const FrameBatchToken& token,
-    size_t offset, size_t len, bool setWritePos) const override;
+    unsigned target, uint64_t duration) override;
-  void WriteFrameBufferRows(const FrameToken& token, void * src,
+  void CompleteFrameTarget(const FrameBatchToken& token,
-    size_t offset, size_t rowBytes, size_t pitch,
+    unsigned target, bool succeeded) override;
-    unsigned rows) const override;
-  void FinalizeFrameBuffer(const FrameToken& token) const override;
-
   void ObserveFrame(uint64_t now) override;
   void ForceFrame() override;
-  bool GetPublishTarget(uint64_t now, uint64_t& target,
-    CFrameScheduler::Schedule& schedule, bool& periodic,
-    bool& republish) override;
-  void FrameMissed(const CFrameScheduler::Schedule& schedule,
-    uint64_t now, bool periodic) override;
   void FrameSuperseded() override;
-  HANDLE GetFrameScheduleEvent() const override;
+  HANDLE GetFrameScheduleEvent() const override { return m_wakeEvent; }
-  void TryRecordFrameTiming(
-    const FrameToken& token, uint64_t duration) override;
 };

idd/LGIdd/transport/CTransportManager.h

@@ -25,7 +25,6 @@
 #include "transport/ITransport.h"
 
 #include <memory>
-#include <vector>
 
 class CTransportManager final
 {
@@ -36,6 +35,19 @@ public:
   using Recovery = ITransport::Recovery;
 
 private:
+  mutable CSRWLock m_lock;
+
+  enum class Phase
+  {
+    IDLE,
+    OPEN,
+    INITIALIZE,
+    SETUP,
+    PROCESS,
+    ACCESS,
+    STOP,
+  };
+
   enum class State
   {
     CLOSED,
@@ -47,41 +59,97 @@ private:
     STOPPED,
   };
 
+  enum class Call
+  {
+    IDLE,
+    LIFECYCLE,
+    PROCESS,
+    RECOVERY,
+    ACCESS,
+  };
+
+  struct RecoveryUpdate
+  {
+    uint64_t session = 0;
+    uint32_t serial  = 0;
+    bool     active  = false;
+    Recovery state  = Recovery::FAILED;
+    uint32_t error   = 0;
+  };
+
   struct Entry
   {
-    BackendId                   id;
+    Entry();
-    const char                * name;
+    ~Entry();
-    bool                        required;
+
-    bool                        primary;
+    mutable CSRWLock            lock;
-    CreateFn                    create;
+    HANDLE                      idleEvent     = nullptr;
-    std::unique_ptr<ITransport> transport;
+    Call                        call          = Call::IDLE;
+    DWORD                       callOwner     = 0;
+    bool                        stopRequested = false;
+    BackendId                   id       = 0;
+    const char                * name     = nullptr;
+    bool                        required = false;
+    bool                        primary  = false;
+    CreateFn                    create   = nullptr;
+    std::shared_ptr<ITransport> transport;
     State                       state   = State::CLOSED;
     uint32_t                    epoch   = 1;
     uint64_t                    retryAt = 0;
     bool                        controlAdded = false;
     bool                        frameAdded   = false;
+    bool                        exposed      = false;
+    bool                        setupDone    = false;
+    bool                        syncPending  = false;
+    bool                        recoveryPending = false;
+    RecoveryUpdate              recovery;
+    FrameMemoryLimits           limits;
+    bool                        limitsValid = false;
+    DirectFrameBufferMemory     directMemory;
+    bool                        directMemoryValid = false;
+    std::shared_ptr<ITransport> exposedTransport;
   };
 
-  std::vector<std::unique_ptr<Entry>> m_entries;
+  std::unique_ptr<Entry>               m_entries[FRAME_MAX_SINKS];
+  unsigned                             m_entryCount  = 0;
   CControlHub                         m_control;
   CFrameHub                           m_frames;
   Entry                              * m_primary     = nullptr;
   bool                                 m_initialized = false;
   bool                                 m_setup       = false;
-  bool                                 m_exposed     = false;
   size_t                               m_alignment   = 0;
+  Phase                                m_phase       = Phase::IDLE;
+  DWORD                                m_phaseOwner  = 0;
+  HANDLE                               m_phaseIdle   = nullptr;
+  HANDLE                               m_stoppedEvent = nullptr;
+  bool                                 m_started     = false;
+  bool                                 m_stopping    = false;
+  bool                                 m_stopped     = false;
+
+  unsigned Entries(Entry * entries[FRAME_MAX_SINKS]) const;
+  Entry * Primary() const;
+
+  bool BeginPhase(Phase phase, bool wait, bool stopping = false);
+  void EndPhase();
+  bool BeginCall(Entry& entry, Call call, bool wait);
+  void EndCall(Entry& entry,
+    const std::shared_ptr<ITransport>& transport, bool drain = true);
+  void DrainRecovery(Entry& entry,
+    const std::shared_ptr<ITransport>& transport);
 
   OpenResult OpenEntry(Entry& entry);
   bool InitializeEntry(Entry& entry);
-  bool SetupEntry(Entry& entry);
+  bool SetupEntry(Entry& entry, size_t alignment);
-  void RetryEntry(Entry& entry, uint64_t now);
+  bool AddServices(Entry& entry);
+  void RetryEntry(Entry& entry, uint64_t now, bool initialized,
+    bool setup, size_t alignment);
   void HandleProcessResult(Entry& entry, ProcessResult result);
   void ScheduleRetry(Entry& entry);
   void RemoveServices(Entry& entry);
-  ITransport& Primary();
+  void Expose(Entry& entry);
 
 public:
-  CTransportManager() = default;
+  CTransportManager();
   ~CTransportManager();
 
   CTransportManager(const CTransportManager&) = delete;

idd/LGIdd/transport/IFrameSink.h

@@ -81,5 +81,6 @@ public:
     uint64_t now, bool periodic) = 0;
   virtual void FrameSuperseded() = 0;
   virtual HANDLE GetFrameScheduleEvent() const = 0;
+  virtual void SetFrameScheduleEvent(HANDLE event) = 0;
   virtual void TryRecordFrameTiming(uint64_t duration) = 0;
 };

idd/LGIdd/transport/IFrameTransport.h

@@ -28,60 +28,69 @@
 #include <stddef.h>
 #include <stdint.h>
 
+struct FramePlanTarget
+{
+  uint32_t                  sink    = 0;
+  uint32_t                  backend = 0;
+  uint32_t                  epoch   = 0;
+  CFrameScheduler::Schedule schedule = {};
+  CFrameScheduler::Schedule commitSchedule = {};
+  bool                      periodic = false;
+  bool                      primary  = false;
+};
+
+struct FramePlan
+{
+  FramePlanTarget targets[FRAME_MAX_SINKS] = {};
+  unsigned        count                    = 0;
+  uint64_t        nextWake                 = 0;
+  bool            progressed               = false;
+};
+
 class IFrameTransport
 {
 public:
   virtual ~IFrameTransport() = default;
 
   virtual size_t GetMaxFrameSize() const = 0;
+  virtual uint64_t NextContentSerial() = 0;
+  virtual void FrameProductReady(uint64_t contentSerial) = 0;
+  virtual bool NeedsFrame() const = 0;
+  virtual bool GetFramePlan(
+    uint64_t now, bool productReady, FramePlan& plan) = 0;
+  virtual bool GetImmediateFramePlan(
+    uint64_t now, FramePlan& plan) = 0;
+  virtual void MissFramePlan(const FramePlan& plan, uint64_t now) = 0;
 
-  virtual bool FrameBufferAvailable(
+  virtual bool PrepareFrameBatch(const FramePlan& plan,
-    const CFrameScheduler::Schedule& schedule,
+    uint64_t contentSerial, unsigned pitch, size_t frameSize,
-    bool allowReadyReplacement = true) = 0;
-  virtual bool HasPublishedFrame() const = 0;
-  virtual void ProcessDeliveries() = 0;
-  virtual bool GetPendingDeliveryTarget(
-    uint64_t now, uint64_t& target) = 0;
-  virtual bool RetryPendingDelivery(uint64_t now, bool& retry) = 0;
-  virtual PreparedFrameBuffer PrepareFrameBuffer(unsigned pitch,
     const D12FrameFormat& srcFormat, const D12FrameFormat& dstFormat,
     const RECT * dirtyRects, unsigned nbDirtyRects,
-    const CFrameScheduler::Schedule& schedule,
+    bool allowReadyReplacement, PreparedFrameBatch& batch) = 0;
-    bool allowReadyReplacement = true) = 0;
+  virtual uint32_t PublishFrameBatch(const FrameBatchToken& token) = 0;
-  virtual bool PublishFrameBuffer(const FrameToken& token,
+  virtual void CommitFrameBatch(const FrameBatchToken& token) = 0;
-    const CFrameScheduler::Schedule& schedule,
+  virtual void AbortFrameBatch(const FrameBatchToken& token) = 0;
-    bool& deliveredToOwner) = 0;
+  virtual void FailFrameBatch(const FrameBatchToken& token) = 0;
-  virtual bool RepublishFrameBuffer(
+
-    const CFrameScheduler::Schedule& schedule) = 0;
+  virtual void WriteFrameTarget(const FrameBatchToken& token,
-  virtual bool TryFrameSubmitted(const FrameToken& token,
+    unsigned target, void * src, size_t offset, size_t len,
-    const CFrameScheduler::Schedule& schedule) = 0;
+    bool setWritePos) const = 0;
-  virtual void CommitFrameBuffer(const FrameToken& token,
+  virtual void WriteFrameTargetRows(const FrameBatchToken& token,
-    const CFrameScheduler::Schedule& schedule, bool periodic,
+    unsigned target, void * src, size_t offset, size_t rowBytes,
-    bool deliveredToOwner) = 0;
+    size_t pitch, unsigned rows) const = 0;
-  virtual void AbortFrameBuffer(const FrameToken& token) = 0;
+  virtual void FinalizeFrameTarget(
-  virtual void FailFrameBuffer(const FrameToken& token) = 0;
+    const FrameBatchToken& token, unsigned target) const = 0;
-  virtual void CompleteFrameBuffer(
+  virtual void SetFrameTargetTiming(const FrameBatchToken& token,
-    const FrameToken& token, bool succeeded) = 0;
+    unsigned target, uint64_t captureTime, uint64_t postProcessTime,
-  virtual void SetFrameTiming(const FrameToken& token, uint64_t captureTime,
+    uint64_t copyTime, uint64_t readyTime, uint64_t holdTime,
-    uint64_t postProcessTime, uint64_t copyTime, uint64_t readyTime,
-    uint64_t holdTime, const CFrameScheduler::Schedule& schedule,
     uint64_t completedAt) = 0;
-  virtual void WriteFrameBuffer(const FrameToken& token, void * src,
+  virtual void TryRecordFrameTiming(const FrameBatchToken& token,
-    size_t offset, size_t len, bool setWritePos) const = 0;
+    unsigned target, uint64_t duration) = 0;
-  virtual void WriteFrameBufferRows(const FrameToken& token, void * src,
+  virtual void CompleteFrameTarget(const FrameBatchToken& token,
-    size_t offset, size_t rowBytes, size_t pitch,
+    unsigned target, bool succeeded) = 0;
-    unsigned rows) const = 0;
-  virtual void FinalizeFrameBuffer(const FrameToken& token) const = 0;
 
   virtual void ObserveFrame(uint64_t now) = 0;
   virtual void ForceFrame() = 0;
-  virtual bool GetPublishTarget(uint64_t now, uint64_t& target,
-    CFrameScheduler::Schedule& schedule, bool& periodic,
-    bool& republish) = 0;
-  virtual void FrameMissed(const CFrameScheduler::Schedule& schedule,
-    uint64_t now, bool periodic) = 0;
   virtual void FrameSuperseded() = 0;
   virtual HANDLE GetFrameScheduleEvent() const = 0;
-  virtual void TryRecordFrameTiming(
-    const FrameToken& token, uint64_t duration) = 0;
 };

idd/LGIdd/transport/PreparedFrameBuffer.h

@@ -20,8 +20,17 @@
 
 #pragma once
 
+#include <stddef.h>
 #include <stdint.h>
 
+enum : unsigned
+{
+  FRAME_MAX_SINKS       = 8,
+  FRAME_BATCHES         = 3,
+  FRAME_SINK_BUFFERS    = 3,
+  FRAME_BUFFER_RESOURCES = FRAME_MAX_SINKS * FRAME_SINK_BUFFERS,
+};
+
 struct FrameToken
 {
   uint32_t sink   = 0;
@@ -33,16 +42,32 @@ struct FrameToken
 struct PreparedFrameBuffer
 {
   FrameToken token;
-  unsigned   resourceSlot;
+  unsigned   resourceSlot = 0;
-  uint8_t * mem;
+  uint8_t  * mem          = nullptr;
-  uint64_t  heapOffset;
+  uint64_t   heapOffset   = 0;
-  bool      fullCopy;
+  size_t     capacity     = 0;
+  bool       direct       = false;
+  bool       fullCopy     = false;
+};
+
+struct FrameBatchToken
+{
+  uint32_t slot   = 0;
+  uint64_t serial = 0;
+};
+
+struct PreparedFrameBatch
+{
+  FrameBatchToken     token;
+  PreparedFrameBuffer targets[FRAME_MAX_SINKS] = {};
+  unsigned            count = 0;
 };
 
 struct SinkTarget
 {
-  unsigned  slot;
+  unsigned  slot       = 0;
-  uint8_t * mem;
+  uint8_t * mem        = nullptr;
-  uint64_t  heapOffset;
+  uint64_t  heapOffset = 0;
-  bool      fullCopy;
+  size_t    capacity   = 0;
+  bool      fullCopy   = false;
 };

idd/LGIdd/transport/lgmp/CLGMPFrameTransport.cpp

@@ -917,7 +917,10 @@ SinkTarget CLGMPFrameTransport::PrepareFrameBuffer(
   result.mem        = fb->data;
   result.heapOffset = reinterpret_cast<uintptr_t>(fb->data) -
     reinterpret_cast<uintptr_t>(m_ivshmem.GetMem());
+  result.capacity   = m_maxFrameSize;
   result.fullCopy   = fullCopy;
+  if (fullCopy)
+    fi->damageRectsCount = 0;
   return result;
 }
 

idd/LGIdd/transport/lgmp/CLGMPFrameTransport.h

@@ -214,5 +214,9 @@ public:
   {
     return m_frameScheduler.GetWakeEvent();
   }
+  void SetFrameScheduleEvent(HANDLE event) override
+  {
+    m_frameScheduler.SetSharedWakeEvent(event);
+  }
   void TryRecordFrameTiming(uint64_t duration) override;
 };