[host] d12: properly handle format changes by effects

host/platform/Windows/capture/D12/backend.h

@@ -21,6 +21,8 @@
 #ifndef _H_D12_BACKEND_
 #define _H_D12_BACKEND_
 
+#include "d12.h"
+
 #include <stdbool.h>
 #include <d3d12.h>
 #include "interface/capture.h"
@@ -29,8 +31,6 @@
 
 typedef struct D12Backend D12Backend;
 
-typedef struct D12FetchDesc D12FetchDesc;
-
 struct D12Backend
 {
   // friendly name
@@ -61,15 +61,7 @@ struct D12Backend
     ID3D12CommandQueue * commandQueue);
 
   ID3D12Resource * (*fetch)(D12Backend * instance, unsigned frameBufferIndex,
-    D12FetchDesc * meta);
+    D12FrameDesc * meta);
-};
-
-struct D12FetchDesc
-{
-  CaptureRotation       rotation;
-  RECT                * dirtyRects;
-  unsigned              nbDirtyRects;
-  DXGI_COLOR_SPACE_TYPE colorSpace;
 };
 
 static inline bool d12_backendCreate(const D12Backend * backend,
@@ -104,7 +96,7 @@ static inline CaptureResult d12_backendSync(D12Backend * instance,
   { return instance->sync(instance, commandQueue); }
 
 static inline ID3D12Resource * d12_backendFetch(D12Backend * instance,
-  unsigned frameBufferIndex, D12FetchDesc * desc)
+  unsigned frameBufferIndex, D12FrameDesc * desc)
   { return instance->fetch(instance, frameBufferIndex, desc); }
 
 // Backend defines

host/platform/Windows/capture/D12/backend/dd.c

@@ -453,7 +453,7 @@ static CaptureResult d12_dd_sync(D12Backend * instance,
 }
 
 static ID3D12Resource * d12_dd_fetch(D12Backend * instance,
-  unsigned frameBufferIndex, D12FetchDesc * desc)
+  unsigned frameBufferIndex, D12FrameDesc * desc)
 {
   DDInstance * this = UPCAST(DDInstance, instance);
 

host/platform/Windows/capture/D12/d12.c

@@ -27,6 +27,7 @@
 #include "common/windebug.h"
 #include "common/option.h"
 #include "common/rects.h"
+#include "common/vector.h"
 #include "com_ref.h"
 
 #include "backend.h"
@@ -57,15 +58,15 @@ struct D12Interface
   CapturePostPointerBuffer postPointerBufferFn;
 
   D12Backend * backend;
-  D12Effect  * effectRGB24;
+  Vector       effects;
 
   // capture format tracking
-  D3D12_RESOURCE_DESC captureFormat;
+  D12FrameFormat captureFormat;
   unsigned       formatVer;
   unsigned       pitch;
 
   // output format tracking
-  D3D12_RESOURCE_DESC dstFormat;
+  D12FrameFormat dstFormat;
 
   // prior frame dirty rects
   RECT      dirtyRects[D12_MAX_DIRTY_RECTS];
@@ -350,10 +351,17 @@ retryCreateCommandQueue:
     this->trackDamage))
     goto exit;
 
+  // create the vector of effects
+  vector_create(&this->effects, sizeof(D12Effect *), 0);
+
+  /* if RGB24 conversion is enabled add the effect to the list
+  NOTE: THIS MUST BE THE LAST EFFECT */
   if (this->allowRGB24)
   {
-    if (!d12_effectCreate(&D12Effect_RGB24, &this->effectRGB24, *device))
+    D12Effect * effect;
+    if (!d12_effectCreate(&D12Effect_RGB24, &effect, *device))
       goto exit;
+    vector_push(&this->effects, &effect);
   }
 
   comRef_toGlobal(this->factory     , factory      );
@@ -379,7 +387,11 @@ static void d12_stop(void)
 static bool d12_deinit(void)
 {
   bool result = true;
-  d12_effectFree(&this->effectRGB24);
+
+  D12Effect * effect;
+  vector_forEach(effect, &this->effects)
+    d12_effectFree(&effect);
+  vector_destroy(&this->effects);
 
   if (!d12_backendDeinit(this->backend))
     result = false;
@@ -428,7 +440,7 @@ static CaptureResult d12_waitFrame(unsigned frameBufferIndex,
   CaptureResult result = CAPTURE_RESULT_ERROR;
   comRef_scopePush(1);
 
-  D12FetchDesc desc;
+  D12FrameDesc desc;
 
   comRef_defineLocal(ID3D12Resource, src);
   *src = d12_backendFetch(this->backend, frameBufferIndex, &desc);
@@ -436,37 +448,79 @@ static CaptureResult d12_waitFrame(unsigned frameBufferIndex,
   {
     DEBUG_ERROR("D12 backend failed to produce an expected frame: %u",
       frameBufferIndex);
-    result = CAPTURE_RESULT_ERROR;
     goto exit;
   }
 
-  D3D12_RESOURCE_DESC srcFormat = ID3D12Resource_GetDesc(*src);
+  D12FrameFormat srcFormat =
-  D3D12_RESOURCE_DESC dstFormat = this->dstFormat;
+  {
+    .desc       = ID3D12Resource_GetDesc(*src),
+    .colorSpace = desc.colorSpace,
+    .width      = srcFormat.desc.Width,
+    .height     = srcFormat.desc.Height
+  };
+
+  switch(srcFormat.desc.Format)
+  {
+    case DXGI_FORMAT_B8G8R8A8_UNORM:
+      srcFormat.format = CAPTURE_FMT_BGRA;
+      break;
+
+    case DXGI_FORMAT_R8G8B8A8_UNORM:
+      srcFormat.format = CAPTURE_FMT_RGBA;
+      break;
+
+    case DXGI_FORMAT_R10G10B10A2_UNORM:
+      srcFormat.format = CAPTURE_FMT_RGBA10;
+      break;
+
+    case DXGI_FORMAT_R16G16B16A16_FLOAT:
+      srcFormat.format = CAPTURE_FMT_RGBA16F;
+      break;
+
+    default:
+      DEBUG_ERROR("Unsupported source format");
+      goto exit;
+  }
+
+  D12FrameFormat dstFormat = this->dstFormat;
 
   // if the input format changed, reconfigure the effects
-  if (dstFormat.Width  == 0 ||
+  if (srcFormat.desc.Width  == 0 ||
-      dstFormat.Width  != this->captureFormat.Width  ||
+      srcFormat.desc.Width  != this->captureFormat.desc.Width  ||
-      dstFormat.Height != this->captureFormat.Height ||
+      srcFormat.desc.Height != this->captureFormat.desc.Height ||
-      dstFormat.Format != this->captureFormat.Format)
+      srcFormat.desc.Format != this->captureFormat.desc.Format ||
+      srcFormat.colorSpace  != this->captureFormat.colorSpace)
   {
-    dstFormat           = srcFormat;
     this->captureFormat = srcFormat;
 
-    //TODO: loop through an effect array
+    D12Effect * effect;
-    if (this->allowRGB24)
+    vector_forEach(effect, &this->effects)
     {
-      if (!d12_effectSetFormat(
+      dstFormat = srcFormat;
-        this->effectRGB24, *this->device, &srcFormat, &dstFormat))
+      switch(d12_effectSetFormat(effect, *this->device, &srcFormat, &dstFormat))
       {
+        case D12_EFFECT_STATUS_OK:
+          effect->enabled = true;
+          break;
+
+        case D12_EFFECT_STATUS_ERROR:
           DEBUG_ERROR("Failed to set the effect input format");
           goto exit;
+
+        case D12_EFFECT_STATUS_BYPASS:
+          effect->enabled = false;
+          break;
       }
     }
 
     // if the output format changed
-    if (dstFormat.Width  != this->dstFormat.Width  ||
+    if (dstFormat.desc.Width  != this->dstFormat.desc.Width  ||
-        dstFormat.Height != this->dstFormat.Height ||
+        dstFormat.desc.Height != this->dstFormat.desc.Height ||
-        dstFormat.Format != this->dstFormat.Format)
+        dstFormat.desc.Format != this->dstFormat.desc.Format ||
+        dstFormat.colorSpace  != this->dstFormat.colorSpace  ||
+        dstFormat.width       != this->dstFormat.width       ||
+        dstFormat.height      != this->dstFormat.height      ||
+        dstFormat.format      != this->dstFormat.format)
     {
       ++this->formatVer;
       this->dstFormat = dstFormat;
@@ -475,7 +529,7 @@ static CaptureResult d12_waitFrame(unsigned frameBufferIndex,
 
   D3D12_PLACED_SUBRESOURCE_FOOTPRINT layout;
   ID3D12Device3_GetCopyableFootprints(*this->device,
-    &srcFormat,
+    &dstFormat.desc,
     0       , // FirstSubresource
     1       , // NumSubresources
     0       , // BaseOffset,
@@ -488,18 +542,17 @@ static CaptureResult d12_waitFrame(unsigned frameBufferIndex,
   const unsigned int maxRows = maxFrameSize / layout.Footprint.RowPitch;
 
   frame->formatVer        = this->formatVer;
-  frame->screenWidth      = srcFormat.Width;
+  frame->screenWidth      = srcFormat.desc.Width;
-  frame->screenHeight     = srcFormat.Height;
+  frame->screenHeight     = srcFormat.desc.Height;
-  frame->dataWidth        = dstFormat.Width;
+  frame->dataWidth        = dstFormat.desc.Width;
-  frame->dataHeight       = min(maxRows, dstFormat.Height);
+  frame->dataHeight       = min(maxRows, dstFormat.desc.Height);
-  frame->frameWidth       = srcFormat.Width;
+  frame->frameWidth       = dstFormat.width;
-  frame->frameHeight      = srcFormat.Height;
+  frame->frameHeight      = dstFormat.height;
-  frame->truncated        = maxRows < dstFormat.Height;
+  frame->truncated        = maxRows < dstFormat.desc.Height;
   frame->pitch            = layout.Footprint.RowPitch;
   frame->stride           = layout.Footprint.RowPitch / 4;
-  frame->format           = this->allowRGB24 ?
+  frame->format           = dstFormat.format;
-    CAPTURE_FMT_BGR_32 : CAPTURE_FMT_BGRA;
+  frame->hdr              = dstFormat.colorSpace ==
-  frame->hdr              = desc.colorSpace ==
     DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020;
   frame->hdrPQ            = false;
   frame->rotation         = desc.rotation;
@@ -543,7 +596,7 @@ static CaptureResult d12_getFrame(unsigned frameBufferIndex,
   CaptureResult result = CAPTURE_RESULT_ERROR;
   comRef_scopePush(3);
 
-  D12FetchDesc desc;
+  D12FrameDesc desc;
 
   comRef_defineLocal(ID3D12Resource, src);
   *src = d12_backendFetch(this->backend, frameBufferIndex, &desc);
@@ -566,14 +619,19 @@ static CaptureResult d12_getFrame(unsigned frameBufferIndex,
   if (result != CAPTURE_RESULT_OK)
     goto exit;
 
-  const bool isSDR = desc.colorSpace == DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709;
-
   ID3D12Resource * next = *src;
-  if (this->allowRGB24 && isSDR)
+  D12Effect * effect;
+  vector_forEach(effect, &this->effects)
   {
-    next = d12_effectRun(
+    if (!effect->enabled)
-      this->effectRGB24, *this->device, *this->computeCommand.gfxList, next,
+      continue;
-      desc.dirtyRects, &desc.nbDirtyRects);
+
+    next = d12_effectRun(effect,
+      *this->device,
+      *this->computeCommand.gfxList,
+      next,
+      desc.dirtyRects,
+      &desc.nbDirtyRects);
   }
 
   // copy into the framebuffer resource
@@ -593,9 +651,9 @@ static CaptureResult d12_getFrame(unsigned frameBufferIndex,
       .Offset = 0,
       .Footprint =
       {
-        .Format   = this->dstFormat.Format,
+        .Format   = this->dstFormat.desc.Format,
-        .Width    = this->dstFormat.Width,
+        .Width    = this->dstFormat.desc.Width,
-        .Height   = this->dstFormat.Height,
+        .Height   = this->dstFormat.desc.Height,
         .Depth    = 1,
         .RowPitch = this->pitch
       }
@@ -674,7 +732,7 @@ static CaptureResult d12_getFrame(unsigned frameBufferIndex,
   }
 
   // execute the compute commands
-  if (this->allowRGB24 && isSDR)
+  if (next != *src)
   {
     d12_commandGroupExecute(*this->computeQueue, &this->computeCommand);
 
@@ -691,10 +749,10 @@ static CaptureResult d12_getFrame(unsigned frameBufferIndex,
 
   // signal the frame is complete
   framebuffer_set_write_ptr(frameBuffer,
-    this->dstFormat.Height * this->pitch);
+    this->dstFormat.desc.Height * this->pitch);
 
   // reset the command queues
-  if (this->allowRGB24 && isSDR)
+  if (next != *src)
     if (!d12_commandGroupReset(&this->computeCommand))
       goto exit;
 

host/platform/Windows/capture/D12/d12.h

@@ -35,6 +35,26 @@ extern ComScope * d12_comScope;
 void d12_updatePointer(
   CapturePointer * pointer, void * shape, size_t shapeSize);
 
+// Structures for backends and effects
+
+typedef struct D12FrameDesc
+{
+  CaptureRotation       rotation;
+  RECT                * dirtyRects;
+  unsigned              nbDirtyRects;
+  DXGI_COLOR_SPACE_TYPE colorSpace;
+}
+D12FrameDesc;
+
+typedef struct D12FrameFormat
+{
+  D3D12_RESOURCE_DESC   desc;
+  DXGI_COLOR_SPACE_TYPE colorSpace;
+  unsigned              width, height;
+  CaptureFormat         format;
+}
+D12FrameFormat;
+
 // DirectX12 library functions
 
 struct DX12

host/platform/Windows/capture/D12/effect.h

@@ -21,24 +21,36 @@
 #ifndef _H_D12_EFFECT_
 #define _H_D12_EFFECT_
 
+#include "d12.h"
+
 #include <stdbool.h>
 #include <d3d12.h>
 
 typedef struct D12Effect D12Effect;
 
+typedef enum D12EffectStatus
+{
+  D12_EFFECT_STATUS_OK,
+  D12_EFFECT_STATUS_ERROR,
+  D12_EFFECT_STATUS_BYPASS
+}
+D12EffectStatus;
+
 struct D12Effect
 {
   const char * name;
 
+  bool enabled;
+
   bool (*create)(D12Effect ** instance, ID3D12Device3 * device);
 
   void (*free)(D12Effect ** instance);
 
   // set the input format, and get the output format of the effect
-  bool (*setFormat)(D12Effect * effect,
+  D12EffectStatus (*setFormat)(D12Effect * effect,
     ID3D12Device3             * device,
-    const D3D12_RESOURCE_DESC * src,
+    const D12FrameFormat     * src,
-          D3D12_RESOURCE_DESC * dst);
+          D12FrameFormat     * dst);
 
   ID3D12Resource * (*run)(D12Effect * effect,
     ID3D12Device3             * device,
@@ -64,10 +76,10 @@ static inline void d12_effectFree(D12Effect ** instance)
   *instance = NULL;
 }
 
-static inline bool d12_effectSetFormat(D12Effect * effect,
+static inline D12EffectStatus d12_effectSetFormat(D12Effect * effect,
   ID3D12Device3         * device,
-  const D3D12_RESOURCE_DESC * src,
+  const D12FrameFormat * src,
-        D3D12_RESOURCE_DESC * dst)
+        D12FrameFormat * dst)
   { return effect->setFormat(effect, device, src, dst); }
 
 static inline ID3D12Resource * d12_effectRun(D12Effect * effect,

host/platform/Windows/capture/D12/effect/rgb24.c

@@ -213,20 +213,20 @@ static void d12_effect_rgb24Free(D12Effect ** instance)
   free(this);
 }
 
-static bool d12_effect_rgb24SetFormat(D12Effect * effect,
+static D12EffectStatus d12_effect_rgb24SetFormat(D12Effect * effect,
   ID3D12Device3             * device,
-  const D3D12_RESOURCE_DESC * src,
+  const D12FrameFormat * src,
-        D3D12_RESOURCE_DESC * dst)
+        D12FrameFormat * dst)
 {
   TestInstance * this = UPCAST(TestInstance, effect);
   comRef_scopePush(1);
 
-  bool result = false;
+  D12EffectStatus result = D12_EFFECT_STATUS_ERROR;
   HRESULT hr;
 
-  if (src->Format != DXGI_FORMAT_B8G8R8A8_UNORM)
+  if (src->desc.Format != DXGI_FORMAT_B8G8R8A8_UNORM)
   {
-    DEBUG_ERROR("RGB24 requires DXGI_FORMAT_B8G8R8A8_UNORM input");
+    result = D12_EFFECT_STATUS_BYPASS;
     goto exit;
   }
 
@@ -239,12 +239,12 @@ static bool d12_effect_rgb24SetFormat(D12Effect * effect,
     .VisibleNodeMask      = 1
   };
 
-  const unsigned packedPitch = ALIGN_TO(src->Width * 3, 4);
+  const unsigned packedPitch = ALIGN_TO(src->desc.Width * 3, 4);
   D3D12_RESOURCE_DESC desc =
   {
     .Format           = DXGI_FORMAT_B8G8R8A8_UNORM,
     .Width            = ALIGN_TO(packedPitch / 4, 64),
-    .Height           = (src->Width * src->Height) / (packedPitch / 3),
+    .Height           = (src->desc.Width * src->desc.Height) / (packedPitch / 3),
     .Dimension        = D3D12_RESOURCE_DIMENSION_TEXTURE2D,
     .Flags            = D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS,
     .MipLevels        = 1,
@@ -268,8 +268,9 @@ static bool d12_effect_rgb24SetFormat(D12Effect * effect,
   this->threadsX = (desc.Width  + (THREADS-1)) / THREADS;
   this->threadsY = (desc.Height + (THREADS-1)) / THREADS;
 
-  *dst   = desc;
+  dst->desc   = desc;
-  result = true;
+  dst->format = CAPTURE_FMT_BGR_32;
+  result      = D12_EFFECT_STATUS_OK;
 
 exit:
   comRef_scopePop();