From 6579fe90987e5b8436ef7c5966b97d1d13ff98db Mon Sep 17 00:00:00 2001
From: Geoffrey McRae <geoff@hostfission.com>
Date: Mon, 24 Aug 2026 13:03:56 +1000
Subject: [PATCH] [host] d12: bound damage metadata

---
 .../platform/Windows/capture/D12/backend/dd.c | 36 +++++++++++++++----
 host/platform/Windows/capture/D12/d12.c       | 36 +++++++++++++++++--
 2 files changed, 63 insertions(+), 9 deletions(-)

diff --git a/host/platform/Windows/capture/D12/backend/dd.c b/host/platform/Windows/capture/D12/backend/dd.c
index cc7bc07c..3e654900 100644
--- a/host/platform/Windows/capture/D12/backend/dd.c
+++ b/host/platform/Windows/capture/D12/backend/dd.c
@@ -538,7 +538,9 @@ static bool d12_dd_handleFrameUpdate(DDInstance * this, IDXGIResource * res)
       &requiredSize);
     if (FAILED(hr))
     {
-      if (hr != DXGI_ERROR_MORE_DATA)
+      if (hr == DXGI_ERROR_MORE_DATA)
+        goto fullDamage;
+      else
       {
         DEBUG_WINERROR("GetFrameDirtyRects failed", hr);
         goto exit;
@@ -546,6 +548,10 @@ static bool d12_dd_handleFrameUpdate(DDInstance * this, IDXGIResource * res)
     }
     else
     {
+      if (requiredSize % sizeof(*this->current->dirtyRects) ||
+          requiredSize > sizeof(this->current->dirtyRects))
+        goto fullDamage;
+
       unsigned nbDirtyRects = requiredSize / sizeof(*this->current->dirtyRects);
 
       // if there is only one damage rect and it covers the entire frame
@@ -559,21 +565,32 @@ static bool d12_dd_handleFrameUpdate(DDInstance * this, IDXGIResource * res)
       this->current->nbDirtyRects = nbDirtyRects;
     }
 
-    DXGI_OUTDUPL_MOVE_RECT moveRects[
-      (ARRAY_LENGTH(this->current->dirtyRects) - this->current->nbDirtyRects) / 2
-    ];
+    const unsigned moveCapacity =
+      (ARRAY_LENGTH(this->current->dirtyRects) -
+        this->current->nbDirtyRects) / 2;
+    if (!moveCapacity)
+      goto fullDamage;
+
+    DXGI_OUTDUPL_MOVE_RECT moveRects[D12_MAX_DIRTY_RECTS / 2];
+    const UINT moveBufferSize =
+      moveCapacity * sizeof(*moveRects);
     hr = IDXGIOutputDuplication_GetFrameMoveRects(*this->dup,
-      sizeof(moveRects), moveRects, &requiredSize);
+      moveBufferSize, moveRects, &requiredSize);
     if (FAILED(hr))
     {
-      this->current->nbDirtyRects = 0;
-      if (hr != DXGI_ERROR_MORE_DATA)
+      if (hr == DXGI_ERROR_MORE_DATA)
+        goto fullDamage;
+      else
       {
         DEBUG_WINERROR("GetFrameMoveRects failed", hr);
         goto exit;
       }
     }
 
+    if (requiredSize % sizeof(*moveRects) ||
+        requiredSize > moveBufferSize)
+      goto fullDamage;
+
     /* Move rects are seemingly not generated on Windows 10, but incase it
      * becomes a thing in the future we still need to implement this */
     const unsigned moveRectCount = requiredSize / sizeof(*moveRects);
@@ -587,6 +604,10 @@ static bool d12_dd_handleFrameUpdate(DDInstance * this, IDXGIResource * res)
           moveRect->SourcePoint.y == moveRect->DestinationRect.top)
         continue;
 
+      if (this->current->nbDirtyRects >
+          ARRAY_LENGTH(this->current->dirtyRects) - 2)
+        goto fullDamage;
+
       /* Add the source rect to the dirty array */
       this->current->dirtyRects[this->current->nbDirtyRects++] = (RECT)
       {
@@ -605,6 +626,7 @@ static bool d12_dd_handleFrameUpdate(DDInstance * this, IDXGIResource * res)
   }
 
 fullDamage:
+  this->current->nbDirtyRects = 0;
   result = true;
 
 exit:
diff --git a/host/platform/Windows/capture/D12/d12.c b/host/platform/Windows/capture/D12/d12.c
index 40d53080..fdf4feaa 100644
--- a/host/platform/Windows/capture/D12/d12.c
+++ b/host/platform/Windows/capture/D12/d12.c
@@ -802,6 +802,13 @@ static CaptureResult d12_waitFrame(unsigned frameBufferIndex,
     goto exit;
   }
 
+  if (!desc.dirtyRects || desc.nbDirtyRects > D12_MAX_DIRTY_RECTS)
+  {
+    DEBUG_ERROR("D12 backend returned invalid damage metadata");
+    desc.dirtyRects   = this->dirtyRects;
+    desc.nbDirtyRects = 0;
+  }
+
   const D3D12_RESOURCE_DESC srcDesc = ID3D12Resource_GetDesc(*src);
   D12FrameFormat srcFormat =
   {
@@ -943,9 +950,15 @@ static CaptureResult d12_waitFrame(unsigned frameBufferIndex,
     if (effect->enabled)
       d12_effectAdjustDamage(effect, desc.dirtyRects, &desc.nbDirtyRects);
 
+  if (desc.nbDirtyRects > D12_MAX_DIRTY_RECTS)
+  {
+    DEBUG_ERROR("D12 effect returned too many damage rectangles");
+    desc.nbDirtyRects = 0;
+  }
+
   {
     // create a clean list of rects
-    FrameDamageRect allRects[desc.nbDirtyRects];
+    FrameDamageRect allRects[D12_MAX_DIRTY_RECTS];
     unsigned count = 0;
     for(const RECT * rect = desc.dirtyRects;
       rect < desc.dirtyRects + desc.nbDirtyRects; ++rect)
@@ -994,12 +1007,12 @@ static CaptureResult d12_getFrame(
   comRef_scopePush(3);
 
   D12FrameDesc desc;
+  FrameDamageRect allRects[D12_MAX_DIRTY_RECTS * 2];
 
   comRef_defineLocal(ID3D12Resource, src);
   DEBUG_TRACE("d12_backendFetch");
   *src = d12_backendFetch(this->backend, frameBufferIndex, &desc);
   unsigned rectCount = 0;
-  FrameDamageRect allRects[this->nbDirtyRects + desc.nbDirtyRects];
 
   if (!*src)
   {
@@ -1008,6 +1021,19 @@ static CaptureResult d12_getFrame(
     goto exit;
   }
 
+  if (!desc.dirtyRects || desc.nbDirtyRects > D12_MAX_DIRTY_RECTS)
+  {
+    DEBUG_ERROR("D12 backend returned invalid damage metadata");
+    desc.dirtyRects   = this->dirtyRects;
+    desc.nbDirtyRects = 0;
+  }
+
+  if (this->nbDirtyRects > D12_MAX_DIRTY_RECTS)
+  {
+    DEBUG_ERROR("Discarding invalid prior D12 damage metadata");
+    this->nbDirtyRects = 0;
+  }
+
   void * map;
   comRef_defineLocal(ID3D12Resource, dst)
   DEBUG_TRACE("d12_frameBufferToResource");
@@ -1047,6 +1073,12 @@ static CaptureResult d12_getFrame(
       &desc.nbDirtyRects);
   }
 
+  if (desc.nbDirtyRects > D12_MAX_DIRTY_RECTS)
+  {
+    DEBUG_ERROR("D12 effect returned too many damage rectangles");
+    desc.nbDirtyRects = 0;
+  }
+
   // copy into the framebuffer resource
   D3D12_TEXTURE_COPY_LOCATION srcLoc =
   {