[host] lgmp: account for frame header capacity

host/include/interface/capture.h

@@ -153,10 +153,12 @@ typedef struct CaptureInterface
   CaptureResult (*waitFrame )(
     unsigned frameBufferIndex,
     CaptureFrame * frame,
+    /* Capacity of FrameBuffer::data, excluding the message and header. */
     const size_t maxFrameSize);
   CaptureResult (*getFrame  )(
     unsigned       frameBufferIndex,
     FrameBuffer  * frame,
+    /* Capacity of FrameBuffer::data, excluding the message and header. */
     const size_t   maxFrameSize,
     CaptureFrame * captureFrame);
 }

host/platform/Windows/capture/DXGI/src/dxgi.c

@@ -1526,12 +1526,21 @@ static CaptureResult dxgi_getFrame(
   }
   else
   {
+    if (!this->pitch || !this->dataHeight ||
+        (size_t)this->dataHeight > maxFrameSize / this->pitch ||
+        (size_t)this->dataHeight * this->pitch > UINT_LEAST32_MAX)
+    {
+      result = CAPTURE_RESULT_ERROR;
+      goto cleanup;
+    }
+    const size_t frameSize = (size_t)this->pitch * this->dataHeight;
+
     if (tex->damageRectsCount                 == 0 ||
         damage->count                          < 0 ||
         damage->count + tex->damageRectsCount  > KVMFR_MAX_DAMAGE_RECTS)
     {
       // damage all
-      framebuffer_write(frame, tex->map, this->pitch * this->dataHeight);
+      framebuffer_write(frame, tex->map, frameSize);
     }
     else
     {
@@ -1556,7 +1565,7 @@ static CaptureResult dxgi_getFrame(
               this->bpp, frame, this->pitch, this->dataHeight,
               tex->map, this->pitch))
           framebuffer_write(frame, tex->map,
-              this->pitch * this->dataHeight);
+              frameSize);
       }
       else
       {
@@ -1564,7 +1573,7 @@ static CaptureResult dxgi_getFrame(
               this->bpp, frame, this->pitch, this->dataHeight,
               tex->map, this->pitch))
           framebuffer_write(frame, tex->map,
-              this->pitch * this->dataHeight);
+              frameSize);
       }
     }
   }

host/src/app.c

@@ -98,6 +98,7 @@ struct app
   unsigned int   pointerShapeIndex;
 
   unsigned       alignSize;
+  size_t         frameMemorySize;
   size_t         maxFrameSize;
   PLGMPHostQueue frameQueue;
   PLGMPMemory    frameMemory[LGMP_Q_FRAME_LEN];
@@ -876,14 +877,37 @@ static bool lgmpSetup(struct IVSHMEM * shmDev)
     memset(lgmpHostMemPtr(app.pointerShapeMemory[i]), 0, MAX_POINTER_SIZE);
   }
 
-  app.maxFrameSize = lgmpHostMemAvail(app.lgmp);
-  app.maxFrameSize = (app.maxFrameSize - (app.alignSize - 1)) & ~(app.alignSize - 1);
-  app.maxFrameSize /= LGMP_Q_FRAME_LEN;
-  DEBUG_INFO("Max Frame Size   : %u MiB", (unsigned int)(app.maxFrameSize / 1048576LL));
+  const size_t frameMemoryAvail = lgmpHostMemAvail(app.lgmp);
+  if (!app.alignSize ||
+      (size_t)app.alignSize < sizeof(KVMFRFrame) + sizeof(FrameBuffer) ||
+      (app.alignSize & (app.alignSize - 1)) ||
+      frameMemoryAvail <= app.alignSize - 1)
+  {
+    DEBUG_ERROR("Invalid framebuffer alignment or shared memory capacity");
+    goto fail_lgmp;
+  }
+
+  /* Reserve the worst-case alignment padding once, then round each message
+   * down so all frame allocations are guaranteed to fit. The FrameBuffer data
+   * begins one alignment unit into each message. */
+  app.frameMemorySize = (frameMemoryAvail - (app.alignSize - 1)) /
+    LGMP_Q_FRAME_LEN;
+  app.frameMemorySize &= ~((size_t)app.alignSize - 1);
+  if (app.frameMemorySize <= app.alignSize ||
+      app.frameMemorySize > UINT32_MAX)
+  {
+    DEBUG_ERROR("Insufficient shared memory for the frame buffers");
+    goto fail_lgmp;
+  }
+
+  app.maxFrameSize = app.frameMemorySize - app.alignSize;
+  DEBUG_INFO("Max Frame Size   : %u MiB",
+      (unsigned int)(app.maxFrameSize / 1048576LL));
 
   for(int i = 0; i < LGMP_Q_FRAME_LEN; ++i)
   {
-    if ((status = lgmpHostMemAllocAligned(app.lgmp, app.maxFrameSize,
+    if ((status = lgmpHostMemAllocAligned(app.lgmp,
+            (uint32_t)app.frameMemorySize,
             app.alignSize, &app.frameMemory[i])) != LGMP_OK)
     {
       DEBUG_ERROR("lgmpHostMemAlloc Failed (Frame): %s", lgmpStatusString(status));