[idd] driver/helper: dont allow resolutions that wont fit in ram

idd/LGCommon/PipeMsg.h

@@ -32,7 +32,8 @@ struct LGPipeMsg
     SETCURSORPOS,
     SETDISPLAYMODE,
     GPUSTATUS,
-    RELOADSETTINGS
+    RELOADSETTINGS,
+    RESOLUTIONREJECTED
   }
   type;
   union
@@ -57,5 +58,13 @@ struct LGPipeMsg
       bool software;
     }
     gpuStatus;
+
+    struct
+    {
+      uint32_t width;
+      uint32_t height;
+      uint32_t requiredSizeMiB;
+    }
+    resolutionRejected;
   };
 };

idd/LGIdd/CIndirectDeviceContext.cpp

@@ -46,6 +46,35 @@ static const struct LGMPQueueConfig POINTER_QUEUE_CONFIG =
 
 static const UINT IDDCX_VERSION_1_10 = 0x1A00;
 
+static const UINT64 MIB = 1024ULL * 1024ULL;
+static const UINT64 FRAME_BYTES_PER_PIXEL = 4;
+
+static bool AlignUp(UINT64 value, UINT64 alignment, UINT64& result)
+{
+  if (!alignment || (alignment & (alignment - 1)))
+    return false;
+
+  const UINT64 mask = alignment - 1;
+  if (value > UINT64_MAX - mask)
+    return false;
+
+  result = (value + mask) & ~mask;
+  return true;
+}
+
+static uint32_t RecommendedIVSHMEMSizeMiB(UINT64 requiredSize)
+{
+  UINT64 sizeMiB = requiredSize / MIB;
+  if (requiredSize % MIB)
+    ++sizeMiB;
+
+  UINT64 result = 1;
+  while (result < sizeMiB && result <= UINT32_MAX / 2)
+    result <<= 1;
+
+  return result < sizeMiB ? UINT32_MAX : (uint32_t)result;
+}
+
 #ifdef HAS_IDDCX_110
 static inline IDDCX_WIRE_BITS_PER_COMPONENT GetWireBitsPerComponent(bool hdr)
 {
@@ -756,8 +785,66 @@ NTSTATUS CIndirectDeviceContext::MonitorQueryTargetModes2(
 }
 #endif
 
-void CIndirectDeviceContext::SetResolution(int width, int height)
+bool CIndirectDeviceContext::GetResolutionMemoryRequirements(
+  uint32_t width, uint32_t height, UINT64& frameSize,
+  UINT64& ivshmemSize) const
 {
+  frameSize = 0;
+  ivshmemSize = 0;
+
+  if (!width || !height || !m_alignSize || !m_frameMemoryOffset)
+    return false;
+
+  UINT64 pitch;
+  if (!AlignUp((UINT64)width * FRAME_BYTES_PER_PIXEL,
+      D3D12_TEXTURE_DATA_PITCH_ALIGNMENT, pitch) ||
+      pitch > UINT64_MAX / height)
+    return false;
+
+  frameSize = pitch * height;
+
+  UINT64 frameAllocationSize;
+  if (frameSize > UINT64_MAX - m_alignSize ||
+      !AlignUp(frameSize + m_alignSize, m_alignSize,
+        frameAllocationSize))
+    return false;
+
+  UINT64 frameMemoryStart;
+  if (!AlignUp(m_frameMemoryOffset, m_alignSize, frameMemoryStart) ||
+      frameAllocationSize >
+        (UINT64_MAX - frameMemoryStart) / LGMP_Q_FRAME_LEN)
+    return false;
+
+  ivshmemSize = frameMemoryStart +
+    frameAllocationSize * LGMP_Q_FRAME_LEN;
+  return true;
+}
+
+void CIndirectDeviceContext::SetResolution(uint32_t width, uint32_t height)
+{
+  UINT64 frameSize;
+  UINT64 requiredIVSHMEMSize;
+  if (!GetResolutionMemoryRequirements(width, height, frameSize,
+      requiredIVSHMEMSize))
+  {
+    DEBUG_WARN("Ignoring invalid resolution request: %ux%u", width, height);
+    return;
+  }
+
+  if (frameSize > m_maxFrameSize)
+  {
+    const uint32_t requiredMiB =
+      RecommendedIVSHMEMSizeMiB(requiredIVSHMEMSize);
+    DEBUG_WARN(
+      "Refusing resolution %ux%u: frame requires %llu bytes, only %llu bytes are available; IVSHMEM must be at least %u MiB",
+      width, height,
+      (unsigned long long)frameSize,
+      (unsigned long long)m_maxFrameSize,
+      requiredMiB);
+    g_pipe.ResolutionRejected(width, height, requiredMiB);
+    return;
+  }
+
   CSettings::DisplayMode mode = {};
   mode.width     = width;
   mode.height    = height;
@@ -915,14 +1002,55 @@ bool CIndirectDeviceContext::SetupLGMP(size_t alignSize)
       sizeof(KVMFRCursor) + sizeof(KVMFRColorTransform));
   }
 
-  m_maxFrameSize = lgmpHostMemAvail(m_lgmp);
-  m_maxFrameSize = (m_maxFrameSize -(m_alignSize - 1)) & ~(m_alignSize - 1);
-  m_maxFrameSize /= LGMP_Q_FRAME_LEN;
-  DEBUG_INFO("Max Frame Size: %u MiB", (unsigned int)(m_maxFrameSize / 1048576LL));
+  if (!m_alignSize || (m_alignSize & (m_alignSize - 1)) ||
+      m_alignSize < sizeof(KVMFRFrame) + sizeof(FrameBuffer))
+  {
+    DEBUG_ERROR("Invalid frame buffer alignment: %llu",
+      (unsigned long long)m_alignSize);
+    return false;
+  }
+
+  const size_t available = lgmpHostMemAvail(m_lgmp);
+  m_frameMemoryOffset = m_ivshmem.GetSize() - available;
+
+  UINT64 alignedFrameMemoryOffset;
+  if (!AlignUp(m_frameMemoryOffset, m_alignSize,
+      alignedFrameMemoryOffset))
+  {
+    DEBUG_ERROR("Unable to align the frame memory offset");
+    return false;
+  }
+
+  const size_t alignmentPadding =
+    (size_t)(alignedFrameMemoryOffset - m_frameMemoryOffset);
+  if (available <= alignmentPadding)
+  {
+    DEBUG_ERROR("Insufficient shared memory for frame buffers");
+    return false;
+  }
+
+  const size_t alignmentMask = m_alignSize - 1;
+  size_t frameAllocationSize =
+    (available - alignmentPadding) / LGMP_Q_FRAME_LEN;
+  frameAllocationSize &= ~alignmentMask;
+  if (frameAllocationSize <= m_alignSize ||
+      frameAllocationSize > UINT32_MAX)
+  {
+    DEBUG_ERROR("Invalid frame allocation size: %llu",
+      (unsigned long long)frameAllocationSize);
+    return false;
+  }
+
+  // The KVMFR frame header and FrameBuffer write position occupy the first
+  // alignment unit. Only the bytes after it are usable for pixel data.
+  m_maxFrameSize = frameAllocationSize - m_alignSize;
+  DEBUG_INFO("Max Frame Data Size: %u MiB",
+    (unsigned int)(m_maxFrameSize / MIB));
 
   for (int i = 0; i < LGMP_Q_FRAME_LEN; ++i)
   {
-    if ((status = lgmpHostMemAllocAligned(m_lgmp, (uint32_t)m_maxFrameSize,
+    if ((status = lgmpHostMemAllocAligned(m_lgmp,
+        (uint32_t)frameAllocationSize,
         (uint32_t)m_alignSize, &m_frameMemory[i])) != LGMP_OK)
     {
       DEBUG_ERROR("lgmpHostMemAllocAligned Failed (Frame): %s", lgmpStatusString(status));

idd/LGIdd/CIndirectDeviceContext.h

@@ -100,6 +100,7 @@ private:
   int m_cursorX = 0, m_cursorY = 0;
 
   size_t         m_alignSize           = 0;
+  size_t         m_frameMemoryOffset   = 0;
   size_t         m_maxFrameSize        = 0;
   int            m_frameIndex          = 0;
   volatile LONG  m_publishedFrameIndex = -1;
@@ -146,6 +147,8 @@ private:
   void ResendCursor();
   void SendColorTransform();
   void InitializeEdid();
+  bool GetResolutionMemoryRequirements(uint32_t width, uint32_t height,
+    UINT64& frameSize, UINT64& ivshmemSize) const;
 
   // Guards m_displayModes and m_edid. The mode list is rebuilt on the LGMP
   // timer thread (SetResolution) while IddCx concurrently enumerates it on its
@@ -196,7 +199,7 @@ public:
     const IDARG_IN_QUERYTARGETMODES2* inArgs, IDARG_OUT_QUERYTARGETMODES* outArgs);
 #endif
 
-  void SetResolution(int width, int height);
+  void SetResolution(uint32_t width, uint32_t height);
 
   size_t GetAlignSize   () const { return m_alignSize     ; }
   size_t GetMaxFrameSize() const { return m_maxFrameSize  ; }

idd/LGIdd/CPipeServer.cpp

@@ -306,3 +306,15 @@ void CPipeServer::SetGPUStatus(bool software)
   msg.gpuStatus.software = software;
   WriteMsg(msg);
 }
+
+void CPipeServer::ResolutionRejected(uint32_t width, uint32_t height,
+  uint32_t requiredSizeMiB)
+{
+  LGPipeMsg msg;
+  msg.size = sizeof(msg);
+  msg.type = LGPipeMsg::RESOLUTIONREJECTED;
+  msg.resolutionRejected.width = width;
+  msg.resolutionRejected.height = height;
+  msg.resolutionRejected.requiredSizeMiB = requiredSizeMiB;
+  WriteMsg(msg);
+}

idd/LGIdd/CPipeServer.h

@@ -68,6 +68,8 @@ class CPipeServer
     void SetCursorPos(uint32_t x, uint32_t y);
     void SetDisplayMode(uint32_t width, uint32_t height, uint32_t refresh);
     void SetGPUStatus(bool software);
+    void ResolutionRejected(uint32_t width, uint32_t height,
+      uint32_t requiredSizeMiB);
 };
 
 extern CPipeServer g_pipe;

idd/LGIddHelper/CNotifyWindow.cpp

@@ -22,12 +22,14 @@
 #include "CConfigWindow.h"
 #include "Resources.h"
 #include <CDebug.h>
+#include <new>
 #include <windowsx.h>
 #include <strsafe.h>
 
 #define WM_NOTIFY_ICON         (WM_USER)
 #define WM_CLEAN_UP_CONFIG     (WM_USER+1)
 #define WM_NO_GPU              (WM_USER+2)
+#define WM_RESOLUTION_REJECTED (WM_USER+3)
 
 #define ID_MENU_SHOW_LOG 3000
 #define ID_MENU_SHOW_CONFIG 3001
@@ -39,6 +41,16 @@
 ATOM CNotifyWindow::s_atom = 0;
 UINT CNotifyWindow::s_taskbarCreated = 0;
 
+namespace
+{
+  struct ResolutionRejectedNotification
+  {
+    uint32_t width;
+    uint32_t height;
+    uint32_t requiredSizeMiB;
+  };
+}
+
 bool CNotifyWindow::registerClass()
 {
   s_taskbarCreated = RegisterWindowMessage(L"TaskbarCreated");
@@ -90,6 +102,16 @@ LRESULT CNotifyWindow::handleMessage(UINT uMsg, WPARAM wParam, LPARAM lParam)
     handleGPUNotification((bool)wParam);
     return 0;
 
+  case WM_RESOLUTION_REJECTED:
+  {
+    std::unique_ptr<ResolutionRejectedNotification> notification(
+      reinterpret_cast<ResolutionRejectedNotification*>(lParam));
+    if (notification)
+      handleResolutionRejected(notification->width, notification->height,
+        notification->requiredSizeMiB);
+    return 0;
+  }
+
   case WM_DISPLAYCHANGE:
     scheduleDisplayCheck(DISPLAY_SETTLE_DELAY);
     return 0;
@@ -251,6 +273,46 @@ void CNotifyWindow::handleGPUNotification(bool hasGPU)
     DEBUG_ERROR_HR(GetLastError(), "Shell_NotifyIcon(NIM_MODIFY)");
 }
 
+void CNotifyWindow::notifyResolutionRejected(uint32_t width, uint32_t height,
+  uint32_t requiredSizeMiB)
+{
+  auto notification = new (std::nothrow) ResolutionRejectedNotification
+    { width, height, requiredSizeMiB };
+  if (!notification)
+  {
+    DEBUG_ERROR("Failed to allocate resolution rejection notification");
+    return;
+  }
+
+  if (!PostMessage(m_hwnd, WM_RESOLUTION_REJECTED, 0,
+      reinterpret_cast<LPARAM>(notification)))
+  {
+    delete notification;
+    DEBUG_ERROR_HR(GetLastError(),
+      "Failed to queue resolution rejection notification");
+  }
+}
+
+void CNotifyWindow::handleResolutionRejected(uint32_t width, uint32_t height,
+  uint32_t requiredSizeMiB)
+{
+  NOTIFYICONDATA nid;
+  memcpy(&nid, &m_iconData, sizeof nid);
+
+  nid.uFlags = NIF_INFO | NIF_SHOWTIP;
+  nid.dwInfoFlags = NIIF_WARNING;
+  StringCbCopy(nid.szInfoTitle, sizeof nid.szInfoTitle,
+    L"Resolution change refused");
+  StringCbPrintf(nid.szInfo, sizeof nid.szInfo,
+    L"The requested resolution %ux%u does not fit in shared memory. "
+    L"Change the total IVSHMEM size to at least %u MiB.",
+    width, height,
+    requiredSizeMiB);
+
+  if (!Shell_NotifyIcon(NIM_MODIFY, &nid))
+    DEBUG_ERROR_HR(GetLastError(), "Shell_NotifyIcon(NIM_MODIFY)");
+}
+
 HWND CNotifyWindow::hwndDialog()
 {
   return m_config ? m_config->hwnd() : nullptr;

idd/LGIddHelper/CNotifyWindow.h

@@ -20,6 +20,7 @@
 
 #pragma once
 #include "CWindow.h"
+#include <stdint.h>
 #include <functional>
 #include <memory>
 #include <optional>
@@ -44,6 +45,8 @@ class CNotifyWindow : public CWindow
   LRESULT onNotifyIcon(UINT uEvent, WORD wIconId, int x, int y);
   void registerIcon();
   void handleGPUNotification(bool hasGPU);
+  void handleResolutionRejected(uint32_t width, uint32_t height,
+    uint32_t requiredSizeMiB);
   void scheduleDisplayCheck(UINT delay);
 
   virtual LRESULT handleMessage(UINT uMsg, WPARAM wParam, LPARAM lParam) override;
@@ -71,6 +74,8 @@ public:
   static bool registerClass();
 
   void setGPU(bool hasGPU);
+  void notifyResolutionRejected(uint32_t width, uint32_t height,
+    uint32_t requiredSizeMiB);
 
   HWND hwndDialog();
   void close();

idd/LGIddHelper/CPipeClient.cpp

@@ -471,6 +471,10 @@ void CPipeClient::Thread()
           HandleGPUStatus(msg);
           break;
 
+        case LGPipeMsg::RESOLUTIONREJECTED:
+          HandleResolutionRejected(msg);
+          break;
+
         default:
           DEBUG_ERROR("Unknown message type %d", msg.type);
           break;
@@ -528,3 +532,11 @@ void CPipeClient::HandleGPUStatus(const LGPipeMsg& msg)
 {
   CNotifyWindow::instance().setGPU(!msg.gpuStatus.software);
 }
+
+void CPipeClient::HandleResolutionRejected(const LGPipeMsg& msg)
+{
+  CNotifyWindow::instance().notifyResolutionRejected(
+    msg.resolutionRejected.width,
+    msg.resolutionRejected.height,
+    msg.resolutionRejected.requiredSizeMiB);
+}

idd/LGIddHelper/CPipeClient.h

@@ -53,6 +53,7 @@ private:
   void HandleSetCursorPos(const LGPipeMsg& msg);
   void HandleSetDisplayMode(const LGPipeMsg& msg);
   void HandleGPUStatus(const LGPipeMsg& msg);
+  void HandleResolutionRejected(const LGPipeMsg& msg);
 
 public:
   ~CPipeClient() { DeInit(); }