[idd] driver/helper: dont allow resolutions that wont fit in ram

idd/LGIddHelper/CNotifyWindow.cpp

@@ -22,12 +22,14 @@
 #include "CConfigWindow.h"
 #include "Resources.h"
 #include <CDebug.h>
+#include <new>
 #include <windowsx.h>
 #include <strsafe.h>
 
-#define WM_NOTIFY_ICON     (WM_USER)
-#define WM_CLEAN_UP_CONFIG (WM_USER+1)
-#define WM_NO_GPU          (WM_USER+2)
+#define WM_NOTIFY_ICON         (WM_USER)
+#define WM_CLEAN_UP_CONFIG     (WM_USER+1)
+#define WM_NO_GPU              (WM_USER+2)
+#define WM_RESOLUTION_REJECTED (WM_USER+3)
 
 #define ID_MENU_SHOW_LOG 3000
 #define ID_MENU_SHOW_CONFIG 3001
@@ -39,6 +41,16 @@
 ATOM CNotifyWindow::s_atom = 0;
 UINT CNotifyWindow::s_taskbarCreated = 0;
 
+namespace
+{
+  struct ResolutionRejectedNotification
+  {
+    uint32_t width;
+    uint32_t height;
+    uint32_t requiredSizeMiB;
+  };
+}
+
 bool CNotifyWindow::registerClass()
 {
   s_taskbarCreated = RegisterWindowMessage(L"TaskbarCreated");
@@ -90,6 +102,16 @@ LRESULT CNotifyWindow::handleMessage(UINT uMsg, WPARAM wParam, LPARAM lParam)
     handleGPUNotification((bool)wParam);
     return 0;
 
+  case WM_RESOLUTION_REJECTED:
+  {
+    std::unique_ptr<ResolutionRejectedNotification> notification(
+      reinterpret_cast<ResolutionRejectedNotification*>(lParam));
+    if (notification)
+      handleResolutionRejected(notification->width, notification->height,
+        notification->requiredSizeMiB);
+    return 0;
+  }
+
   case WM_DISPLAYCHANGE:
     scheduleDisplayCheck(DISPLAY_SETTLE_DELAY);
     return 0;
@@ -251,6 +273,46 @@ void CNotifyWindow::handleGPUNotification(bool hasGPU)
     DEBUG_ERROR_HR(GetLastError(), "Shell_NotifyIcon(NIM_MODIFY)");
 }
 
+void CNotifyWindow::notifyResolutionRejected(uint32_t width, uint32_t height,
+  uint32_t requiredSizeMiB)
+{
+  auto notification = new (std::nothrow) ResolutionRejectedNotification
+    { width, height, requiredSizeMiB };
+  if (!notification)
+  {
+    DEBUG_ERROR("Failed to allocate resolution rejection notification");
+    return;
+  }
+
+  if (!PostMessage(m_hwnd, WM_RESOLUTION_REJECTED, 0,
+      reinterpret_cast<LPARAM>(notification)))
+  {
+    delete notification;
+    DEBUG_ERROR_HR(GetLastError(),
+      "Failed to queue resolution rejection notification");
+  }
+}
+
+void CNotifyWindow::handleResolutionRejected(uint32_t width, uint32_t height,
+  uint32_t requiredSizeMiB)
+{
+  NOTIFYICONDATA nid;
+  memcpy(&nid, &m_iconData, sizeof nid);
+
+  nid.uFlags = NIF_INFO | NIF_SHOWTIP;
+  nid.dwInfoFlags = NIIF_WARNING;
+  StringCbCopy(nid.szInfoTitle, sizeof nid.szInfoTitle,
+    L"Resolution change refused");
+  StringCbPrintf(nid.szInfo, sizeof nid.szInfo,
+    L"The requested resolution %ux%u does not fit in shared memory. "
+    L"Change the total IVSHMEM size to at least %u MiB.",
+    width, height,
+    requiredSizeMiB);
+
+  if (!Shell_NotifyIcon(NIM_MODIFY, &nid))
+    DEBUG_ERROR_HR(GetLastError(), "Shell_NotifyIcon(NIM_MODIFY)");
+}
+
 HWND CNotifyWindow::hwndDialog()
 {
   return m_config ? m_config->hwnd() : nullptr;

idd/LGIddHelper/CNotifyWindow.h

@@ -20,6 +20,7 @@
 
 #pragma once
 #include "CWindow.h"
+#include <stdint.h>
 #include <functional>
 #include <memory>
 #include <optional>
@@ -44,6 +45,8 @@ class CNotifyWindow : public CWindow
   LRESULT onNotifyIcon(UINT uEvent, WORD wIconId, int x, int y);
   void registerIcon();
   void handleGPUNotification(bool hasGPU);
+  void handleResolutionRejected(uint32_t width, uint32_t height,
+    uint32_t requiredSizeMiB);
   void scheduleDisplayCheck(UINT delay);
 
   virtual LRESULT handleMessage(UINT uMsg, WPARAM wParam, LPARAM lParam) override;
@@ -71,6 +74,8 @@ public:
   static bool registerClass();
 
   void setGPU(bool hasGPU);
+  void notifyResolutionRejected(uint32_t width, uint32_t height,
+    uint32_t requiredSizeMiB);
 
   HWND hwndDialog();
   void close();

idd/LGIddHelper/CPipeClient.cpp

@@ -471,6 +471,10 @@ void CPipeClient::Thread()
           HandleGPUStatus(msg);
           break;
 
+        case LGPipeMsg::RESOLUTIONREJECTED:
+          HandleResolutionRejected(msg);
+          break;
+
         default:
           DEBUG_ERROR("Unknown message type %d", msg.type);
           break;
@@ -528,3 +532,11 @@ void CPipeClient::HandleGPUStatus(const LGPipeMsg& msg)
 {
   CNotifyWindow::instance().setGPU(!msg.gpuStatus.software);
 }
+
+void CPipeClient::HandleResolutionRejected(const LGPipeMsg& msg)
+{
+  CNotifyWindow::instance().notifyResolutionRejected(
+    msg.resolutionRejected.width,
+    msg.resolutionRejected.height,
+    msg.resolutionRejected.requiredSizeMiB);
+}

idd/LGIddHelper/CPipeClient.h

@@ -53,6 +53,7 @@ private:
   void HandleSetCursorPos(const LGPipeMsg& msg);
   void HandleSetDisplayMode(const LGPipeMsg& msg);
   void HandleGPUStatus(const LGPipeMsg& msg);
+  void HandleResolutionRejected(const LGPipeMsg& msg);
 
 public:
   ~CPipeClient() { DeInit(); }