From 57e20007dbd8f690b82b06760bd0bb705d0f9124 Mon Sep 17 00:00:00 2001
From: Geoffrey McRae <geoff@hostfission.com>
Date: Wed, 18 May 2022 15:38:42 +1000
Subject: [PATCH] [host] dxgi: don't try to use devices without d3d support

This change should allow LG to work even if a virtual device is still
attached to the VM even though it might be capturing the wrong display.
---
 host/platform/Windows/capture/DXGI/src/dxgi.c | 29 ++++++++++++++-----
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/host/platform/Windows/capture/DXGI/src/dxgi.c b/host/platform/Windows/capture/DXGI/src/dxgi.c
index 396373d2..2417db6d 100644
--- a/host/platform/Windows/capture/DXGI/src/dxgi.c
+++ b/host/platform/Windows/capture/DXGI/src/dxgi.c
@@ -309,6 +309,20 @@ static bool dxgi_init(void)
         goto fail;
       }
 
+      // check for virtual devices without D3D support
+      if (
+          // Microsoft Basic Render Driver
+          (adapterDesc.VendorId == 0x1414 && adapterDesc.DeviceId == 0x008c) ||
+          // QXL
+          (adapterDesc.VendorId == 0x1b36 && adapterDesc.DeviceId == 0x000d) ||
+          // QEMU Standard VGA
+          (adapterDesc.VendorId == 0x1234 && adapterDesc.DeviceId == 0x1111))
+      {
+        DEBUG_INFO("Not using unsupported adapter: %ls",
+            adapterDesc.Description);
+        goto next_adapter;
+      }
+
       const size_t s = (wcslen(adapterDesc.Description)+1) * 2;
       char * desc = malloc(s);
       wcstombs(desc, adapterDesc.Description, s);
@@ -317,13 +331,16 @@ static bool dxgi_init(void)
       {
         DEBUG_INFO("Not using adapter: %ls", adapterDesc.Description);
         free(desc);
-        IDXGIAdapter1_Release(this->adapter);
-        this->adapter = NULL;
-        continue;
+        goto next_adapter;
       }
       free(desc);
 
       DEBUG_INFO("Adapter matched, trying: %ls", adapterDesc.Description);
+      break;
+
+next_adapter:
+      IDXGIAdapter1_Release(this->adapter);
+      this->adapter = NULL;
     }
 
     for (int n = 0; IDXGIAdapter1_EnumOutputs(this->adapter, n, &this->output) != DXGI_ERROR_NOT_FOUND; ++n)
@@ -339,11 +356,8 @@ static bool dxgi_init(void)
         {
           DEBUG_INFO("Not using adapter output: %ls", outputDesc.DeviceName);
           free(desc);
-          IDXGIOutput_Release(this->output);
-          this->output = NULL;
-          continue;
+          goto next_output;
         }
-
         free(desc);
 
         DEBUG_INFO("Adapter output matched, trying: %ls", outputDesc.DeviceName);
@@ -352,6 +366,7 @@ static bool dxgi_init(void)
       if (outputDesc.AttachedToDesktop)
         break;
 
+next_output:
       IDXGIOutput_Release(this->output);
       this->output = NULL;
     }