From ffa72c7992087d1f94e9ce8e649775c10ac04ce2 Mon Sep 17 00:00:00 2001 From: Quantum Date: Wed, 20 Jan 2021 02:11:49 -0500 Subject: [PATCH] [host] nvfbc: force composition to capture some full screen apps NvFBC is unable to capture certain applications that bypasses the DWM compositor, for example, Firefox playing video in full screen. This has been a known issue for a long time with Nvidia's ShadowPlay, see: * https://www.nvidia.com/en-us/geforce/forums/geforce-experience/14/233709/ * https://crbug.com/609857 Nvidia won't fix this, but there are workarounds. For example, we create a transparent 1x1 layered window, which forces desktop composition to be enabled. Note that SetLayeredWindowAttributes also supports alpha-based transparency, but setting transparency to 0 will cause DWM to skip composition. We could use a transparency of 1, but this ruins the image by the slightest bit, which is unacceptable. Therefore, we must use chroma key-based transparency, which tricks DWM into compositing despite being fully transparent. --- host/platform/Windows/CMakeLists.txt | 1 + .../Windows/capture/NVFBC/src/nvfbc.c | 4 + .../Windows/include/windows/force_compose.h | 21 +++ host/platform/Windows/src/force_compose.c | 139 ++++++++++++++++++ 4 files changed, 165 insertions(+) create mode 100644 host/platform/Windows/include/windows/force_compose.h create mode 100644 host/platform/Windows/src/force_compose.c diff --git a/host/platform/Windows/CMakeLists.txt b/host/platform/Windows/CMakeLists.txt index 29a08815..97dcb837 100644 --- a/host/platform/Windows/CMakeLists.txt +++ b/host/platform/Windows/CMakeLists.txt @@ -9,6 +9,7 @@ add_library(platform_Windows STATIC src/platform.c src/service.c src/mousehook.c + src/force_compose.c ) # allow use of functions for Windows Vista or later diff --git a/host/platform/Windows/capture/NVFBC/src/nvfbc.c b/host/platform/Windows/capture/NVFBC/src/nvfbc.c index ffc5bfd1..ec1c5522 100644 --- a/host/platform/Windows/capture/NVFBC/src/nvfbc.c +++ b/host/platform/Windows/capture/NVFBC/src/nvfbc.c @@ -21,6 +21,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "interface/platform.h" #include "common/windebug.h" #include "windows/mousehook.h" +#include "windows/force_compose.h" #include "common/option.h" #include "common/framebuffer.h" #include "common/event.h" @@ -191,6 +192,8 @@ static bool nvfbc_init(void) if (this->seperateCursor) this->cursorEvents[1] = lgWrapEvent(event); + dwmForceComposition(); + DEBUG_INFO("Cursor mode : %s", this->seperateCursor ? "decoupled" : "integrated"); Sleep(100); @@ -221,6 +224,7 @@ static void nvfbc_stop(void) static bool nvfbc_deinit(void) { mouseHook_remove(); + dwmUnforceComposition(); if (this->cursorEvents[0]) { diff --git a/host/platform/Windows/include/windows/force_compose.h b/host/platform/Windows/include/windows/force_compose.h new file mode 100644 index 00000000..f179944c --- /dev/null +++ b/host/platform/Windows/include/windows/force_compose.h @@ -0,0 +1,21 @@ +/* +Looking Glass - KVM FrameRelay (KVMFR) Client +Copyright (C) 2021 Guanzhong Chen +https://looking-glass.io + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +void dwmForceComposition(void); +void dwmUnforceComposition(void); diff --git a/host/platform/Windows/src/force_compose.c b/host/platform/Windows/src/force_compose.c new file mode 100644 index 00000000..287c265c --- /dev/null +++ b/host/platform/Windows/src/force_compose.c @@ -0,0 +1,139 @@ +/* +Looking Glass - KVM FrameRelay (KVMFR) Client +Copyright (C) 2021 Guanzhong Chen +https://looking-glass.io + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#include "windows/force_compose.h" +#include "common/windebug.h" + +#include +#include + +struct ForceCompose +{ + HANDLE event; + HANDLE thread; +}; + +static struct ForceCompose forceCompose = { 0 }; + +LRESULT CALLBACK windowProc(HWND hwnd, UINT message, WPARAM wParam, LPARAM lParam) +{ + switch (message) + { + case WM_CLOSE: + return 0; // Don't allow close + + case WM_DESTROY: + PostQuitMessage(0); + return 0; + + case WM_PAINT: + { + PAINTSTRUCT ps; + HDC hdc = BeginPaint(hwnd, &ps); + FillRect(hdc, &ps.rcPaint, (HBRUSH) (COLOR_WINDOW+1)); + EndPaint(hwnd, &ps); + return 0; + } + } + return DefWindowProc(hwnd, message, wParam, lParam); +} + +static DWORD WINAPI threadProc(LPVOID lParam) +{ + WNDCLASSA wc = { 0 }; + + wc.lpfnWndProc = windowProc; + wc.hInstance = (HINSTANCE) GetModuleHandle(NULL); + wc.lpszClassName = "looking-glass-force-composition"; + RegisterClass(&wc); + + HWND hwnd = CreateWindowEx( + WS_EX_TOOLWINDOW | WS_EX_TOPMOST | WS_EX_LAYERED, wc.lpszClassName, + "Looking Glass Helper Window", WS_POPUP, 0, 0, 1, 1, NULL, NULL, + wc.hInstance, NULL + ); + + if (!hwnd) + { + DEBUG_ERROR("Failed to create window to force composition"); + goto exit; + } + + SetLayeredWindowAttributes(hwnd, GetSysColor(COLOR_WINDOW), 0, LWA_COLORKEY); + ShowWindow(hwnd, SW_SHOW); + DEBUG_INFO("Created window to force composition"); + + MSG msg; + while (true) + { + switch (MsgWaitForMultipleObjects(1, &forceCompose.event, FALSE, INFINITE, QS_ALLINPUT)) + { + case WAIT_OBJECT_0: + DEBUG_INFO("Force composition received quit request"); + DestroyWindow(hwnd); + + // Do not wait on the event after it has been signaled. + while (GetMessage(&msg, NULL, 0, 0) > 0) + { + TranslateMessage(&msg); + DispatchMessage(&msg); + } + goto exit; + case WAIT_OBJECT_0 + 1: + while (PeekMessage(&msg, NULL, 0, 0, PM_REMOVE)) + { + if (msg.message == WM_QUIT) + goto exit; + TranslateMessage(&msg); + DispatchMessage(&msg); + } + break; + default: + goto exit; + } + } + + exit: + UnregisterClass(wc.lpszClassName, wc.hInstance); + return 0; +} + +void dwmForceComposition(void) +{ + if (!forceCompose.event) + { + forceCompose.event = CreateEventA(NULL, FALSE, FALSE, NULL); + if (!forceCompose.event) + { + DEBUG_WINERROR("Failed to create unforce composition event", GetLastError()); + return; + } + } + forceCompose.thread = CreateThread(NULL, 0, threadProc, NULL, 0, NULL); +} + +void dwmUnforceComposition(void) +{ + if (!forceCompose.event) + return; + SetEvent(forceCompose.event); + WaitForSingleObject(forceCompose.thread, INFINITE); + CloseHandle(forceCompose.thread); + forceCompose.thread = NULL; +}