[c-host] added initial nvfbc support

c-host/cmake/NVFBC.cmake

@@ -0,0 +1 @@
+

c-host/include/interface/capture.h

@@ -36,6 +36,7 @@ typedef enum CaptureFormat
   // frame formats
   CAPTURE_FMT_BGRA  ,
   CAPTURE_FMT_RGBA  ,
+  CAPTURE_FMT_ARGB  ,
   CAPTURE_FMT_RGBA10,
   CAPTURE_FMT_YUV420,
 

c-host/platform/Windows/capture/CMakeLists.txt

@@ -3,7 +3,24 @@ project(capture LANGUAGES C)
 
 include("PreCapture")
 
-add_capture("DXGI")
+if(NOT DEFINED NVFBC_SDK)
+  set(NVFBC_SDK "C:\\Program Files (x86)\\NVIDIA Corporation\\NVIDIA Capture SDK")
+endif()
+  
+if(EXISTS "${NVFBC_SDK}\\inc" AND IS_DIRECTORY "${NVFBC_SDK}\\inc")
+  set(USE_NVFBC ON)
+endif()
+
+option(USE_NVFBC "Enable NVFBC Support" ${USE_NVFBC})
+option(USE_DXGI  "Enable DXGI Support" ON)
+
+if(USE_NVFBC)
+  add_capture("NVFBC")
+endif()
+
+if(USE_DXGI)
+  add_capture("DXGI")
+endif()
 
 include("PostCapture")
 

c-host/platform/Windows/capture/NVFBC/CMakeLists.txt

@@ -0,0 +1,14 @@
+cmake_minimum_required(VERSION 3.0)
+project(capture_NVFBC LANGUAGES C CXX)
+
+add_library(capture_NVFBC STATIC
+	src/nvFBC.c
+	src/wrapper.cpp
+)
+
+include_directories("${NVFBC_SDK}\\inc")
+
+target_include_directories(capture_NVFBC
+	PRIVATE
+		src
+)

c-host/platform/Windows/capture/NVFBC/src/nvfbc.c

@@ -0,0 +1,254 @@
+/*
+Looking Glass - KVM FrameRelay (KVMFR) Client
+Copyright (C) 2017-2019 Geoffrey McRae <geoff@hostfission.com>
+https://looking-glass.hostfission.com
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; either version 2 of the License, or (at your option) any later
+version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+#include "interface/capture.h"
+#include "interface/platform.h"
+#include "debug.h"
+#include <assert.h>
+#include <stdlib.h>
+#include <windows.h>
+
+#include <NvFBC/nvFBC.h>
+#include "wrapper.h"
+
+struct iface
+{
+  bool         reinit;
+  NvFBCToSys * nvfbc;
+
+  void       * pointerShape;
+  unsigned int pointerSize;
+  unsigned int width, height;
+
+  uint8_t * frameBuffer;
+  uint8_t * diffMap;
+
+  NvFBCFrameGrabInfo grabInfo;
+
+  osEventHandle * frameEvent;
+  osEventHandle * pointerEvent;
+};
+
+static struct iface * this = NULL;
+
+static void nvfbc_free();
+
+static void getDesktopSize(unsigned int * width, unsigned int * height)
+{
+  HMONITOR    monitor     = MonitorFromWindow(GetDesktopWindow(), MONITOR_DEFAULTTOPRIMARY);
+  MONITORINFO monitorInfo = {
+    .cbSize = sizeof(MONITORINFO)
+  };
+
+  GetMonitorInfo(monitor, &monitorInfo);
+  CloseHandle(monitor);
+
+  *width  = monitorInfo.rcMonitor.right  - monitorInfo.rcMonitor.left;
+  *height = monitorInfo.rcMonitor.bottom - monitorInfo.rcMonitor.top;
+}
+
+static const char * nvfbc_getName()
+{
+  return "NVFBC (NVidia Frame Buffer Capture)";
+};
+
+static bool nvfbc_create()
+{
+  if (!NvFBCInit())
+    return false;
+
+  this = (struct iface *)calloc(sizeof(struct iface), 1);
+
+  if (!NvFBCToSysCreate(NULL, 0, &this->nvfbc))
+  {
+    nvfbc_free();
+    return false;
+  }
+
+  this->frameEvent = os_createEvent(true);
+  if (!this->frameEvent)
+  {
+    DEBUG_ERROR("failed to create the frame event");
+    nvfbc_free();
+    return false;
+  }
+
+  this->pointerEvent = os_createEvent(true);
+  if (!this->pointerEvent)
+  {
+    DEBUG_ERROR("failed to create the pointer event");
+    nvfbc_free();
+    return false;
+  }
+
+  return true;
+}
+
+static bool nvfbc_init(void * pointerShape, const unsigned int pointerSize)
+{
+  this->reinit       = false;
+  this->pointerShape = pointerShape;
+  this->pointerSize  = pointerSize;
+
+  getDesktopSize(&this->width, &this->height);
+  os_resetEvent(this->frameEvent);
+  os_resetEvent(this->pointerEvent);
+
+  if (!NvFBCToSysSetup(
+    this->nvfbc,
+    BUFFER_FMT_ARGB,
+    true,
+    true,
+    DIFFMAP_BLOCKSIZE_128X128,
+    (void **)&this->frameBuffer,
+    (void **)&this->diffMap
+  ))
+  {
+    return false;
+  }
+
+  Sleep(100);
+
+  return true;
+}
+
+static bool nvfbc_deinit()
+{
+  return true;
+}
+
+static void nvfbc_free()
+{
+  NvFBCToSysRelease(&this->nvfbc);
+
+  if (this->frameEvent)
+    os_freeEvent(this->frameEvent);
+
+  if (this->pointerEvent)
+    os_freeEvent(this->pointerEvent);
+
+  free(this);
+  this = NULL;
+  NvFBCFree();
+}
+
+static unsigned int nvfbc_getMaxFrameSize()
+{
+  return this->width * this->height * 4;
+}
+
+static CaptureResult nvfbc_capture()
+{
+  // check if the resolution has changed, if it has we need to re-init to avoid capturing
+  // black areas as NvFBC doesn't tell us about the change.
+  unsigned int width, height;
+  getDesktopSize(&width, &height);
+  if (this->width != width || this->height != height)
+  {
+    DEBUG_INFO("Resolution change detected");
+
+    this->reinit = true;
+    os_signalEvent(this->frameEvent  );
+    os_signalEvent(this->pointerEvent);
+
+    return CAPTURE_RESULT_REINIT;
+  }
+
+  NvFBCFrameGrabInfo grabInfo;
+  CaptureResult result = NvFBCToSysCapture(
+    this->nvfbc,
+    1000,
+    0, 0,
+    this->width,
+    this->height,
+    &grabInfo
+  );
+
+  if (result != CAPTURE_RESULT_OK)
+    return result;
+
+  // NvFBC doesn't tell us when a timeout occurs, so check the diff map
+  // to see if anything actually changed
+
+  const int dw = (grabInfo.dwWidth  + 0x7F) >> 7;
+  const int dh = (grabInfo.dwHeight + 0x7F) >> 7;
+  bool diff = false;
+  for(int y = 0; y < dh && !diff; ++y)
+    for(int x = 0; x < dw; ++x)
+      if (this->diffMap[y * dw + x])
+      {
+        diff = true;
+        break;
+      }
+
+  if (!diff)
+    return CAPTURE_RESULT_TIMEOUT;
+
+  memcpy(&this->grabInfo, &grabInfo, sizeof(grabInfo));
+  os_signalEvent(this->frameEvent);
+  return CAPTURE_RESULT_OK;
+}
+
+static CaptureResult nvfbc_getFrame(CaptureFrame * frame)
+{
+  if (!os_waitEvent(this->frameEvent, TIMEOUT_INFINITE))
+  {
+    DEBUG_ERROR("Failed to wait on the frame event");
+    return CAPTURE_RESULT_ERROR;
+  }
+
+  if (this->reinit)
+    return CAPTURE_RESULT_REINIT;
+
+  frame->width  = this->grabInfo.dwWidth;
+  frame->height = this->grabInfo.dwHeight;
+  frame->pitch  = this->grabInfo.dwBufferWidth * 4;
+  frame->stride = this->grabInfo.dwBufferWidth;
+  frame->format = CAPTURE_FMT_BGRA;
+
+  memcpy(frame->data, this->frameBuffer, frame->pitch * frame->height);
+  return CAPTURE_RESULT_OK;
+}
+
+static CaptureResult nvfbc_getPointer(CapturePointer * pointer)
+{
+  if (!os_waitEvent(this->pointerEvent, TIMEOUT_INFINITE))
+  {
+    DEBUG_ERROR("Failed to wait on the pointer event");
+    return CAPTURE_RESULT_ERROR;
+  }
+
+  if (this->reinit)
+    return CAPTURE_RESULT_REINIT;
+
+  return CAPTURE_RESULT_ERROR;
+}
+
+struct CaptureInterface Capture_NVFBC =
+{
+  .getName         = nvfbc_getName,
+  .create          = nvfbc_create,
+  .init            = nvfbc_init,
+  .deinit          = nvfbc_deinit,
+  .free            = nvfbc_free,
+  .getMaxFrameSize = nvfbc_getMaxFrameSize,
+  .capture         = nvfbc_capture,
+  .getFrame        = nvfbc_getFrame,
+  .getPointer      = nvfbc_getPointer
+};

c-host/platform/Windows/capture/NVFBC/src/wrapper.cpp

@@ -0,0 +1,196 @@
+/*
+Looking Glass - KVM FrameRelay (KVMFR) Client
+Copyright (C) 2017-2019 Geoffrey McRae <geoff@hostfission.com>
+https://looking-glass.hostfission.com
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; either version 2 of the License, or (at your option) any later
+version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+#include "wrapper.h"
+#include "debug.h"
+#include "windows/windebug.h"
+#include <windows.h>
+
+#ifdef _WIN64
+#define NVFBC_DLL "NvFBC64.dll"
+#else
+#define NVFBC_DLL "NvFBC.dll"
+#endif
+
+struct NVAPI
+{
+  bool initialized;
+  HMODULE dll;
+
+  NvFBC_CreateFunctionExType    createEx;
+  NvFBC_SetGlobalFlagsType      setGlobalFlags;
+  NvFBC_GetStatusExFunctionType getStatusEx;
+  NvFBC_EnableFunctionType      enable;
+};
+
+static NVAPI nvapi;
+
+bool NvFBCInit()
+{
+  if (nvapi.initialized)
+    return true;
+
+  nvapi.dll = LoadLibraryA(NVFBC_DLL);
+  if (!nvapi.dll)
+  {
+    DEBUG_WINERROR("Failed to load " NVFBC_DLL, GetLastError());
+    return false;
+  }
+
+  nvapi.createEx       = (NvFBC_CreateFunctionExType   )GetProcAddress(nvapi.dll, "NvFBC_CreateEx"      );
+  nvapi.setGlobalFlags = (NvFBC_SetGlobalFlagsType     )GetProcAddress(nvapi.dll, "NvFBC_SetGlobalFlags");
+  nvapi.getStatusEx    = (NvFBC_GetStatusExFunctionType)GetProcAddress(nvapi.dll, "NvFBC_GetStatusEx"   );
+  nvapi.enable         = (NvFBC_EnableFunctionType     )GetProcAddress(nvapi.dll, "NvFBC_Enable"        );
+
+  nvapi.initialized = true;
+  return true;
+}
+
+void NvFBCFree()
+{
+  if (!nvapi.initialized)
+    return;
+
+  FreeLibrary(nvapi.dll);
+  nvapi.initialized = false;
+}
+
+bool NvFBCToSysCreate(void * privData, unsigned int privDataSize, NvFBCToSys ** nvfbc)
+{
+  NvFBCCreateParams params = {0};
+
+  params.dwVersion         = NVFBC_CREATE_PARAMS_VER;
+  params.dwInterfaceType   = NVFBC_TO_SYS;
+  params.pDevice           = NULL;
+  params.dwAdapterIdx      = 0;
+  params.dwPrivateDataSize = privDataSize;
+  params.pPrivateData      = privData;
+
+  if (nvapi.createEx(&params) != NVFBC_SUCCESS)
+  {
+    DEBUG_ERROR("Failed to create an instance of NvFBCToSys");
+    *nvfbc = NULL;
+    return false;
+  }
+
+  *nvfbc = static_cast<NvFBCToSys *>(params.pNvFBC);
+  return true;
+}
+
+void NvFBCToSysRelease(NvFBCToSys ** nvfbc)
+{
+  if (!*nvfbc)
+    return;
+
+  (*nvfbc)->NvFBCToSysRelease();
+  (*nvfbc) = NULL;
+}
+
+bool NvFBCToSysSetup(
+  NvFBCToSys          * nvfbc,
+  enum                  BufferFormat format,
+  bool                  hwCursor,
+  bool                  useDiffMap,
+  enum DiffMapBlockSize diffMapBlockSize,
+  void **               frameBuffer,
+  void **               diffMap
+)
+{
+  NVFBC_TOSYS_SETUP_PARAMS params = {0};
+  params.dwVersion = NVFBC_TOSYS_SETUP_PARAMS_VER;
+
+  switch(format)
+  {
+    case BUFFER_FMT_ARGB      : params.eMode = NVFBC_TOSYS_ARGB      ; break;
+    case BUFFER_FMT_RGB       : params.eMode = NVFBC_TOSYS_RGB       ; break;
+    case BUFFER_FMT_YYYYUV420p: params.eMode = NVFBC_TOSYS_YYYYUV420p; break;
+    case BUFFER_FMT_RGB_PLANAR: params.eMode = NVFBC_TOSYS_RGB_PLANAR; break;
+    case BUFFER_FMT_XOR       : params.eMode = NVFBC_TOSYS_XOR       ; break;
+    case BUFFER_FMT_YUV444p   : params.eMode = NVFBC_TOSYS_YUV444p   ; break;
+    case BUFFER_FMT_ARGB10    : params.eMode = NVFBC_TOSYS_ARGB10    ; break;
+
+    default:
+      DEBUG_INFO("Invalid format");
+      return false;
+  }
+
+  params.bWithHWCursor = hwCursor   ? TRUE : FALSE;
+  params.bDiffMap      = useDiffMap ? TRUE : FALSE;
+
+  switch(diffMapBlockSize)
+  {
+    case DIFFMAP_BLOCKSIZE_128X128: params.eDiffMapBlockSize = NVFBC_TOSYS_DIFFMAP_BLOCKSIZE_128X128; break;
+    case DIFFMAP_BLOCKSIZE_16X16  : params.eDiffMapBlockSize = NVFBC_TOSYS_DIFFMAP_BLOCKSIZE_16X16  ; break;
+    case DIFFMAP_BLOCKSIZE_32X32  : params.eDiffMapBlockSize = NVFBC_TOSYS_DIFFMAP_BLOCKSIZE_32X32  ; break;
+    case DIFFMAP_BLOCKSIZE_64X64  : params.eDiffMapBlockSize = NVFBC_TOSYS_DIFFMAP_BLOCKSIZE_64X64  ; break;
+
+    default:
+      DEBUG_ERROR("Invalid diffMapBlockSize");
+      return false;
+  }
+
+  params.ppBuffer  = frameBuffer;
+  params.ppDiffMap = diffMap;
+
+  return nvfbc->NvFBCToSysSetUp(&params) == NVFBC_SUCCESS;
+}
+
+CaptureResult NvFBCToSysCapture(
+  NvFBCToSys * nvfbc,
+  const unsigned int waitTime,
+  const unsigned int x,
+  const unsigned int y,
+  const unsigned int width,
+  const unsigned int height,
+  NvFBCFrameGrabInfo * grabInfo
+)
+{
+  NVFBC_TOSYS_GRAB_FRAME_PARAMS params = {0};
+
+  params.dwVersion           = NVFBC_TOSYS_GRAB_FRAME_PARAMS_VER;
+  params.dwFlags             = NVFBC_TOSYS_WAIT_WITH_TIMEOUT;
+  params.dwWaitTime          = waitTime;
+  params.eGMode              = NVFBC_TOSYS_SOURCEMODE_CROP;
+  params.dwStartX            = x;
+  params.dwStartY            = y;
+  params.dwTargetWidth       = width;
+  params.dwTargetHeight      = height;
+  params.pNvFBCFrameGrabInfo = grabInfo;
+
+  NVFBCRESULT status = nvfbc->NvFBCToSysGrabFrame(&params);
+  switch(status)
+  {
+    case NVFBC_SUCCESS:
+      break;
+
+    case NVFBC_ERROR_DYNAMIC_DISABLE:
+      DEBUG_ERROR("NvFBC was disabled by someone else");
+      return CAPTURE_RESULT_ERROR;
+
+    case NVFBC_ERROR_INVALIDATED_SESSION:
+      DEBUG_WARN("Session was invalidated, attempting to restart");
+      return CAPTURE_RESULT_REINIT;
+
+    default:
+      DEBUG_ERROR("Unknown NVFBCRESULT failure 0x%x", status);
+      return CAPTURE_RESULT_ERROR;
+  }
+
+  return CAPTURE_RESULT_OK;
+}

c-host/platform/Windows/capture/NVFBC/src/wrapper.h

@@ -0,0 +1,82 @@
+/*
+Looking Glass - KVM FrameRelay (KVMFR) Client
+Copyright (C) 2017-2019 Geoffrey McRae <geoff@hostfission.com>
+https://looking-glass.hostfission.com
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; either version 2 of the License, or (at your option) any later
+version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+#include <stdbool.h>
+#include <NvFBC/nvFBC.h>
+
+#ifndef __cplusplus
+typedef void * NvFBCToSys;
+#else
+#include <NvFBC/nvFBCToSys.h>
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "interface/capture.h"
+
+enum BufferFormat
+{
+  BUFFER_FMT_ARGB,
+  BUFFER_FMT_RGB,
+  BUFFER_FMT_YYYYUV420p,
+  BUFFER_FMT_RGB_PLANAR,
+  BUFFER_FMT_XOR,
+  BUFFER_FMT_YUV444p,
+  BUFFER_FMT_ARGB10
+};
+
+enum DiffMapBlockSize
+{
+  DIFFMAP_BLOCKSIZE_128X128 = 0,
+  DIFFMAP_BLOCKSIZE_16X16,
+  DIFFMAP_BLOCKSIZE_32X32,
+  DIFFMAP_BLOCKSIZE_64X64
+};
+
+bool NvFBCInit();
+void NvFBCFree();
+
+bool NvFBCToSysCreate(void * privData, unsigned int privDataSize, NvFBCToSys ** nvfbc);
+void NvFBCToSysRelease(NvFBCToSys ** nvfbc);
+
+bool NvFBCToSysSetup(
+  NvFBCToSys          * nvfbc,
+  enum                  BufferFormat format,
+  bool                  hwCursor,
+  bool                  useDiffMap,
+  enum DiffMapBlockSize diffMapBlockSize,
+  void **               frameBuffer,
+  void **               diffMap
+);
+
+CaptureResult NvFBCToSysCapture(
+  NvFBCToSys * nvfbc,
+  const unsigned int waitTime,
+  const unsigned int x,
+  const unsigned int y,
+  const unsigned int width,
+  const unsigned int height,
+  NvFBCFrameGrabInfo * grabInfo
+);
+
+#ifdef __cplusplus
+}
+#endif

c-host/platform/Windows/include/windows/windebug.h

@@ -22,6 +22,14 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 #include "debug.h"
 #include <windows.h>
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 void DebugWinError(const char * file, const unsigned int line, const char * function, const char * desc, HRESULT status);
 
-#define DEBUG_WINERROR(x, y) DebugWinError(STRIPPATH(__FILE__), __LINE__, __FUNCTION__, x, y)
+#define DEBUG_WINERROR(x, y) DebugWinError(STRIPPATH(__FILE__), __LINE__, __FUNCTION__, x, y)
+
+#ifdef __cplusplus
+}
+#endif