[host] nvfbc: initial updates to re-enable support

host/Capture/NvFBC.cpp

@@ -48,6 +48,11 @@ NvFBC::~NvFBC()
 {
 }
 
+bool Capture::NvFBC::CanInitialize()
+{
+  return true;
+}
+
 bool NvFBC::Initialize(CaptureOptions * options)
 {
   if (m_initialized)
@@ -89,21 +94,36 @@ bool NvFBC::Initialize(CaptureOptions * options)
   NVFBCRESULT ret = m_fnGetStatusEx(&status);
   if (ret != NVFBC_SUCCESS)
   {
-    DEBUG_INFO("Attempting to enable NvFBC");
+    DEBUG_ERROR("Failed to get NvFBC status");
-    if (m_fnEnable(NVFBC_STATE_ENABLE) == NVFBC_SUCCESS)
+    DeInitialize();
-    {
+    return false;
-      DEBUG_INFO("Success, attempting to get status again");
-      ret = m_fnGetStatusEx(&status);
   }
 
-    if (ret != NVFBC_SUCCESS)
+  if (!status.bIsCapturePossible)
+  {
+    DEBUG_INFO("Attempting to enable NvFBC");
+    switch(m_fnEnable(NVFBC_STATE_ENABLE))
+    {
+      case NVFBC_SUCCESS:
+        DEBUG_INFO("Success, attempting to get status again");
+        if (m_fnGetStatusEx(&status) != NVFBC_SUCCESS)
         {
           DEBUG_ERROR("Failed to get NvFBC status");
           DeInitialize();
           return false;
         }
-  }
+        break;
 
+      case NVFBC_ERROR_INSUFFICIENT_PRIVILEGES:
+        DEBUG_ERROR("Please run once as administrator to enable the NvFBC API");
+        DeInitialize();
+        return false;
+
+      default:
+        DEBUG_ERROR("Unknown failure enabling NvFBC");
+        DeInitialize();
+        return false;
+    }
 
     if (!status.bIsCapturePossible)
     {
@@ -111,6 +131,7 @@ bool NvFBC::Initialize(CaptureOptions * options)
       DeInitialize();
       return false;
     }
+  }
 
   if (!status.bCanCreateNow)
   {
@@ -143,7 +164,7 @@ bool NvFBC::Initialize(CaptureOptions * options)
   setupParams.eMode             = NVFBC_TOSYS_ARGB;
   setupParams.bWithHWCursor     = TRUE;
   setupParams.bDiffMap          = TRUE;
-  setupParams.eDiffMapBlockSize = (NvU32)NVFBC_TOSYS_DIFFMAP_BLOCKSIZE_128X128;
+  setupParams.eDiffMapBlockSize = NVFBC_TOSYS_DIFFMAP_BLOCKSIZE_128X128;
   setupParams.ppBuffer          = (void **)&m_frameBuffer;
   setupParams.ppDiffMap         = (void **)&m_diffMap;
 
@@ -157,16 +178,24 @@ bool NvFBC::Initialize(CaptureOptions * options)
   // this is required according to NVidia sample code
   Sleep(100);
 
+  HMONITOR monitor = MonitorFromWindow(GetDesktopWindow(), MONITOR_DEFAULTTOPRIMARY);
+  MONITORINFO monitorInfo;
+  monitorInfo.cbSize = sizeof(MONITORINFO);
+  unsigned int screenWidth, screenHeight;
+  GetMonitorInfo(monitor, &monitorInfo);
+  screenWidth  = monitorInfo.rcMonitor.right  - monitorInfo.rcMonitor.left;
+  screenHeight = monitorInfo.rcMonitor.bottom - monitorInfo.rcMonitor.top;
+
   ZeroMemory(&m_grabFrameParams, sizeof(NVFBC_TOSYS_GRAB_FRAME_PARAMS));
   ZeroMemory(&m_grabInfo, sizeof(NvFBCFrameGrabInfo));
   m_grabFrameParams.dwVersion           = NVFBC_TOSYS_GRAB_FRAME_PARAMS_VER;
   m_grabFrameParams.dwFlags             = m_optNoWait ? NVFBC_TOSYS_NOWAIT : NVFBC_TOSYS_WAIT_WITH_TIMEOUT;
   m_grabFrameParams.dwWaitTime          = 1000;
-  m_grabFrameParams.eGMode = NVFBC_TOSYS_SOURCEMODE_FULL;
+  m_grabFrameParams.eGMode              = NVFBC_TOSYS_SOURCEMODE_CROP;
   m_grabFrameParams.dwStartX            = 0;
   m_grabFrameParams.dwStartY            = 0;
-  m_grabFrameParams.dwTargetWidth = 0;
+  m_grabFrameParams.dwTargetWidth       = screenWidth;
-  m_grabFrameParams.dwTargetHeight = 0;
+  m_grabFrameParams.dwTargetHeight      = screenHeight;
   m_grabFrameParams.pNvFBCFrameGrabInfo = &m_grabInfo;
 
   m_initialized = true;
@@ -204,7 +233,7 @@ FrameType NvFBC::GetFrameType()
   if (!m_initialized)
     return FRAME_TYPE_INVALID;
 
-  return FRAME_TYPE_ARGB;
+  return FRAME_TYPE_BGRA;
 }
 
 size_t NvFBC::GetMaxFrameSize()
@@ -215,21 +244,20 @@ size_t NvFBC::GetMaxFrameSize()
   return m_maxCaptureWidth * m_maxCaptureHeight * 4;
 }
 
-enum GrabStatus NvFBC::GrabFrame(struct FrameInfo & frame, struct CursorInfo & cursor)
+unsigned int Capture::NvFBC::Capture()
 {
   if (!m_initialized)
     return GRAB_STATUS_ERROR;
 
-  for(int i = 0; i < 2; ++i)
+  for (int i = 0; i < 2; ++i)
   {
     NVFBCRESULT status = m_nvFBC->NvFBCToSysGrabFrame(&m_grabFrameParams);
-
     if (status == NVFBC_SUCCESS)
     {
       const int diffW = (m_grabInfo.dwWidth + 0x7F) >> 7;
       const int diffH = (m_grabInfo.dwHeight + 0x7F) >> 7;
       bool hasDiff = false;
-      for(int y = 0; y < diffH && !hasDiff; ++y)
+      for (int y = 0; y < diffH && !hasDiff; ++y)
         for (int x = 0; x < diffW; ++x)
           if (m_diffMap[y * diffW + x])
           {
@@ -242,62 +270,10 @@ enum GrabStatus NvFBC::GrabFrame(struct FrameInfo & frame, struct CursorInfo & c
         i = 0;
         continue;
       }
-
+      break;
-      unsigned int dataWidth;
-      unsigned int dataOffset;
-
-      if (m_optNoCrop)
-      {
-        dataWidth    = m_grabInfo.dwWidth * 4;
-        dataOffset   = 0;
-        frame.width  = m_grabInfo.dwWidth;
-        frame.height = m_grabInfo.dwHeight;
     }
     else
     {
-        // get the actual resolution
-        HMONITOR monitor = MonitorFromWindow(GetDesktopWindow(), MONITOR_DEFAULTTOPRIMARY);
-        MONITORINFO monitorInfo;
-        monitorInfo.cbSize = sizeof(MONITORINFO);
-
-        unsigned int screenWidth, screenHeight;
-        if (!GetMonitorInfo(monitor, &monitorInfo))
-        {
-          DEBUG_WARN("Failed to get monitor dimensions, assuming no cropping required");
-          screenWidth  = m_grabInfo.dwWidth;
-          screenHeight = m_grabInfo.dwHeight;;
-        }
-        else
-        {
-          screenWidth  = monitorInfo.rcMonitor.right - monitorInfo.rcMonitor.left;
-          screenHeight = monitorInfo.rcMonitor.bottom - monitorInfo.rcMonitor.top;
-        }
-
-        // use the smaller or the two dimensions provided just to be sure we don't overflow the buffer
-        const unsigned int realHeight = LG_MIN(m_grabInfo.dwHeight, screenHeight);
-        const unsigned int realWidth  = LG_MIN(m_grabInfo.dwWidth , screenWidth );
-
-        // calculate the new data width and offset to the start of the data
-        dataWidth  = realWidth * 4;
-        dataOffset =
-          (((m_grabInfo.dwHeight - realHeight) >> 1)  * m_grabInfo.dwBufferWidth +
-            ((m_grabInfo.dwWidth  - realWidth ) >> 1)) * 4;
-
-        // update the frame size
-        frame.width  = realWidth;
-        frame.height = realHeight;
-      }
-
-      frame.stride = frame.width;
-      frame.pitch  = dataWidth;
-      uint8_t *src = (uint8_t *)m_frameBuffer + dataOffset;
-      uint8_t *dst = (uint8_t *)frame.buffer;
-      for(unsigned int y = 0; y < frame.height; ++y, dst += dataWidth, src += m_grabInfo.dwBufferWidth * 4)
-        memcpySSE(dst, src, dataWidth);
-
-      return GRAB_STATUS_OK;
-    }
-
       if (status == NVFBC_ERROR_DYNAMIC_DISABLE)
       {
         DEBUG_ERROR("NvFBC was disabled by someone else");
@@ -309,10 +285,62 @@ enum GrabStatus NvFBC::GrabFrame(struct FrameInfo & frame, struct CursorInfo & c
         DEBUG_WARN("Session was invalidated, attempting to restart");
         return GRAB_STATUS_REINIT;
       }
+
+      if (i == 1)
+      {
+        DEBUG_ERROR("NvFBCToSysGrabFrame failed");
+        return GRAB_STATUS_ERROR;
+      }
+    }
   }
 
-  DEBUG_ERROR("Failed to grab frame");
+  // if the capture size doesn't match the screen resolution then re-initialize to avoid
+  // copying black/blank areas of the screen
+  HMONITOR monitor = MonitorFromWindow(GetDesktopWindow(), MONITOR_DEFAULTTOPRIMARY);
+  MONITORINFO monitorInfo;
+  monitorInfo.cbSize = sizeof(MONITORINFO);
+  unsigned int screenWidth, screenHeight;
+  GetMonitorInfo(monitor, &monitorInfo);
+  screenWidth  = monitorInfo.rcMonitor.right - monitorInfo.rcMonitor.left;
+  screenHeight = monitorInfo.rcMonitor.bottom - monitorInfo.rcMonitor.top;
+  if (m_grabInfo.dwWidth != screenWidth || m_grabInfo.dwHeight != screenHeight)
+  {
+    DEBUG_INFO("Resolution change detected");
+    return GRAB_STATUS_REINIT;
+  }
+
+  return GRAB_STATUS_OK | GRAB_STATUS_FRAME;
+}
+
+bool Capture::NvFBC::GetCursor(CursorInfo & cursor)
+{
+  cursor.hasShape = false;
+  cursor.hasPos   = false;
+  cursor.visible  = false;
+  return false;
+}
+
+void Capture::NvFBC::FreeCursor()
+{
+}
+
+GrabStatus Capture::NvFBC::DiscardFrame()
+{
+  return GrabStatus();
+}
+
+enum GrabStatus NvFBC::GetFrame(struct FrameInfo & frame)
+{
+  if (!m_initialized)
     return GRAB_STATUS_ERROR;
+
+  frame.width  = m_grabInfo.dwWidth;
+  frame.height = m_grabInfo.dwHeight;
+  frame.stride = m_grabInfo.dwBufferWidth;
+  frame.pitch  = m_grabInfo.dwBufferWidth * 4;
+
+  memcpySSE((uint8_t*)frame.buffer, (uint8_t *)m_frameBuffer, frame.pitch * frame.height);
+  return GRAB_STATUS_OK;
 }
 
 #endif// CONFIG_CAPTURE_NVFBC

host/Capture/NvFBC.h

@@ -37,7 +37,7 @@ namespace Capture
     ~NvFBC();
 
     const char * GetName() { return "NvFBC"; }
-
+    bool CanInitialize();
     bool Initialize(CaptureOptions * options);
     void DeInitialize();
     bool ReInitialize()
@@ -47,7 +47,11 @@ namespace Capture
     }
     enum FrameType GetFrameType();
     size_t GetMaxFrameSize();
-    enum GrabStatus GrabFrame(struct FrameInfo & frame, struct CursorInfo & cursor);
+    unsigned int Capture();
+    enum GrabStatus GetFrame(struct FrameInfo & frame);
+    bool GetCursor(CursorInfo & cursor);
+    void FreeCursor() ;
+    enum GrabStatus DiscardFrame();
 
   private:
     CaptureOptions * m_options;

host/looking-glass-host.vcxproj

@@ -123,42 +123,42 @@
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <LinkIncremental>true</LinkIncremental>
-    <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);;C:\Program Files %28x86%29\NVIDIA Corporation\NVIDIA Capture SDK\inc</IncludePath>
+    <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);C:\Program Files %28x86%29\NVIDIA Corporation\NVIDIA Capture SDK\inc</IncludePath>
     <ExecutablePath>C:\Program Files (x86)\Windows Kits\10\bin\$(TargetPlatformVersion)\$(PreferredToolArchitecture);$(ExecutablePath)</ExecutablePath>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug - NvFBC|Win32'">
     <LinkIncremental>true</LinkIncremental>
-    <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);;C:\Program Files %28x86%29\NVIDIA Corporation\NVIDIA Capture SDK\inc</IncludePath>
+    <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);C:\Program Files %28x86%29\NVIDIA Corporation\NVIDIA Capture SDK\inc</IncludePath>
     <ExecutablePath>C:\Program Files (x86)\Windows Kits\10\bin\$(TargetPlatformVersion)\$(PreferredToolArchitecture);$(ExecutablePath)</ExecutablePath>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <LinkIncremental>true</LinkIncremental>
-    <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);;C:\Program Files %28x86%29\NVIDIA Corporation\NVIDIA Capture SDK\inc</IncludePath>
+    <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);C:\Program Files %28x86%29\NVIDIA Corporation\NVIDIA Capture SDK\inc</IncludePath>
     <ExecutablePath>C:\Program Files (x86)\Windows Kits\10\bin\$(TargetPlatformVersion)\$(PreferredToolArchitecture);$(ExecutablePath)</ExecutablePath>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug - NvFBC|x64'">
     <LinkIncremental>true</LinkIncremental>
-    <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);;C:\Program Files %28x86%29\NVIDIA Corporation\NVIDIA Capture SDK\inc</IncludePath>
+    <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);C:\Program Files %28x86%29\NVIDIA Corporation\NVIDIA Capture SDK\inc</IncludePath>
     <ExecutablePath>C:\Program Files (x86)\Windows Kits\10\bin\$(TargetPlatformVersion)\$(PreferredToolArchitecture);$(ExecutablePath)</ExecutablePath>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <LinkIncremental>false</LinkIncremental>
-    <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);;C:\Program Files %28x86%29\NVIDIA Corporation\NVIDIA Capture SDK\inc</IncludePath>
+    <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);C:\Program Files %28x86%29\NVIDIA Corporation\NVIDIA Capture SDK\inc</IncludePath>
     <ExecutablePath>C:\Program Files (x86)\Windows Kits\10\bin\$(TargetPlatformVersion)\$(PreferredToolArchitecture);$(ExecutablePath)</ExecutablePath>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release - NvFBC|Win32'">
     <LinkIncremental>false</LinkIncremental>
-    <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);;C:\Program Files %28x86%29\NVIDIA Corporation\NVIDIA Capture SDK\inc</IncludePath>
+    <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);C:\Program Files %28x86%29\NVIDIA Corporation\NVIDIA Capture SDK\inc</IncludePath>
     <ExecutablePath>C:\Program Files (x86)\Windows Kits\10\bin\$(TargetPlatformVersion)\$(PreferredToolArchitecture);$(ExecutablePath)</ExecutablePath>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <LinkIncremental>false</LinkIncremental>
-    <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);;C:\Program Files %28x86%29\NVIDIA Corporation\NVIDIA Capture SDK\inc</IncludePath>
+    <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);C:\Program Files %28x86%29\NVIDIA Corporation\NVIDIA Capture SDK\inc</IncludePath>
     <ExecutablePath>C:\Program Files (x86)\Windows Kits\10\bin\$(TargetPlatformVersion)\$(PreferredToolArchitecture);$(ExecutablePath)</ExecutablePath>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release - NvFBC|x64'">
     <LinkIncremental>false</LinkIncremental>
-    <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);;C:\Program Files %28x86%29\NVIDIA Corporation\NVIDIA Capture SDK\inc</IncludePath>
+    <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);C:\Program Files %28x86%29\NVIDIA Corporation\NVIDIA Capture SDK\inc</IncludePath>
     <ExecutablePath>C:\Program Files (x86)\Windows Kits\10\bin\$(TargetPlatformVersion)\$(PreferredToolArchitecture);$(ExecutablePath)</ExecutablePath>
   </PropertyGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">