[host] NvFBC is now functional

host/.gitignore

@@ -1,4 +1,4 @@
-a## Ignore Visual Studio temporary files, build results, and
+## Ignore Visual Studio temporary files, build results, and
 ## files generated by popular Visual Studio add-ons.
 ##
 ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore

host/Capture/NvFBC.cpp

@@ -15,7 +15,9 @@ namespace Capture
 {
 
   NvFBC::NvFBC() :
-    m_hDLL(NULL)
+    m_initialized(false),
+    m_hDLL(NULL),
+    m_nvFBC(NULL)
   {
   }
 
@@ -81,8 +83,6 @@ namespace Capture
     params.pDevice         = NULL;
     params.dwAdapterIdx    = 0;
 
-    // do not remove this
-    #include "NvFBCSpecial.h"
 
     if (m_fnCreateEx(&params) != NVFBC_SUCCESS)
     {
@@ -93,7 +93,35 @@ namespace Capture
 
 m_maxCaptureWidth = params.dwMaxDisplayWidth;
 m_maxCaptureHeight = params.dwMaxDisplayHeight;
+m_nvFBC = static_cast<NvFBCToSys *>(params.pNvFBC);
 
+NVFBC_TOSYS_SETUP_PARAMS setupParams;
+ZeroMemory(&setupParams, sizeof(NVFBC_TOSYS_SETUP_PARAMS));
+setupParams.dwVersion = NVFBC_TOSYS_SETUP_PARAMS_VER;
+setupParams.eMode = NVFBC_TOSYS_RGB;
+setupParams.bWithHWCursor = TRUE;
+setupParams.bDiffMap = FALSE;
+setupParams.ppBuffer = (void **)&m_frameBuffer;
+setupParams.ppDiffMap = NULL;
+
+if (m_nvFBC->NvFBCToSysSetUp(&setupParams) != NVFBC_SUCCESS)
+{
+  DEBUG_ERROR("NvFBCToSysSetUp Failed");
+  DeInitialize();
+  return false;
+}
+
+// this is required according to NVidia sample code
+Sleep(100);
+
+ZeroMemory(&m_grabFrameParams, sizeof(NVFBC_TOSYS_GRAB_FRAME_PARAMS));
+ZeroMemory(&m_grabInfo, sizeof(NvFBCFrameGrabInfo));
+m_grabFrameParams.dwVersion = NVFBC_TOSYS_GRAB_FRAME_PARAMS_VER;
+m_grabFrameParams.dwFlags = NVFBC_TOSYS_NOFLAGS;
+m_grabFrameParams.dwStartX = 0;
+m_grabFrameParams.dwStartY = 0;
+m_grabFrameParams.eGMode = NVFBC_TOSYS_SOURCEMODE_SCALE;
+m_grabFrameParams.pNvFBCFrameGrabInfo = &m_grabInfo;
 
 m_initialized = true;
 return true;
@@ -101,34 +129,97 @@ namespace Capture
 
   void NvFBC::DeInitialize()
   {
+    m_frameBuffer = NULL;
+
+    if (m_nvFBC)
+    {
+      m_nvFBC->NvFBCToSysRelease();
+      m_nvFBC = NULL;
+    }
+
+    m_maxCaptureWidth = 0;
+    m_maxCaptureHeight = 0;
     m_fnCreateEx = NULL;
     m_fnSetGlobalFlags = NULL;
     m_fnGetStatusEx = NULL;
     m_fnEnable = NULL;
 
+    if (m_hDLL)
+    {
       FreeLibrary(m_hDLL);
       m_hDLL = NULL;
+    }
 
     m_initialized = false;
   }
 
   FrameType NvFBC::GetFrameType()
   {
+    if (!m_initialized)
+      return FRAME_TYPE_INVALID;
+
     return FRAME_TYPE_RGB;
   }
 
   FrameComp NvFBC::GetFrameCompression()
   {
+    if (!m_initialized)
+      return FRAME_COMP_NONE;
+
     return FRAME_COMP_NONE;
   }
 
   size_t NvFBC::GetMaxFrameSize()
   {
-    return 0;
+    if (!m_initialized)
+      return false;
+
+    return m_maxCaptureWidth * m_maxCaptureHeight * 3;
   }
 
-  bool NvFBC::GrabFrame(void * buffer, size_t bufferSize, size_t * outLen)
+  bool NvFBC::GrabFrame(struct FrameInfo & frame)
   {
+    if (!m_initialized)
+      return false;
+
+    const HWND hDesktop = GetDesktopWindow();
+    RECT desktop;
+    GetWindowRect(hDesktop, &desktop);
+
+    m_grabFrameParams.dwTargetWidth  = desktop.right;
+    m_grabFrameParams.dwTargetHeight = desktop.bottom;
+    for(int i = 0; i < 2; ++i)
+    {
+      NVFBCRESULT status = m_nvFBC->NvFBCToSysGrabFrame(&m_grabFrameParams);
+      if (status == NVFBC_SUCCESS)
+      {
+        frame.width   = m_grabInfo.dwWidth;
+        frame.height  = m_grabInfo.dwHeight;
+        frame.stride  = m_grabInfo.dwBufferWidth;
+        frame.outSize = m_grabInfo.dwBufferWidth * m_grabInfo.dwHeight * 3;
+        memcpy(frame.buffer, m_frameBuffer, frame.outSize);
+        return true;
+      }
+
+      if (status == NVFBC_ERROR_DYNAMIC_DISABLE)
+      {
+        DEBUG_ERROR("NvFBC was disabled by someone else");
+        return false;
+      }
+
+      if (status == NVFBC_ERROR_INVALIDATED_SESSION)
+      {
+        DEBUG_WARN("Session was invalidated, attempting to restart");
+        DeInitialize();
+        if (!Initialize())
+        {
+          DEBUG_ERROR("Failed to re-iniaialize");
+          return false;
+        }
+      }
+    }
+
+    DEBUG_ERROR("Failed to grab frame");
     return false;
   }
 

host/Capture/NvFBC.h

@@ -20,7 +20,7 @@ namespace Capture
     enum FrameType GetFrameType();
     enum FrameComp GetFrameCompression();
     size_t GetMaxFrameSize();
-    bool GrabFrame(void * buffer, size_t bufferSize, size_t * outLen);
+    bool GrabFrame(struct FrameInfo & frame);
 
   private:
     bool m_initialized;
@@ -32,5 +32,9 @@ namespace Capture
     NvFBC_EnableFunctionType      m_fnEnable;
 
     DWORD m_maxCaptureWidth, m_maxCaptureHeight;
+    NvFBCToSys * m_nvFBC;
+    void * m_frameBuffer;
+    NvFBCFrameGrabInfo m_grabInfo;
+    NVFBC_TOSYS_GRAB_FRAME_PARAMS m_grabFrameParams;
   };
 };

host/CaptureFactory.h

@@ -6,7 +6,7 @@
 #include "ICapture.h"
 #include "Capture\NvFBC.h"
 
-static class CaptureFactory
+class CaptureFactory
 {
 public:
   static ICapture * GetCaptureDevice()

host/ICapture.h

@@ -2,6 +2,16 @@
 
 #include "common/KVMGFXHeader.h"
 
+struct FrameInfo
+{
+  unsigned int width;
+  unsigned int height;
+  unsigned int stride;
+  void * buffer;
+  size_t bufferSize;
+  size_t outSize;
+};
+
 __interface ICapture
 {
 public:
@@ -10,5 +20,5 @@ public:
   enum FrameType GetFrameType();
   enum FrameComp GetFrameCompression();
   size_t GetMaxFrameSize();
-  bool GrabFrame(void * buffer, size_t bufferSize, size_t * outLen);
+  bool GrabFrame(struct FrameInfo & frame);
 };

host/Service.cpp

@@ -65,11 +65,16 @@ bool Service::Initialize()
   }
 
   KVMGFXHeader * header = static_cast<KVMGFXHeader*>(m_memory);
+
+  // we save this as it might actually be valid
+  UINT16 hostID = header->hostID;
+
   ZeroMemory(header, sizeof(KVMGFXHeader));
   memcpy(header->magic, KVMGFX_HEADER_MAGIC, sizeof(KVMGFX_HEADER_MAGIC));
 
   header->version   = 2;
   header->guestID   = m_ivshmem->GetPeerID();
+  header->hostID    = hostID;
 
   m_initialized = true;
   return true;
@@ -109,17 +114,26 @@ bool Service::Process(HANDLE stopEvent)
   // setup the header
   header->frameType = m_capture->GetFrameType();
   header->compType  = m_capture->GetFrameCompression();
+  header->dataLen   = 0;
+
+  FrameInfo frame;
+  frame.buffer     = data;
+  frame.bufferSize = m_ivshmem->GetSize() - sizeof(KVMGFXHeader);
 
   // capture a frame of data
-  if (!m_capture->GrabFrame(
+  if (!m_capture->GrabFrame(frame))
-    data,
-    m_ivshmem->GetSize() - sizeof(KVMGFXHeader),
-    &header->dataLen))
   {
+    header->dataLen = 0;
     DEBUG_ERROR("Capture failed");
     return false;
   }
 
+  // copy the frame details into the header
+  header->width   = frame.width;
+  header->height  = frame.height;
+  header->stride  = frame.stride;
+  header->dataLen = frame.outSize;
+
   // tell the host where the cursor is
   POINT cursorPos;
   GetCursorPos(&cursorPos);

host/Util.h

@@ -3,7 +3,7 @@
 
 #include "common\debug.h"
 
-static class Util
+class Util
 {
 public:
   static std::string GetSystemRoot()