[host] converted program to a windows hidden/background user app.

NvFBC will not start when running as a service, likely intentional
security of the API to prevent it from being abused.

host/Capture/NvFBC.cpp

@@ -55,13 +55,25 @@ namespace Capture
     status.dwVersion    = NVFBC_STATUS_VER;
     status.dwAdapterIdx = 0;
 
-    if (m_fnGetStatusEx(&status) != NVFBC_SUCCESS)
+    NVFBCRESULT ret = m_fnGetStatusEx(&status);
+    if (ret != NVFBC_SUCCESS)
     {
-      DEBUG_ERROR("Failed to get NvFBC status");
+      DEBUG_INFO("Attempting to enable NvFBC");
-      DeInitialize();
+      if (m_fnEnable(NVFBC_STATE_ENABLE) == NVFBC_SUCCESS)
-      return false;
+      {
+        DEBUG_INFO("Success, attempting to get status again");
+        ret = m_fnGetStatusEx(&status);
+      }
+
+      if (ret != NVFBC_SUCCESS)
+      {
+        DEBUG_ERROR("Failed to get NvFBC status");
+        DeInitialize();
+        return false;
+      }
     }
 
+
     if (!status.bIsCapturePossible)
     {
       DEBUG_ERROR("Capture is not possible, unsupported device or driver");

host/Service.cpp

@@ -97,7 +97,7 @@ void Service::DeInitialize()
   m_initialized = false;
 }
 
-bool Service::Process(HANDLE stopEvent)
+bool Service::Process()
 {
   if (!m_initialized)
     return false;
@@ -142,10 +142,7 @@ bool Service::Process(HANDLE stopEvent)
 
   // wait for the host to notify that is it is ready to proceed
   ResetEvent(m_readyEvent);
-  while(
+  while(true)
-    stopEvent == INVALID_HANDLE_VALUE ||
-    (WaitForSingleObject(stopEvent, 0) == WAIT_OBJECT_0)
-  )
   {
     if (!m_ivshmem->RingDoorbell(header->hostID, 0))
     {

host/Service.h

@@ -19,7 +19,7 @@ public:
 
   bool Initialize();
   void DeInitialize();
-  bool Process(HANDLE stopEvent);
+  bool Process();
 
 private:
   static Service * m_instance;

host/kvm-ivshmem-host.vcxproj

@@ -94,7 +94,7 @@
       <AdditionalIncludeDirectories>..\;.\;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
-      <SubSystem>Console</SubSystem>
+      <SubSystem>Windows</SubSystem>
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <AdditionalDependencies>kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;setupapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <AdditionalLibraryDirectories>%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
@@ -111,7 +111,7 @@
       <AdditionalIncludeDirectories>..\;.\;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
-      <SubSystem>Console</SubSystem>
+      <SubSystem>Windows</SubSystem>
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <AdditionalDependencies>kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;setupapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <AdditionalLibraryDirectories>%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
@@ -130,7 +130,7 @@
       <AdditionalIncludeDirectories>..\;.\;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
-      <SubSystem>Console</SubSystem>
+      <SubSystem>Windows</SubSystem>
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>true</GenerateDebugInformation>
@@ -151,7 +151,7 @@
       <AdditionalIncludeDirectories>..\;.\;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
-      <SubSystem>Console</SubSystem>
+      <SubSystem>Windows</SubSystem>
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>true</GenerateDebugInformation>

host/main.cpp

@@ -4,161 +4,22 @@
 
 #include "Service.h"
 
-#define SERVICE_NAME "kvm-ivshmem-host"
+int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, PSTR szCmdParam, int iCmdShow)
-
-//=============================================================================
-
-struct App
 {
-  SERVICE_STATUS        serviceStatus;
+#ifdef DEBUG
-  SERVICE_STATUS_HANDLE statusHandle;
+  AllocConsole();
-  HANDLE                serviceStopEvent;
+#endif
-};
 
-struct App app =
-{
-  {0},
-  NULL,
-  INVALID_HANDLE_VALUE
-};
-
-//=============================================================================
-
-VOID WINAPI ServiceMain(DWORD argc, LPTSTR *argv);
-VOID WINAPI ServiceCtrlHandler(DWORD CtrlCode);
-DWORD WINAPI ServiceWorkerThread(LPVOID lpParam);
-
-//=============================================================================
-
-int main(int argc, TCHAR *argv[])
-{
-  SERVICE_TABLE_ENTRY ServiceTable[] =
-  {
-    {_T(SERVICE_NAME), (LPSERVICE_MAIN_FUNCTION)ServiceMain},
-    {NULL, NULL}
-  };
-
-  if (StartServiceCtrlDispatcher(ServiceTable) == FALSE)
-  {
-    DWORD lastError = GetLastError();
-    if (lastError == ERROR_FAILED_SERVICE_CONTROLLER_CONNECT)
-    {
-      ServiceWorkerThread(INVALID_HANDLE_VALUE);
-      getc(stdin);
-      return 0;
-    }
-
-    return lastError;
-  }
-
-  return 0;
-}
-
-//=============================================================================
-
-VOID WINAPI ServiceMain(DWORD argc, LPTSTR *argv)
-{
-  app.statusHandle = RegisterServiceCtrlHandler(_T(SERVICE_NAME), ServiceCtrlHandler);
-  if (!app.statusHandle)
-    return;
-
-  ZeroMemory(&app.serviceStatus, sizeof(app.serviceStatus));
-  app.serviceStatus.dwServiceType             = SERVICE_WIN32_OWN_PROCESS;
-  app.serviceStatus.dwControlsAccepted        = 0;
-  app.serviceStatus.dwCurrentState            = SERVICE_START_PENDING;
-  app.serviceStatus.dwWin32ExitCode           = 0;
-  app.serviceStatus.dwServiceSpecificExitCode = 0;
-  app.serviceStatus.dwCheckPoint              = 0;
-
-  if (SetServiceStatus(app.statusHandle, &app.serviceStatus) == FALSE)
-  {
-    DEBUG_ERROR("SetServiceStatus failed");
-    return;
-  }
-
-  app.serviceStopEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
-  if (app.serviceStopEvent == INVALID_HANDLE_VALUE)
-  {
-    app.serviceStatus.dwControlsAccepted = 0;
-    app.serviceStatus.dwCurrentState     = SERVICE_STOPPED;
-    app.serviceStatus.dwWin32ExitCode    = GetLastError();
-    app.serviceStatus.dwCheckPoint       = 1;
-    if (SetServiceStatus(app.statusHandle, &app.serviceStatus) == FALSE)
-    {
-      DEBUG_ERROR("SetServiceStatus failed");
-      return;
-    }
-  }
-
-  app.serviceStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP;
-  app.serviceStatus.dwCurrentState     = SERVICE_RUNNING;
-  app.serviceStatus.dwWin32ExitCode    = 0;
-  app.serviceStatus.dwCheckPoint       = 0;
-  if (SetServiceStatus(app.statusHandle, &app.serviceStatus) == FALSE)
-  {
-    DEBUG_ERROR("SetServiceStatus failed");
-    return;
-  }
-
-  HANDLE hThread = CreateThread(NULL, 0, ServiceWorkerThread, NULL, 0, NULL);
-  WaitForSingleObject(hThread, INFINITE);
-
-  CloseHandle(app.serviceStopEvent);
-  app.serviceStatus.dwControlsAccepted = 0;
-  app.serviceStatus.dwCurrentState     = SERVICE_STOPPED;
-  app.serviceStatus.dwWin32ExitCode    = 0;
-  app.serviceStatus.dwCheckPoint       = 3;
-
-  if (SetServiceStatus(app.statusHandle, &app.serviceStatus) == FALSE)
-  {
-    DEBUG_ERROR("SetServiceStatus failed");
-    return;
-  }
-}
-
-//=============================================================================
-
-VOID WINAPI ServiceCtrlHandler(DWORD CtrlCode)
-{
-  switch (CtrlCode)
-  {
-    case SERVICE_CONTROL_STOP:
-      if (app.serviceStatus.dwCurrentState != SERVICE_RUNNING)
-        break;
-
-      app.serviceStatus.dwControlsAccepted = 0;
-      app.serviceStatus.dwCurrentState     = SERVICE_STOP_PENDING;
-      app.serviceStatus.dwWin32ExitCode    = 0;
-      app.serviceStatus.dwCheckPoint       = 4;
-
-      if (SetServiceStatus(app.statusHandle, &app.serviceStatus) == FALSE)
-        DEBUG_ERROR("SetServiceStatus failed");
-
-      SetEvent(app.serviceStopEvent);
-      break;
-
-    default:
-      break;
-  }
-}
-
-//=============================================================================
-
-DWORD WINAPI ServiceWorkerThread(LPVOID lpParam)
-{
   Service *svc = svc->Get();
   if (!svc->Initialize())
   {
     DEBUG_ERROR("Failed to initialize service");
-    return ERROR;
+    return -1;
   }
 
-  while (WaitForSingleObject(app.serviceStopEvent, 0) != WAIT_OBJECT_0)
+  while (true)
-    if (!svc->Process(app.serviceStopEvent))
+    svc->Process();
-      break;
 
   svc->DeInitialize();
-  return ERROR_SUCCESS;
+  return 0;
-}
+}
-
-//=============================================================================