Merge pull request #211 from Snawoot/win_fbc_pd_sniffer

win: fbc: sniff/replay

win/nvfbcwrp/README.md

@@ -5,7 +5,7 @@ Wrapper for `NvFBC64.dll` library which injects private keys into `NvFBC_CreateE
 
 ## Usage
 
-1. Obtain `nvfbcwrp64.dll` and `nvfbcwrp32.dll` files. You may build them yourself with MSVS 2019 or download latest release here: [nvfbcwrp64.dll](https://gist.github.com/Snawoot/2d78c569b5ffb29080cf9bfca401adfa/raw/166c5e10f8301000c171d5f59cc2ae0b551cd1b3/nvfbcwrp64.dll), [nvfbcwrp32.dll](https://gist.github.com/Snawoot/2d78c569b5ffb29080cf9bfca401adfa/raw/166c5e10f8301000c171d5f59cc2ae0b551cd1b3/nvfbcwrp32.dll).
+1. Obtain `nvfbcwrp64.dll` and `nvfbcwrp32.dll` files. You may build them yourself with MSVS 2019 or download latest release here: [nvfbcwrp64.dll](https://gist.github.com/Snawoot/17b14e7ce0f7412b91587c2723719eff/raw/e8e9658fd20751ad875477f37b49ea158ece896d/nvfbcwrp64.dll), [nvfbcwrp32.dll](https://gist.github.com/Snawoot/17b14e7ce0f7412b91587c2723719eff/raw/e8e9658fd20751ad875477f37b49ea158ece896d/nvfbcwrp32.dll).
 2. Backup your `%WINDIR\system32\NvFBC64.dll` and `%WINDIR\SysWOW64\NvFBC.dll` files.
 3. Rename file `%WINDIR\system32\NvFBC64.dll` to `%WINDIR\system32\NvFBC64_.dll`
 4. Rename file `%WINDIR\SysWOW64\NvFBC.dll` to `%WINDIR\SysWOW64\NvFBC_.dll`
@@ -14,3 +14,16 @@ Wrapper for `NvFBC64.dll` library which injects private keys into `NvFBC_CreateE
 7. Restart any applications using this library. That's it.
 
 This procedure has to be repeated after any driver reinstall/update, so keep your copies of `nvfbcwrp64.dll` and `nvfbcwrp32.dll` files.
+
+## Advanced Usage
+
+`nvfbcwrp` allows user to capture and replay `privateData` used by other NvFBC applications (like GeForce Experience, Shadow Play and so on). It may be useful if built-in `privateData` will render invalid for some reason. Wrapper recognizes two environment variables:
+
+* `NVFBCWRP_DUMP_DIR` - output directory for dumps of `privateData` sent by applications.
+* `NVFBCWRP_PRIVDATA_FILE` - name of file with `privateData` which should be used instead of default built-in vector. These files can be produced as output of `NVFBCWRP_DUMP_DIR` option. If file is not found or can't be loaded, default magic vector is used.
+
+Hence, if default magic baked into nvfbcwrp doesn't work for you, you have to:
+
+1. Specify environment variable `NVFBCWRP_DUMP_DIR` in your configuration with path to existing writable directory. Here is a [guide](http://web.archive.org/web/20191207221102/https://docs.oracle.com/en/database/oracle/r-enterprise/1.5.1/oread/creating-and-modifying-environment-variables-on-windows.html) about environment variables edit. It's sufficient to add "user" environment variable.
+2. Run some NvFBC application with valid `privateData` keys and initiate recording session.
+3. Grab some output file and specify it's path in `NVFBCWRP_PRIVDATA_FILE`. At this point you can unset `NVFBCWRP_DUMP_DIR` to stop `privateData` capture.

win/nvfbcwrp/framework.h

@@ -3,3 +3,5 @@
 #define WIN32_LEAN_AND_MEAN             // Исключите редко используемые компоненты из заголовков Windows
 // Файлы заголовков Windows
 #include <windows.h>
+#include <new>
+#include <tuple>

win/nvfbcwrp/nvfbcdefs.h

@@ -1,8 +1,5 @@
 #pragma once
 
-// Magic code which is passed as pPrivateData and enables NvFBC to work on GeForce
-int magic[] = { 0x0D7BC620, 0x4C17E142, 0x5E6B5997, 0x4B5A855B };
-
 typedef unsigned long      NvU32; /* 0 to 4294967295                         */
 
 /**

win/nvfbcwrp/nvfbcwrp.vcxproj

@@ -30,27 +30,27 @@
     <ConfigurationType>DynamicLibrary</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
     <PlatformToolset>v142</PlatformToolset>
-    <CharacterSet>MultiByte</CharacterSet>
+    <CharacterSet>Unicode</CharacterSet>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
     <ConfigurationType>DynamicLibrary</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
     <PlatformToolset>v142</PlatformToolset>
     <WholeProgramOptimization>true</WholeProgramOptimization>
-    <CharacterSet>MultiByte</CharacterSet>
+    <CharacterSet>Unicode</CharacterSet>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
     <ConfigurationType>DynamicLibrary</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
     <PlatformToolset>v142</PlatformToolset>
-    <CharacterSet>MultiByte</CharacterSet>
+    <CharacterSet>Unicode</CharacterSet>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
     <ConfigurationType>DynamicLibrary</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
     <PlatformToolset>v142</PlatformToolset>
     <WholeProgramOptimization>true</WholeProgramOptimization>
-    <CharacterSet>MultiByte</CharacterSet>
+    <CharacterSet>Unicode</CharacterSet>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
@@ -132,6 +132,7 @@
       <PreprocessorDefinitions>WIN32;NDEBUG;NVFBCWRP_EXPORTS;_WINDOWS;_USRDLL;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <PrecompiledHeaderFile>pch.h</PrecompiledHeaderFile>
+      <BufferSecurityCheck>true</BufferSecurityCheck>
     </ClCompile>
     <Link>
       <SubSystem>Windows</SubSystem>
@@ -154,6 +155,7 @@
       <PreprocessorDefinitions>NDEBUG;NVFBCWRP_EXPORTS;_WINDOWS;_USRDLL;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <PrecompiledHeaderFile>pch.h</PrecompiledHeaderFile>
+      <BufferSecurityCheck>true</BufferSecurityCheck>
     </ClCompile>
     <Link>
       <SubSystem>Windows</SubSystem>

win/nvfbcwrp/nvfbcwrp.vcxproj.filters

@@ -15,15 +15,15 @@
     </Filter>
   </ItemGroup>
   <ItemGroup>
-    <ClInclude Include="framework.h">
-      <Filter>Файлы заголовков</Filter>
-    </ClInclude>
     <ClInclude Include="pch.h">
       <Filter>Файлы заголовков</Filter>
     </ClInclude>
     <ClInclude Include="nvfbcdefs.h">
       <Filter>Файлы заголовков</Filter>
     </ClInclude>
+    <ClInclude Include="framework.h">
+      <Filter>Файлы заголовков</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="pch.cpp">

win/nvfbcwrp/nvfbcwrp_main.cpp

@@ -1,29 +1,117 @@
 #include "pch.h"
 #include "nvfbcdefs.h"
-#include <windows.h>
 
 #ifdef _WIN64
-#define LIBNAME ".\\NvFBC64_.dll"
+#define LIBNAME TEXT(".\\NvFBC64_.dll")
 #else
-#define LIBNAME ".\\NvFBC_.dll"
+#define LIBNAME TEXT(".\\NvFBC_.dll")
 #endif
 
-HINSTANCE hLThis = 0;
+#define ENV_NVFBC_DUMPDIR TEXT("NVFBCWRP_DUMP_DIR")
+#define ENV_NVFBC_PRIVDATA_FILE TEXT("NVFBCWRP_PRIVDATA_FILE")
+#define NVFBC_PRIVDATA_DUMP_PREFIX TEXT("pd_")
+
 extern "C" {
 	FARPROC ORIG_NvFBC_Create, ORIG_NvFBC_Enable, ORIG_NvFBC_GetSDKVersion,
 		ORIG_NvFBC_GetStatus, ORIG_NvFBC_GetStatusEx, ORIG_NvFBC_SetGlobalFlags,
 		ORIG_NvOptimusEnablement;
 }
 NvFBC_CreateFunctionExType ORIG_NvFBC_CreateEx;
-HINSTANCE hL = 0;
+
+// Default magic code which is passed as pPrivateData
+// and enables NvFBC to work on GeForce
+DWORD default_magic[] = { 0xAEF57AC5, 0x401D1A39, 0x1B856BBE, 0x9ED0CEBA };
+void* magic = default_magic;
+NvU32 magic_size = sizeof(default_magic);
+
+TCHAR* dumpPath = NULL;
+
+TCHAR* getEnvVar(const TCHAR* name, DWORD size = MAX_PATH) {
+	TCHAR* buf = NULL;
+	try {
+		buf = new TCHAR[size];
+	}
+	catch (std::bad_alloc&) {
+		return NULL;
+	}
+	DWORD ret = GetEnvironmentVariable(name, buf, size);
+	if (ret != 0 && size > ret) {
+		return buf;
+	}
+	else {
+		delete buf;
+		return NULL;
+	}
+}
+
+std::tuple<NvU32, void*> tryGetMagic() {
+	TCHAR* filename = getEnvVar(ENV_NVFBC_PRIVDATA_FILE);
+	if (!filename) {
+		return { 0, NULL };
+	}
+	
+	HANDLE hFile = INVALID_HANDLE_VALUE;
+	hFile = CreateFile(filename,
+		GENERIC_READ,
+		FILE_SHARE_READ,
+		NULL,
+		OPEN_EXISTING,
+		FILE_ATTRIBUTE_NORMAL,
+		NULL);
+	delete filename;
+	if (hFile == INVALID_HANDLE_VALUE) {
+		return { 0, NULL };
+	}
+	
+	LARGE_INTEGER size;
+	LARGE_INTEGER zero;
+	zero.QuadPart = 0;
+	if (!SetFilePointerEx(hFile, zero, &size, FILE_END)) {
+		CloseHandle(hFile);
+		return { 0, NULL };
+	}
+	if (size.HighPart) {
+		CloseHandle(hFile);
+		return { 0, NULL };
+	}
+	if (!SetFilePointerEx(hFile, zero, NULL, FILE_BEGIN)) {
+		CloseHandle(hFile);
+		return { 0, NULL };
+	}
+
+	char* buf = NULL;
+	try {
+		buf = new char[size.LowPart];
+	}
+	catch (std::bad_alloc&)
+	{
+		CloseHandle(hFile);
+		return { 0, NULL };
+	}
+	DWORD bytes_read = 0;
+	if (ReadFile(hFile, buf, size.LowPart, &bytes_read, NULL) &&
+	  bytes_read == size.LowPart) {
+		CloseHandle(hFile);
+		return { size.LowPart, buf };
+	}
+	else {
+		CloseHandle(hFile);
+		delete buf;
+		return { 0, NULL };
+	}
+}
+
 
 BOOL WINAPI DllMain(HINSTANCE hInst,DWORD reason,LPVOID)
 {
+	HINSTANCE hL = 0;
 	if (reason == DLL_PROCESS_ATTACH)
 	{
-		//hLThis = hInst;
 		hL = LoadLibrary(LIBNAME);
 		if (!hL) return false;
+
+		// DllMain calls are serialized by system on process level, so we are clear
+		// to set required variables.
 		ORIG_NvFBC_Create = GetProcAddress(hL, "NvFBC_Create");
 		if (!ORIG_NvFBC_Create) return false;
 		ORIG_NvFBC_CreateEx = (NvFBC_CreateFunctionExType)::GetProcAddress(hL, "NvFBC_CreateEx");
@@ -40,24 +128,69 @@ BOOL WINAPI DllMain(HINSTANCE hInst,DWORD reason,LPVOID)
 		if (!ORIG_NvFBC_SetGlobalFlags) return false;
 		ORIG_NvOptimusEnablement = GetProcAddress(hL, "NvOptimusEnablement");
 		if (!ORIG_NvOptimusEnablement) return false;
+
+		// Check dump settings
+		if (TCHAR* dumpDirName = getEnvVar(ENV_NVFBC_DUMPDIR))
+		{
+			dumpPath = dumpDirName;
 		}
-	if (reason == DLL_PROCESS_DETACH)
+
+		// Check external magic file
+		NvU32 new_magic_size;
+		void* new_magic;
+		std::tie(new_magic_size, new_magic) = tryGetMagic();
+		if (new_magic) {
+			magic = new_magic;
+			magic_size = new_magic_size;
+		}
+	}
+	if (reason == DLL_PROCESS_DETACH && hL)
 	{
 		FreeLibrary(hL);
+		if (dumpPath) {
+			delete dumpPath;
+			dumpPath = NULL;
+		}
+		if (magic != default_magic) {
+			delete magic;
+			magic = NULL;
+		}
 		return true;
 	}
 
 	return true;
 }
 
+void tryDumpPrivateData(NvU32 size, void* data)
+{
+	TCHAR szTmpFilename[MAX_PATH];
+	if (dumpPath && GetTempFileName(dumpPath, NVFBC_PRIVDATA_DUMP_PREFIX, 0, szTmpFilename) != 0) {
+		// Privdata spying enabled and tempfile created.
+		// Attempt to dump private data.
+		HANDLE hTempFile = INVALID_HANDLE_VALUE;
+		hTempFile = CreateFile((LPTSTR)szTmpFilename,
+			GENERIC_WRITE,
+			0,
+			NULL,
+			CREATE_ALWAYS,
+			FILE_ATTRIBUTE_NORMAL,
+			NULL);
+		if (hTempFile != INVALID_HANDLE_VALUE) {
+			// File is open, dumping data.
+			WriteFile(hTempFile, data, size, NULL, NULL);
+			CloseHandle(hTempFile);
+		}
+	}
+}
+
 NVFBCRESULT NVFBCAPI PROXY_NvFBC_CreateEx(NvFBCCreateParams* params) {
 	if (params->dwPrivateDataSize == 0 && params->pPrivateData == NULL) {
 		//Backup old values
 		void* bkp_privdata = params->pPrivateData;
 		NvU32 bkp_privdatasize = params->dwPrivateDataSize;
 		// Inject private keys into structure
-		params->dwPrivateDataSize = sizeof(magic);
+		params->dwPrivateDataSize = magic_size;
-		params->pPrivateData = &magic;
+		params->pPrivateData = magic;
 		// Invoke original function
 		NVFBCRESULT res = ORIG_NvFBC_CreateEx(params);
 		// Rollback private data changes in params structure
@@ -66,6 +199,9 @@ NVFBCRESULT NVFBCAPI PROXY_NvFBC_CreateEx(NvFBCCreateParams* params) {
 		return res;
 	}
 	else {
+		if (params->dwPrivateDataSize > 0 && params->pPrivateData != NULL) {
+			tryDumpPrivateData(params->dwPrivateDataSize, params->pPrivateData);
+		}
 		return ORIG_NvFBC_CreateEx((void*)params);
 	}
 }