Add support for packed clients (#802)

* First iteration of pack reader and interface

* Fix memory leak and remove logs

* Complete packed asset interface and begin on file loading replacement

* Implement proper BinaryIO error

* Improve AssetMemoryBuffer for reading and implement more reading

* Repair more file loading code and improve how navmeshes are loaded

* Missing checks implementation

* Revert addition of Manifest class and migration changes

* Resolved all feedback.

dCommon/dClient/AssetManager.cpp

@@ -0,0 +1,201 @@
+#include "AssetManager.h"
+
+#include <zlib.h>
+
+AssetManager::AssetManager(const std::string& path) {
+	if (!std::filesystem::is_directory(path)) {
+		throw std::runtime_error("Attempted to load asset bundle (" + path + ") however it is not a valid directory.");
+	}
+
+	m_Path = std::filesystem::path(path);
+
+	if (std::filesystem::exists(m_Path / "client") && std::filesystem::exists(m_Path / "versions")) {
+		m_AssetBundleType = eAssetBundleType::Packed;
+
+		m_RootPath = m_Path;
+		m_ResPath = (m_Path / "client" / "res");
+	} else if (std::filesystem::exists(m_Path / ".." / "versions") && std::filesystem::exists(m_Path / "res")) {
+		m_AssetBundleType = eAssetBundleType::Packed;
+
+		m_RootPath = (m_Path / "..");
+		m_ResPath = (m_Path / "res");
+	} else if (std::filesystem::exists(m_Path / "pack") && std::filesystem::exists(m_Path / ".." / ".." / "versions")) {
+		m_AssetBundleType = eAssetBundleType::Packed;
+
+		m_RootPath = (m_Path / ".." / "..");
+		m_ResPath = m_Path;
+	} else if (std::filesystem::exists(m_Path / "res" / "cdclient.fdb") && !std::filesystem::exists(m_Path / "res" / "pack")) {
+		m_AssetBundleType = eAssetBundleType::Unpacked;
+
+		m_ResPath = (m_Path / "res");
+	} else if (std::filesystem::exists(m_Path / "cdclient.fdb") && !std::filesystem::exists(m_Path / "pack")) {
+		m_AssetBundleType = eAssetBundleType::Unpacked;
+
+		m_ResPath = m_Path;
+	}
+
+	if (m_AssetBundleType == eAssetBundleType::None) {
+		throw std::runtime_error("Failed to identify client type, cannot read client data.");
+	}
+
+	switch (m_AssetBundleType) {
+		case eAssetBundleType::Packed: {
+			this->LoadPackIndex();
+
+			this->UnpackRequiredAssets();
+
+			break;
+		}
+	}
+}
+
+void AssetManager::LoadPackIndex() {
+	m_PackIndex = new PackIndex(m_RootPath);
+}
+
+std::filesystem::path AssetManager::GetResPath() {
+	return m_ResPath;
+}
+
+eAssetBundleType AssetManager::GetAssetBundleType() {
+	return m_AssetBundleType;
+}
+
+bool AssetManager::HasFile(const char* name) {
+	auto fixedName = std::string(name);
+	std::transform(fixedName.begin(), fixedName.end(), fixedName.begin(), [](uint8_t c) { return std::tolower(c); });
+	std::replace(fixedName.begin(), fixedName.end(), '/', '\\');
+
+	auto realPathName = fixedName;
+
+	if (fixedName.rfind("client\\res\\", 0) != 0) {
+		fixedName = "client\\res\\" + fixedName;
+	}
+
+	if (std::filesystem::exists(m_ResPath / realPathName)) {
+		return true;
+	}
+
+	uint32_t crc = crc32b(0xFFFFFFFF, (uint8_t*)fixedName.c_str(), fixedName.size());
+	crc = crc32b(crc, (Bytef*)"\0\0\0\0", 4);
+
+	for (const auto& item : this->m_PackIndex->GetPackFileIndices()) {
+		if (item.m_Crc == crc) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
+bool AssetManager::GetFile(const char* name, char** data, uint32_t* len) {
+	auto fixedName = std::string(name);
+	std::transform(fixedName.begin(), fixedName.end(), fixedName.begin(), [](uint8_t c) { return std::tolower(c); });
+	std::replace(fixedName.begin(), fixedName.end(), '/', '\\');
+
+	auto realPathName = fixedName;
+
+	if (fixedName.rfind("client\\res\\", 0) != 0) {
+		fixedName = "client\\res\\" + fixedName;
+	}
+
+	if (std::filesystem::exists(m_ResPath / realPathName)) {
+		FILE* file;
+#ifdef _WIN32
+		fopen_s(&file, (m_ResPath / realPathName).string().c_str(), "rb");
+#elif __APPLE__
+		// macOS has 64bit file IO by default
+		file = fopen((m_ResPath / realPathName).string().c_str(), "rb");
+#else
+		file = fopen64((m_ResPath / realPathName).string().c_str(), "rb");
+#endif
+		fseek(file, 0, SEEK_END);
+		*len = ftell(file);
+		*data = (char*)malloc(*len);
+		fseek(file, 0, SEEK_SET);
+		fread(*data, sizeof(uint8_t), *len, file);
+		fclose(file);
+
+		return true;
+	}
+
+	if (this->m_AssetBundleType == eAssetBundleType::Unpacked) return false;
+
+	int32_t packIndex = -1;
+	uint32_t crc = crc32b(0xFFFFFFFF, (uint8_t*)fixedName.c_str(), fixedName.size());
+	crc = crc32b(crc, (Bytef*)"\0\0\0\0", 4);
+
+	for (const auto& item : this->m_PackIndex->GetPackFileIndices()) {
+		if (item.m_Crc == crc) {
+			packIndex = item.m_PackFileIndex;
+			crc = item.m_Crc;
+			break;
+		}
+	}
+
+	if (packIndex == -1 || !crc) {
+		return false;
+	}
+
+	auto packs = this->m_PackIndex->GetPacks();
+	auto* pack = packs.at(packIndex);
+
+	bool success = pack->ReadFileFromPack(crc, data, len);
+
+	return success;
+}
+
+AssetMemoryBuffer AssetManager::GetFileAsBuffer(const char* name) {
+	char* buf;
+	uint32_t len;
+
+	bool success = this->GetFile(name, &buf, &len);
+
+	return AssetMemoryBuffer(buf, len, success);
+}
+
+void AssetManager::UnpackRequiredAssets() {
+	if (std::filesystem::exists(m_ResPath / "cdclient.fdb")) return;
+
+	char* data;
+	uint32_t size;
+
+	bool success = this->GetFile("cdclient.fdb", &data, &size);
+
+	if (!success) {
+		Game::logger->Log("AssetManager", "Failed to extract required files from the packs.");
+
+		delete data;
+
+		return;
+	}
+
+	std::ofstream cdclientOutput(m_ResPath / "cdclient.fdb", std::ios::out | std::ios::binary);
+	cdclientOutput.write(data, size);
+	cdclientOutput.close();
+
+	delete data;
+
+	return;
+}
+
+uint32_t AssetManager::crc32b(uint32_t base, uint8_t* message, size_t l) {
+	size_t i, j;
+	uint32_t crc, msb;
+
+	crc = base;
+	for (i = 0; i < l; i++) {
+		// xor next byte to upper bits of crc
+		crc ^= (((unsigned int)message[i]) << 24);
+		for (j = 0; j < 8; j++) { // Do eight times.
+			msb = crc >> 31;
+			crc <<= 1;
+			crc ^= (0 - msb) & 0x04C11DB7;
+		}
+	}
+	return crc; // don't complement crc on output
+}
+
+AssetManager::~AssetManager() {
+	delete m_PackIndex;
+}

dCommon/dClient/AssetManager.h

@@ -0,0 +1,78 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <unordered_map>
+#include <filesystem>
+
+#include "Pack.h"
+#include "PackIndex.h"
+
+enum class eAssetBundleType {
+	None,
+	Unpacked,
+	Packed
+};
+
+struct AssetMemoryBuffer : std::streambuf {
+	char* m_Base;
+	bool m_Success;
+
+	AssetMemoryBuffer(char* base, std::ptrdiff_t n, bool success) {
+		m_Base = base;
+		m_Success = success;
+		if (!m_Success) return;
+		this->setg(base, base, base + n);
+	}
+
+	pos_type seekpos(pos_type sp, std::ios_base::openmode which) override {
+		return seekoff(sp - pos_type(off_type(0)), std::ios_base::beg, which);
+	}
+
+	pos_type seekoff(off_type off,
+		std::ios_base::seekdir dir,
+		std::ios_base::openmode which = std::ios_base::in) override {
+		if (dir == std::ios_base::cur)
+			gbump(off);
+		else if (dir == std::ios_base::end)
+			setg(eback(), egptr() + off, egptr());
+		else if (dir == std::ios_base::beg)
+			setg(eback(), eback() + off, egptr());
+		return gptr() - eback();
+	}
+
+	void close() {
+		delete m_Base;
+	}
+};
+
+class AssetManager {
+public:
+	AssetManager(const std::string& path);
+	~AssetManager();
+
+	std::filesystem::path GetResPath();
+	eAssetBundleType GetAssetBundleType();
+
+	bool HasFile(const char* name);
+	bool GetFile(const char* name, char** data, uint32_t* len);
+	AssetMemoryBuffer GetFileAsBuffer(const char* name);
+
+private:
+	void LoadPackIndex();
+	void UnpackRequiredAssets();
+
+	// Modified crc algorithm (mpeg2)
+	// Reference: https://stackoverflow.com/questions/54339800/how-to-modify-crc-32-to-crc-32-mpeg-2
+	inline uint32_t crc32b(uint32_t base, uint8_t* message, size_t l);
+
+	bool m_SuccessfullyLoaded;
+
+	std::filesystem::path m_Path;
+	std::filesystem::path m_RootPath;
+	std::filesystem::path m_ResPath;
+
+	eAssetBundleType m_AssetBundleType = eAssetBundleType::None;
+
+	PackIndex* m_PackIndex;
+};

dCommon/dClient/CMakeLists.txt

@@ -0,0 +1,6 @@
+set(DCOMMON_DCLIENT_SOURCES
+	"PackIndex.cpp"
+	"Pack.cpp"
+	"AssetManager.cpp"
+	PARENT_SCOPE
+)

dCommon/dClient/Pack.cpp

@@ -0,0 +1,118 @@
+#include "Pack.h"
+
+#include "ZCompression.h"
+
+Pack::Pack(const std::filesystem::path& filePath) {
+	m_FilePath = filePath;
+
+	if (!std::filesystem::exists(filePath)) {
+		return;
+	}
+
+	m_FileStream = std::ifstream(filePath, std::ios::in | std::ios::binary);
+
+	m_FileStream.read(m_Version, 7); 
+
+	m_FileStream.seekg(-8, std::ios::end); // move file pointer to 8 bytes before the end (location of the address of the record count)
+
+	uint32_t recordCountPos = 0;
+	BinaryIO::BinaryRead<uint32_t>(m_FileStream, recordCountPos);
+
+	m_FileStream.seekg(recordCountPos, std::ios::beg);
+
+	BinaryIO::BinaryRead<uint32_t>(m_FileStream, m_RecordCount);
+
+	for (int i = 0; i < m_RecordCount; i++) {
+		PackRecord record;
+		BinaryIO::BinaryRead<PackRecord>(m_FileStream, record);
+
+		m_Records.push_back(record);
+	}
+
+	m_FileStream.close();
+}
+
+bool Pack::HasFile(uint32_t crc) {
+	for (const auto& record : m_Records) {
+		if (record.m_Crc == crc) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
+bool Pack::ReadFileFromPack(uint32_t crc, char** data, uint32_t* len) {
+	// Time for some wacky C file reading for speed reasons
+
+	PackRecord pkRecord{};
+
+	for (const auto& record : m_Records) {
+		if (record.m_Crc == crc) {
+			pkRecord = record;
+			break;
+		}
+	}
+
+	if (pkRecord.m_Crc == 0) return false;
+
+	size_t pos = 0;
+	pos += pkRecord.m_FilePointer;
+
+	bool isCompressed = (pkRecord.m_IsCompressed & 0xff) > 0;
+	auto inPackSize = isCompressed ? pkRecord.m_CompressedSize : pkRecord.m_UncompressedSize;
+
+	FILE* file;
+#ifdef _WIN32
+	fopen_s(&file, m_FilePath.string().c_str(), "rb");
+#elif __APPLE__
+	// macOS has 64bit file IO by default
+	file = fopen(m_FilePath.string().c_str(), "rb");
+#else
+	file = fopen64(m_FilePath.string().c_str(), "rb");
+#endif
+
+	fseek(file, pos, SEEK_SET);
+
+	if (!isCompressed) {
+		char* tempData = (char*)malloc(pkRecord.m_UncompressedSize);
+		fread(tempData, sizeof(uint8_t), pkRecord.m_UncompressedSize, file);
+
+		*data = tempData;
+		*len = pkRecord.m_UncompressedSize;
+		fclose(file);
+
+		return true;
+	}
+
+	pos += 5; // skip header
+
+	fseek(file, pos, SEEK_SET);
+
+	char* decompressedData = (char*)malloc(pkRecord.m_UncompressedSize);
+	uint32_t currentReadPos = 0;
+
+	while (true) {
+		if (currentReadPos >= pkRecord.m_UncompressedSize) break;
+
+		uint32_t size;
+		fread(&size, sizeof(uint32_t), 1, file);
+		pos += 4; // Move pointer position 4 to the right
+
+		char* chunk = (char*)malloc(size);
+		fread(chunk, sizeof(int8_t), size, file);
+		pos += size; // Move pointer position the amount of bytes read to the right
+
+		int32_t err;
+		currentReadPos += ZCompression::Decompress((uint8_t*)chunk, size, reinterpret_cast<uint8_t*>(decompressedData + currentReadPos), ZCompression::MAX_SD0_CHUNK_SIZE, err);
+
+		free(chunk);
+	}
+
+	*data = decompressedData;
+	*len = pkRecord.m_UncompressedSize;
+
+	fclose(file);
+
+	return true;
+}

dCommon/dClient/Pack.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include <vector>
+#include <string>
+#include <filesystem>
+
+#pragma pack(push, 1)
+struct PackRecord {
+	uint32_t m_Crc;
+	int32_t m_LowerCrc;
+	int32_t m_UpperCrc;
+	uint32_t m_UncompressedSize;
+	char m_UncompressedHash[32];
+	uint32_t m_Padding1;
+	uint32_t m_CompressedSize;
+	char m_CompressedHash[32];
+	uint32_t m_Padding2;
+	uint32_t m_FilePointer;
+	uint32_t m_IsCompressed; // u32 bool
+};
+#pragma pack(pop)
+
+class Pack {
+public:
+	Pack(const std::filesystem::path& filePath);
+	~Pack() = default;
+
+	bool HasFile(uint32_t crc);
+	bool ReadFileFromPack(uint32_t crc, char** data, uint32_t* len);
+private:
+	std::ifstream m_FileStream;
+	std::filesystem::path m_FilePath;
+
+	char m_Version[7];
+
+	uint32_t m_RecordCount;
+	std::vector<PackRecord> m_Records;
+};

dCommon/dClient/PackIndex.cpp

@@ -0,0 +1,50 @@
+#include "PackIndex.h"
+
+
+PackIndex::PackIndex(const std::filesystem::path& filePath) {
+	m_FileStream = std::ifstream(filePath / "versions" / "primary.pki", std::ios::in | std::ios::binary);
+
+	BinaryIO::BinaryRead<uint32_t>(m_FileStream, m_Version);
+	BinaryIO::BinaryRead<uint32_t>(m_FileStream, m_PackPathCount);
+
+	for (int i = 0; i < m_PackPathCount; i++) {
+		uint32_t stringLen = 0;
+		BinaryIO::BinaryRead<uint32_t>(m_FileStream, stringLen);
+
+		std::string path;
+
+		for (int j = 0; j < stringLen; j++) {
+			char inChar;
+			BinaryIO::BinaryRead<char>(m_FileStream, inChar);
+
+			path += inChar;
+		}
+
+		m_PackPaths.push_back(path);
+	}
+
+	BinaryIO::BinaryRead<uint32_t>(m_FileStream, m_PackFileIndexCount);
+
+	for (int i = 0; i < m_PackFileIndexCount; i++) {
+		PackFileIndex packFileIndex;
+		BinaryIO::BinaryRead<PackFileIndex>(m_FileStream, packFileIndex);
+
+		m_PackFileIndices.push_back(packFileIndex);
+	}
+
+	Game::logger->Log("PackIndex", "Loaded pack catalog with %i pack files and %i files", m_PackPaths.size(), m_PackFileIndices.size());
+
+	for (const auto& item : m_PackPaths) {
+		auto* pack = new Pack(filePath / item);
+
+		m_Packs.push_back(pack);
+	}
+
+	m_FileStream.close();
+}
+
+PackIndex::~PackIndex() {
+	for (const auto* item : m_Packs) {
+		delete item;
+	}
+}

dCommon/dClient/PackIndex.h

@@ -0,0 +1,40 @@
+#pragma once
+
+#include <cstdint>
+
+#include <string>
+#include <vector>
+#include <filesystem>
+
+#include "Pack.h"
+
+#pragma pack(push, 1)
+struct PackFileIndex {
+	uint32_t m_Crc;
+	int32_t m_LowerCrc;
+	int32_t m_UpperCrc;
+	uint32_t m_PackFileIndex;
+	uint32_t m_IsCompressed; // u32 bool?
+};
+#pragma pack(pop)
+
+class PackIndex {
+public:
+	PackIndex(const std::filesystem::path& filePath);
+	~PackIndex();
+
+	const std::vector<std::string>& GetPackPaths() { return m_PackPaths; }
+	const std::vector<PackFileIndex>& GetPackFileIndices() { return m_PackFileIndices; }
+	const std::vector<Pack*>& GetPacks() { return m_Packs; }
+private:
+	std::ifstream m_FileStream;
+
+	uint32_t m_Version;
+
+	uint32_t m_PackPathCount;
+	std::vector<std::string> m_PackPaths;
+	uint32_t m_PackFileIndexCount;
+	std::vector<PackFileIndex> m_PackFileIndices;
+
+	std::vector<Pack*> m_Packs;
+};