Add support for packed clients (#802)

* First iteration of pack reader and interface

* Fix memory leak and remove logs

* Complete packed asset interface and begin on file loading replacement

* Implement proper BinaryIO error

* Improve AssetMemoryBuffer for reading and implement more reading

* Repair more file loading code and improve how navmeshes are loaded

* Missing checks implementation

* Revert addition of Manifest class and migration changes

* Resolved all feedback.

CMakeLists.txt

@@ -101,6 +101,17 @@ foreach(resource_file ${RESOURCE_FILES})
 	endif()
 endforeach()
 
+# Copy navmesh data on first build and extract it
+if (NOT EXISTS ${PROJECT_BINARY_DIR}/navmeshes/)
+	configure_file(
+		${CMAKE_SOURCE_DIR}/resources/navmeshes.zip ${PROJECT_BINARY_DIR}/navmeshes.zip
+		COPYONLY
+	)
+
+	file(ARCHIVE_EXTRACT INPUT ${PROJECT_BINARY_DIR}/navmeshes.zip)
+	file(REMOVE ${PROJECT_BINARY_DIR}/navmeshes.zip)
+endif()
+
 # Copy vanity files on first build
 set(VANITY_FILES "CREDITS.md" "INFO.md" "TESTAMENT.md" "NPC.xml")
 foreach(file ${VANITY_FILES})
@@ -145,6 +156,7 @@ set(INCLUDED_DIRECTORIES
 	"dGame/dEntity"
 	"dGame/dPropertyBehaviors"
 	"dGame/dUtilities"
+	"dCommon/dClient"
 	"dPhysics"
 	"dNavigation"
 	"dNavigation/dTerrain"

dChatServer/ChatServer.cpp

@@ -12,6 +12,7 @@
 #include "dMessageIdentifiers.h"
 #include "dChatFilter.h"
 #include "Diagnostics.h"
+#include "AssetManager.h"
 
 #include "PlayerContainer.h"
 #include "ChatPacketHandler.h"
@@ -22,6 +23,7 @@ namespace Game {
 	dServer* server;
 	dConfig* config;
 	dChatFilter* chatFilter;
+	AssetManager* assetManager;
 }
 
 //RakNet includes:
@@ -50,6 +52,16 @@ int main(int argc, char** argv) {
 	Game::logger->SetLogToConsole(bool(std::stoi(config.GetValue("log_to_console"))));
 	Game::logger->SetLogDebugStatements(config.GetValue("log_debug_statements") == "1");
 
+	try {
+		std::string client_path = config.GetValue("client_location");
+		if (client_path.empty()) client_path = "./res";
+		Game::assetManager = new AssetManager(config.GetValue("client_location"));
+	} catch (std::runtime_error& ex) {
+		Game::logger->Log("ChatServer", "Got an error while setting up assets: %s", ex.what());
+
+		return EXIT_FAILURE;
+	}
+
 	//Connect to the MySQL Database
 	std::string mysql_host = config.GetValue("mysql_host");
 	std::string mysql_database = config.GetValue("mysql_database");
@@ -87,7 +99,7 @@ int main(int argc, char** argv) {
 
 	Game::server = new dServer(config.GetValue("external_ip"), ourPort, 0, maxClients, false, true, Game::logger, masterIP, masterPort, ServerType::Chat);
 
-	Game::chatFilter = new dChatFilter("./res/chatplus_en_us", bool(std::stoi(config.GetValue("dont_generate_dcf"))));
+	Game::chatFilter = new dChatFilter(Game::assetManager->GetResPath().string() + "/chatplus_en_us", bool(std::stoi(config.GetValue("dont_generate_dcf"))));
 
 	//Run it until server gets a kill message from Master:
 	auto t = std::chrono::high_resolution_clock::now();

dCommon/BinaryIO.cpp

@@ -10,7 +10,7 @@ void BinaryIO::WriteString(const std::string& stringToWrite, std::ofstream& outs
 }
 
 //For reading null-terminated strings
-std::string BinaryIO::ReadString(std::ifstream& instream) {
+std::string BinaryIO::ReadString(std::istream& instream) {
 	std::string toReturn;
 	char buffer;
 
@@ -25,7 +25,7 @@ std::string BinaryIO::ReadString(std::ifstream& instream) {
 }
 
 //For reading strings of a specific size
-std::string BinaryIO::ReadString(std::ifstream& instream, size_t size) {
+std::string BinaryIO::ReadString(std::istream& instream, size_t size) {
 	std::string toReturn;
 	char buffer;
 
@@ -37,7 +37,7 @@ std::string BinaryIO::ReadString(std::ifstream& instream, size_t size) {
 	return toReturn;
 }
 
-std::string BinaryIO::ReadWString(std::ifstream& instream) {
+std::string BinaryIO::ReadWString(std::istream& instream) {
 	size_t size;
 	BinaryRead(instream, size);
 	//toReturn.resize(size);

dCommon/BinaryIO.h

@@ -10,16 +10,15 @@ namespace BinaryIO {
 
 	template<typename T>
 	std::istream& BinaryRead(std::istream& stream, T& value) {
-		if (!stream.good())
+		if (!stream.good()) throw std::runtime_error("Failed to read from istream.");
-			printf("bla");
 
 		return stream.read(reinterpret_cast<char*>(&value), sizeof(T));
 	}
 
 	void WriteString(const std::string& stringToWrite, std::ofstream& outstream);
-	std::string ReadString(std::ifstream& instream);
+	std::string ReadString(std::istream& instream);
-	std::string ReadString(std::ifstream& instream, size_t size);
+	std::string ReadString(std::istream& instream, size_t size);
-	std::string ReadWString(std::ifstream& instream);
+	std::string ReadWString(std::istream& instream);
 
 	inline bool DoesFileExist(const std::string& name) {
 		std::ifstream f(name.c_str());

dCommon/BrickByBrickFix.cpp

@@ -49,10 +49,10 @@ uint32_t BrickByBrickFix::TruncateBrokenBrickByBrickXml() {
 				}
 
 				// Ignore the valgrind warning about uninitialized values.  These are discarded later when we know the actual uncompressed size.
-				std::unique_ptr<uint8_t[]> uncompressedChunk(new uint8_t[MAX_SD0_CHUNK_SIZE]);
+				std::unique_ptr<uint8_t[]> uncompressedChunk(new uint8_t[ZCompression::MAX_SD0_CHUNK_SIZE]);
 				int32_t err{};
 				int32_t actualUncompressedSize = ZCompression::Decompress(
-					compressedChunk.get(), chunkSize, uncompressedChunk.get(), MAX_SD0_CHUNK_SIZE, err);
+					compressedChunk.get(), chunkSize, uncompressedChunk.get(), ZCompression::MAX_SD0_CHUNK_SIZE, err);
 
 				if (actualUncompressedSize != -1) {
 					uint32_t previousSize = completeUncompressedModel.size();

dCommon/BrickByBrickFix.h

@@ -17,10 +17,4 @@ namespace BrickByBrickFix {
 	 * @return The number of BrickByBrick models that were updated
 	 */
 	uint32_t UpdateBrickByBrickModelsToSd0();
-
-	/**
-	 * @brief Max size of an inflated sd0 zlib chunk
-	 * 
-	 */
-	constexpr uint32_t MAX_SD0_CHUNK_SIZE = 1024 * 256;
 };

dCommon/CMakeLists.txt

@@ -17,6 +17,12 @@ set(DCOMMON_SOURCES "AMFFormat.cpp"
 		"BrickByBrickFix.cpp"
 )
 
+add_subdirectory(dClient)
+
+foreach(file ${DCOMMON_DCLIENT_SOURCES})
+	set(DCOMMON_SOURCES ${DCOMMON_SOURCES} "dClient/${file}")
+endforeach()
+
 include_directories(${PROJECT_SOURCE_DIR}/dCommon/)
 
 add_library(dCommon STATIC ${DCOMMON_SOURCES})

dCommon/Game.h

@@ -10,6 +10,7 @@ class dChatFilter;
 class dConfig;
 class dLocale;
 class RakPeerInterface;
+class AssetManager;
 struct SystemAddress;
 
 namespace Game {
@@ -22,5 +23,6 @@ namespace Game {
 	extern dLocale* locale;
 	extern std::mt19937 randomEngine;
 	extern RakPeerInterface* chatServer;
+	extern AssetManager* assetManager;
 	extern SystemAddress chatSysAddr;
 }

dCommon/GeneralUtils.cpp

@@ -50,6 +50,7 @@ bool _IsSuffixChar(uint8_t c) {
 
 bool GeneralUtils::_NextUTF8Char(std::string_view& slice, uint32_t& out) {
 	size_t rem = slice.length();
+	if (slice.empty()) return false;
 	const uint8_t* bytes = (const uint8_t*)&slice.front();
 	if (rem > 0) {
 		uint8_t first = bytes[0];

dCommon/ZCompression.h

@@ -8,5 +8,11 @@ namespace ZCompression {
 	int32_t Compress(const uint8_t* abSrc, int32_t nLenSrc, uint8_t* abDst, int32_t nLenDst);
 
 	int32_t Decompress(const uint8_t* abSrc, int32_t nLenSrc, uint8_t* abDst, int32_t nLenDst, int32_t& nErr);
+
+	/**
+	 * @brief Max size of an inflated sd0 zlib chunk
+	 *
+	 */
+	constexpr uint32_t MAX_SD0_CHUNK_SIZE = 1024 * 256;
 }
 

dCommon/dClient/AssetManager.cpp

@@ -0,0 +1,201 @@
+#include "AssetManager.h"
+
+#include <zlib.h>
+
+AssetManager::AssetManager(const std::string& path) {
+	if (!std::filesystem::is_directory(path)) {
+		throw std::runtime_error("Attempted to load asset bundle (" + path + ") however it is not a valid directory.");
+	}
+
+	m_Path = std::filesystem::path(path);
+
+	if (std::filesystem::exists(m_Path / "client") && std::filesystem::exists(m_Path / "versions")) {
+		m_AssetBundleType = eAssetBundleType::Packed;
+
+		m_RootPath = m_Path;
+		m_ResPath = (m_Path / "client" / "res");
+	} else if (std::filesystem::exists(m_Path / ".." / "versions") && std::filesystem::exists(m_Path / "res")) {
+		m_AssetBundleType = eAssetBundleType::Packed;
+
+		m_RootPath = (m_Path / "..");
+		m_ResPath = (m_Path / "res");
+	} else if (std::filesystem::exists(m_Path / "pack") && std::filesystem::exists(m_Path / ".." / ".." / "versions")) {
+		m_AssetBundleType = eAssetBundleType::Packed;
+
+		m_RootPath = (m_Path / ".." / "..");
+		m_ResPath = m_Path;
+	} else if (std::filesystem::exists(m_Path / "res" / "cdclient.fdb") && !std::filesystem::exists(m_Path / "res" / "pack")) {
+		m_AssetBundleType = eAssetBundleType::Unpacked;
+
+		m_ResPath = (m_Path / "res");
+	} else if (std::filesystem::exists(m_Path / "cdclient.fdb") && !std::filesystem::exists(m_Path / "pack")) {
+		m_AssetBundleType = eAssetBundleType::Unpacked;
+
+		m_ResPath = m_Path;
+	}
+
+	if (m_AssetBundleType == eAssetBundleType::None) {
+		throw std::runtime_error("Failed to identify client type, cannot read client data.");
+	}
+
+	switch (m_AssetBundleType) {
+		case eAssetBundleType::Packed: {
+			this->LoadPackIndex();
+
+			this->UnpackRequiredAssets();
+
+			break;
+		}
+	}
+}
+
+void AssetManager::LoadPackIndex() {
+	m_PackIndex = new PackIndex(m_RootPath);
+}
+
+std::filesystem::path AssetManager::GetResPath() {
+	return m_ResPath;
+}
+
+eAssetBundleType AssetManager::GetAssetBundleType() {
+	return m_AssetBundleType;
+}
+
+bool AssetManager::HasFile(const char* name) {
+	auto fixedName = std::string(name);
+	std::transform(fixedName.begin(), fixedName.end(), fixedName.begin(), [](uint8_t c) { return std::tolower(c); });
+	std::replace(fixedName.begin(), fixedName.end(), '/', '\\');
+
+	auto realPathName = fixedName;
+
+	if (fixedName.rfind("client\\res\\", 0) != 0) {
+		fixedName = "client\\res\\" + fixedName;
+	}
+
+	if (std::filesystem::exists(m_ResPath / realPathName)) {
+		return true;
+	}
+
+	uint32_t crc = crc32b(0xFFFFFFFF, (uint8_t*)fixedName.c_str(), fixedName.size());
+	crc = crc32b(crc, (Bytef*)"\0\0\0\0", 4);
+
+	for (const auto& item : this->m_PackIndex->GetPackFileIndices()) {
+		if (item.m_Crc == crc) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
+bool AssetManager::GetFile(const char* name, char** data, uint32_t* len) {
+	auto fixedName = std::string(name);
+	std::transform(fixedName.begin(), fixedName.end(), fixedName.begin(), [](uint8_t c) { return std::tolower(c); });
+	std::replace(fixedName.begin(), fixedName.end(), '/', '\\');
+
+	auto realPathName = fixedName;
+
+	if (fixedName.rfind("client\\res\\", 0) != 0) {
+		fixedName = "client\\res\\" + fixedName;
+	}
+
+	if (std::filesystem::exists(m_ResPath / realPathName)) {
+		FILE* file;
+#ifdef _WIN32
+		fopen_s(&file, (m_ResPath / realPathName).string().c_str(), "rb");
+#elif __APPLE__
+		// macOS has 64bit file IO by default
+		file = fopen((m_ResPath / realPathName).string().c_str(), "rb");
+#else
+		file = fopen64((m_ResPath / realPathName).string().c_str(), "rb");
+#endif
+		fseek(file, 0, SEEK_END);
+		*len = ftell(file);
+		*data = (char*)malloc(*len);
+		fseek(file, 0, SEEK_SET);
+		fread(*data, sizeof(uint8_t), *len, file);
+		fclose(file);
+
+		return true;
+	}
+
+	if (this->m_AssetBundleType == eAssetBundleType::Unpacked) return false;
+
+	int32_t packIndex = -1;
+	uint32_t crc = crc32b(0xFFFFFFFF, (uint8_t*)fixedName.c_str(), fixedName.size());
+	crc = crc32b(crc, (Bytef*)"\0\0\0\0", 4);
+
+	for (const auto& item : this->m_PackIndex->GetPackFileIndices()) {
+		if (item.m_Crc == crc) {
+			packIndex = item.m_PackFileIndex;
+			crc = item.m_Crc;
+			break;
+		}
+	}
+
+	if (packIndex == -1 || !crc) {
+		return false;
+	}
+
+	auto packs = this->m_PackIndex->GetPacks();
+	auto* pack = packs.at(packIndex);
+
+	bool success = pack->ReadFileFromPack(crc, data, len);
+
+	return success;
+}
+
+AssetMemoryBuffer AssetManager::GetFileAsBuffer(const char* name) {
+	char* buf;
+	uint32_t len;
+
+	bool success = this->GetFile(name, &buf, &len);
+
+	return AssetMemoryBuffer(buf, len, success);
+}
+
+void AssetManager::UnpackRequiredAssets() {
+	if (std::filesystem::exists(m_ResPath / "cdclient.fdb")) return;
+
+	char* data;
+	uint32_t size;
+
+	bool success = this->GetFile("cdclient.fdb", &data, &size);
+
+	if (!success) {
+		Game::logger->Log("AssetManager", "Failed to extract required files from the packs.");
+
+		delete data;
+
+		return;
+	}
+
+	std::ofstream cdclientOutput(m_ResPath / "cdclient.fdb", std::ios::out | std::ios::binary);
+	cdclientOutput.write(data, size);
+	cdclientOutput.close();
+
+	delete data;
+
+	return;
+}
+
+uint32_t AssetManager::crc32b(uint32_t base, uint8_t* message, size_t l) {
+	size_t i, j;
+	uint32_t crc, msb;
+
+	crc = base;
+	for (i = 0; i < l; i++) {
+		// xor next byte to upper bits of crc
+		crc ^= (((unsigned int)message[i]) << 24);
+		for (j = 0; j < 8; j++) { // Do eight times.
+			msb = crc >> 31;
+			crc <<= 1;
+			crc ^= (0 - msb) & 0x04C11DB7;
+		}
+	}
+	return crc; // don't complement crc on output
+}
+
+AssetManager::~AssetManager() {
+	delete m_PackIndex;
+}

dCommon/dClient/AssetManager.h

@@ -0,0 +1,78 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <unordered_map>
+#include <filesystem>
+
+#include "Pack.h"
+#include "PackIndex.h"
+
+enum class eAssetBundleType {
+	None,
+	Unpacked,
+	Packed
+};
+
+struct AssetMemoryBuffer : std::streambuf {
+	char* m_Base;
+	bool m_Success;
+
+	AssetMemoryBuffer(char* base, std::ptrdiff_t n, bool success) {
+		m_Base = base;
+		m_Success = success;
+		if (!m_Success) return;
+		this->setg(base, base, base + n);
+	}
+
+	pos_type seekpos(pos_type sp, std::ios_base::openmode which) override {
+		return seekoff(sp - pos_type(off_type(0)), std::ios_base::beg, which);
+	}
+
+	pos_type seekoff(off_type off,
+		std::ios_base::seekdir dir,
+		std::ios_base::openmode which = std::ios_base::in) override {
+		if (dir == std::ios_base::cur)
+			gbump(off);
+		else if (dir == std::ios_base::end)
+			setg(eback(), egptr() + off, egptr());
+		else if (dir == std::ios_base::beg)
+			setg(eback(), eback() + off, egptr());
+		return gptr() - eback();
+	}
+
+	void close() {
+		delete m_Base;
+	}
+};
+
+class AssetManager {
+public:
+	AssetManager(const std::string& path);
+	~AssetManager();
+
+	std::filesystem::path GetResPath();
+	eAssetBundleType GetAssetBundleType();
+
+	bool HasFile(const char* name);
+	bool GetFile(const char* name, char** data, uint32_t* len);
+	AssetMemoryBuffer GetFileAsBuffer(const char* name);
+
+private:
+	void LoadPackIndex();
+	void UnpackRequiredAssets();
+
+	// Modified crc algorithm (mpeg2)
+	// Reference: https://stackoverflow.com/questions/54339800/how-to-modify-crc-32-to-crc-32-mpeg-2
+	inline uint32_t crc32b(uint32_t base, uint8_t* message, size_t l);
+
+	bool m_SuccessfullyLoaded;
+
+	std::filesystem::path m_Path;
+	std::filesystem::path m_RootPath;
+	std::filesystem::path m_ResPath;
+
+	eAssetBundleType m_AssetBundleType = eAssetBundleType::None;
+
+	PackIndex* m_PackIndex;
+};

dCommon/dClient/CMakeLists.txt

@@ -0,0 +1,6 @@
+set(DCOMMON_DCLIENT_SOURCES
+	"PackIndex.cpp"
+	"Pack.cpp"
+	"AssetManager.cpp"
+	PARENT_SCOPE
+)

dCommon/dClient/Pack.cpp

@@ -0,0 +1,118 @@
+#include "Pack.h"
+
+#include "ZCompression.h"
+
+Pack::Pack(const std::filesystem::path& filePath) {
+	m_FilePath = filePath;
+
+	if (!std::filesystem::exists(filePath)) {
+		return;
+	}
+
+	m_FileStream = std::ifstream(filePath, std::ios::in | std::ios::binary);
+
+	m_FileStream.read(m_Version, 7); 
+
+	m_FileStream.seekg(-8, std::ios::end); // move file pointer to 8 bytes before the end (location of the address of the record count)
+
+	uint32_t recordCountPos = 0;
+	BinaryIO::BinaryRead<uint32_t>(m_FileStream, recordCountPos);
+
+	m_FileStream.seekg(recordCountPos, std::ios::beg);
+
+	BinaryIO::BinaryRead<uint32_t>(m_FileStream, m_RecordCount);
+
+	for (int i = 0; i < m_RecordCount; i++) {
+		PackRecord record;
+		BinaryIO::BinaryRead<PackRecord>(m_FileStream, record);
+
+		m_Records.push_back(record);
+	}
+
+	m_FileStream.close();
+}
+
+bool Pack::HasFile(uint32_t crc) {
+	for (const auto& record : m_Records) {
+		if (record.m_Crc == crc) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
+bool Pack::ReadFileFromPack(uint32_t crc, char** data, uint32_t* len) {
+	// Time for some wacky C file reading for speed reasons
+
+	PackRecord pkRecord{};
+
+	for (const auto& record : m_Records) {
+		if (record.m_Crc == crc) {
+			pkRecord = record;
+			break;
+		}
+	}
+
+	if (pkRecord.m_Crc == 0) return false;
+
+	size_t pos = 0;
+	pos += pkRecord.m_FilePointer;
+
+	bool isCompressed = (pkRecord.m_IsCompressed & 0xff) > 0;
+	auto inPackSize = isCompressed ? pkRecord.m_CompressedSize : pkRecord.m_UncompressedSize;
+
+	FILE* file;
+#ifdef _WIN32
+	fopen_s(&file, m_FilePath.string().c_str(), "rb");
+#elif __APPLE__
+	// macOS has 64bit file IO by default
+	file = fopen(m_FilePath.string().c_str(), "rb");
+#else
+	file = fopen64(m_FilePath.string().c_str(), "rb");
+#endif
+
+	fseek(file, pos, SEEK_SET);
+
+	if (!isCompressed) {
+		char* tempData = (char*)malloc(pkRecord.m_UncompressedSize);
+		fread(tempData, sizeof(uint8_t), pkRecord.m_UncompressedSize, file);
+
+		*data = tempData;
+		*len = pkRecord.m_UncompressedSize;
+		fclose(file);
+
+		return true;
+	}
+
+	pos += 5; // skip header
+
+	fseek(file, pos, SEEK_SET);
+
+	char* decompressedData = (char*)malloc(pkRecord.m_UncompressedSize);
+	uint32_t currentReadPos = 0;
+
+	while (true) {
+		if (currentReadPos >= pkRecord.m_UncompressedSize) break;
+
+		uint32_t size;
+		fread(&size, sizeof(uint32_t), 1, file);
+		pos += 4; // Move pointer position 4 to the right
+
+		char* chunk = (char*)malloc(size);
+		fread(chunk, sizeof(int8_t), size, file);
+		pos += size; // Move pointer position the amount of bytes read to the right
+
+		int32_t err;
+		currentReadPos += ZCompression::Decompress((uint8_t*)chunk, size, reinterpret_cast<uint8_t*>(decompressedData + currentReadPos), ZCompression::MAX_SD0_CHUNK_SIZE, err);
+
+		free(chunk);
+	}
+
+	*data = decompressedData;
+	*len = pkRecord.m_UncompressedSize;
+
+	fclose(file);
+
+	return true;
+}

dCommon/dClient/Pack.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include <vector>
+#include <string>
+#include <filesystem>
+
+#pragma pack(push, 1)
+struct PackRecord {
+	uint32_t m_Crc;
+	int32_t m_LowerCrc;
+	int32_t m_UpperCrc;
+	uint32_t m_UncompressedSize;
+	char m_UncompressedHash[32];
+	uint32_t m_Padding1;
+	uint32_t m_CompressedSize;
+	char m_CompressedHash[32];
+	uint32_t m_Padding2;
+	uint32_t m_FilePointer;
+	uint32_t m_IsCompressed; // u32 bool
+};
+#pragma pack(pop)
+
+class Pack {
+public:
+	Pack(const std::filesystem::path& filePath);
+	~Pack() = default;
+
+	bool HasFile(uint32_t crc);
+	bool ReadFileFromPack(uint32_t crc, char** data, uint32_t* len);
+private:
+	std::ifstream m_FileStream;
+	std::filesystem::path m_FilePath;
+
+	char m_Version[7];
+
+	uint32_t m_RecordCount;
+	std::vector<PackRecord> m_Records;
+};

dCommon/dClient/PackIndex.cpp

@@ -0,0 +1,50 @@
+#include "PackIndex.h"
+
+
+PackIndex::PackIndex(const std::filesystem::path& filePath) {
+	m_FileStream = std::ifstream(filePath / "versions" / "primary.pki", std::ios::in | std::ios::binary);
+
+	BinaryIO::BinaryRead<uint32_t>(m_FileStream, m_Version);
+	BinaryIO::BinaryRead<uint32_t>(m_FileStream, m_PackPathCount);
+
+	for (int i = 0; i < m_PackPathCount; i++) {
+		uint32_t stringLen = 0;
+		BinaryIO::BinaryRead<uint32_t>(m_FileStream, stringLen);
+
+		std::string path;
+
+		for (int j = 0; j < stringLen; j++) {
+			char inChar;
+			BinaryIO::BinaryRead<char>(m_FileStream, inChar);
+
+			path += inChar;
+		}
+
+		m_PackPaths.push_back(path);
+	}
+
+	BinaryIO::BinaryRead<uint32_t>(m_FileStream, m_PackFileIndexCount);
+
+	for (int i = 0; i < m_PackFileIndexCount; i++) {
+		PackFileIndex packFileIndex;
+		BinaryIO::BinaryRead<PackFileIndex>(m_FileStream, packFileIndex);
+
+		m_PackFileIndices.push_back(packFileIndex);
+	}
+
+	Game::logger->Log("PackIndex", "Loaded pack catalog with %i pack files and %i files", m_PackPaths.size(), m_PackFileIndices.size());
+
+	for (const auto& item : m_PackPaths) {
+		auto* pack = new Pack(filePath / item);
+
+		m_Packs.push_back(pack);
+	}
+
+	m_FileStream.close();
+}
+
+PackIndex::~PackIndex() {
+	for (const auto* item : m_Packs) {
+		delete item;
+	}
+}

dCommon/dClient/PackIndex.h

@@ -0,0 +1,40 @@
+#pragma once
+
+#include <cstdint>
+
+#include <string>
+#include <vector>
+#include <filesystem>
+
+#include "Pack.h"
+
+#pragma pack(push, 1)
+struct PackFileIndex {
+	uint32_t m_Crc;
+	int32_t m_LowerCrc;
+	int32_t m_UpperCrc;
+	uint32_t m_PackFileIndex;
+	uint32_t m_IsCompressed; // u32 bool?
+};
+#pragma pack(pop)
+
+class PackIndex {
+public:
+	PackIndex(const std::filesystem::path& filePath);
+	~PackIndex();
+
+	const std::vector<std::string>& GetPackPaths() { return m_PackPaths; }
+	const std::vector<PackFileIndex>& GetPackFileIndices() { return m_PackFileIndices; }
+	const std::vector<Pack*>& GetPacks() { return m_Packs; }
+private:
+	std::ifstream m_FileStream;
+
+	uint32_t m_Version;
+
+	uint32_t m_PackPathCount;
+	std::vector<std::string> m_PackPaths;
+	uint32_t m_PackFileIndexCount;
+	std::vector<PackFileIndex> m_PackFileIndices;
+
+	std::vector<Pack*> m_Packs;
+};

dGame/dInventory/Item.cpp

@@ -13,6 +13,7 @@
 #include "PossessableComponent.h"
 #include "CharacterComponent.h"
 #include "eItemType.h"
+#include "AssetManager.h"
 
 class Inventory;
 
@@ -340,18 +341,23 @@ void Item::DisassembleModel() {
 	std::string renderAsset = result.fieldIsNull(0) ? "" : std::string(result.getStringField(0));
 	std::vector<std::string> renderAssetSplit = GeneralUtils::SplitString(renderAsset, '\\');
 
-	std::string lxfmlPath = "res/BrickModels/" + GeneralUtils::SplitString(renderAssetSplit.back(), '.')[0] + ".lxfml";
+	std::string lxfmlPath = "BrickModels/" + GeneralUtils::SplitString(renderAssetSplit.back(), '.').at(0) + ".lxfml";
-	std::ifstream file(lxfmlPath);
+	auto buffer = Game::assetManager->GetFileAsBuffer(lxfmlPath.c_str());
+
+	std::istream file(&buffer);
 
 	result.finalize();
 
 	if (!file.good()) {
+		buffer.close();
 		return;
 	}
 
 	std::stringstream data;
 	data << file.rdbuf();
 
+	buffer.close();
+
 	if (data.str().empty()) {
 		return;
 	}

dGame/dUtilities/BrickDatabase.cpp

@@ -3,6 +3,7 @@
 
 #include "BrickDatabase.h"
 #include "Game.h"
+#include "AssetManager.h"
 
 std::vector<Brick> BrickDatabase::emptyCache{};
 BrickDatabase* BrickDatabase::m_Address = nullptr;
@@ -17,7 +18,8 @@ std::vector<Brick>& BrickDatabase::GetBricks(const std::string& lxfmlPath) {
 		return cached->second;
 	}
 
-	std::ifstream file(lxfmlPath);
+	AssetMemoryBuffer buffer = Game::assetManager->GetFileAsBuffer(("client/" + lxfmlPath).c_str());
+	std::istream file(&buffer);
 	if (!file.good()) {
 		return emptyCache;
 	}
@@ -25,9 +27,12 @@ std::vector<Brick>& BrickDatabase::GetBricks(const std::string& lxfmlPath) {
 	std::stringstream data;
 	data << file.rdbuf();
 	if (data.str().empty()) {
+		buffer.close();
 		return emptyCache;
 	}
 
+	buffer.close();
+
 	auto* doc = new tinyxml2::XMLDocument();
 	if (doc->Parse(data.str().c_str(), data.str().size()) != 0) {
 		delete doc;

dGame/dUtilities/SlashCommandHandler.cpp

@@ -63,6 +63,7 @@
 #include "GameConfig.h"
 #include "ScriptedActivityComponent.h"
 #include "LevelProgressionComponent.h"
+#include "AssetManager.h"
 
 void SlashCommandHandler::HandleChatCommand(const std::u16string& command, Entity* entity, const SystemAddress& sysAddr) {
 	std::string chatCommand;
@@ -582,7 +583,8 @@ void SlashCommandHandler::HandleChatCommand(const std::u16string& command, Entit
 		if (args[0].find("/") != std::string::npos) return;
 		if (args[0].find("\\") != std::string::npos) return;
 
-		std::ifstream infile("./res/macros/" + args[0] + ".scm");
+		auto buf = Game::assetManager->GetFileAsBuffer(("macros/" + args[0] + ".scm").c_str());
+		std::istream infile(&buf);
 
 		if (infile.good()) {
 			std::string line;
@@ -593,6 +595,8 @@ void SlashCommandHandler::HandleChatCommand(const std::u16string& command, Entit
 			ChatPackets::SendSystemMessage(sysAddr, u"Unknown macro! Is the filename right?");
 		}
 
+		buf.close();
+
 		return;
 	}
 
@@ -1904,10 +1908,7 @@ bool SlashCommandHandler::CheckIfAccessibleZone(const unsigned int zoneID) {
 	CDZoneTableTable* zoneTable = CDClientManager::Instance()->GetTable<CDZoneTableTable>("ZoneTable");
 	const CDZoneTable* zone = zoneTable->Query(zoneID);
 	if (zone != nullptr) {
-		std::string zonePath = "./res/maps/" + zone->zoneName;
+		return Game::assetManager->HasFile(("maps/" + zone->zoneName).c_str());
-		std::transform(zonePath.begin(), zonePath.end(), zonePath.begin(), ::tolower);
-		std::ifstream f(zonePath.c_str());
-		return f.good();
 	} else {
 		return false;
 	}

dMasterServer/MasterServer.cpp

@@ -25,6 +25,7 @@
 #include "dConfig.h"
 #include "dLogger.h"
 #include "dServer.h"
+#include "AssetManager.h"
 
 //RakNet includes:
 #include "RakNetDefines.h"
@@ -44,6 +45,7 @@ namespace Game {
 	dServer* server;
 	InstanceManager* im;
 	dConfig* config;
+	AssetManager* assetManager;
 } //namespace Game
 
 bool shutdownSequenceStarted = false;
@@ -99,44 +101,49 @@ int main(int argc, char** argv) {
 		return EXIT_FAILURE;
 	}
 
-	MigrationRunner::RunMigrations();
+	try {
+		std::string client_path = config.GetValue("client_location");
+		if (client_path.empty()) client_path = "./res";
+		Game::assetManager = new AssetManager(config.GetValue("client_location"));
+	} catch (std::runtime_error& ex) {
+		Game::logger->Log("MasterServer", "Got an error while setting up assets: %s", ex.what());
 
-	//Check CDClient exists
-	const std::string cdclient_path = "./res/CDServer.sqlite";
-	std::ifstream cdclient_fd(cdclient_path);
-	if (!cdclient_fd.good()) {
-		Game::logger->Log("WorldServer", "%s could not be opened.  Looking for cdclient.fdb to convert to sqlite.", cdclient_path.c_str());
-		cdclient_fd.close();
-
-		const std::string cdclientFdbPath = "./res/cdclient.fdb";
-		cdclient_fd.open(cdclientFdbPath);
-		if (!cdclient_fd.good()) {
-			Game::logger->Log(
-				"WorldServer", "%s could not be opened."
-				"Please move a cdclient.fdb or an already converted database to build/res.", cdclientFdbPath.c_str());
 		return EXIT_FAILURE;
 	}
-		Game::logger->Log("WorldServer", "Found %s.  Clearing cdserver migration_history then copying and converting to sqlite.", cdclientFdbPath.c_str());
+
+	MigrationRunner::RunMigrations();
+
+	// Check CDClient exists
+	if (!std::filesystem::exists(Game::assetManager->GetResPath() / "CDServer.sqlite")) {
+		Game::logger->Log("WorldServer", "CDServer.sqlite could not be opened. Looking for cdclient.fdb to convert to sqlite.");
+
+		if (!std::filesystem::exists(Game::assetManager->GetResPath() / "cdclient.fdb")) {
+			Game::logger->Log("WorldServer", "cdclient.fdb could not be opened. Please move a cdclient.fdb or an already converted database to build/res.");
+			return EXIT_FAILURE;
+		}
+
+		Game::logger->Log("WorldServer", "Found cdclient.fdb.  Clearing cdserver migration_history then copying and converting to sqlite.");
 		auto stmt = Database::CreatePreppedStmt(R"#(DELETE FROM migration_history WHERE name LIKE "%cdserver%";)#");
 		stmt->executeUpdate();
 		delete stmt;
-		cdclient_fd.close();
 
-		std::string res = "python3 ../thirdparty/docker-utils/utils/fdb_to_sqlite.py " + cdclientFdbPath;
+		std::string res = "python3 ../thirdparty/docker-utils/utils/fdb_to_sqlite.py " + (Game::assetManager->GetResPath() / "cdclient.fdb").string();
-		int r = system(res.c_str());
+
-		if (r != 0) {
+		int result = system(res.c_str());
+		if (result != 0) {
 			Game::logger->Log("MasterServer", "Failed to convert fdb to sqlite");
 			return EXIT_FAILURE;
 		}
-		if (std::rename("./cdclient.sqlite", "./res/CDServer.sqlite") != 0) {
+
-			Game::logger->Log("MasterServer", "failed to move cdclient file.");
+		if (std::rename("./cdclient.sqlite", (Game::assetManager->GetResPath() / "CDServer.sqlite").string().c_str()) != 0) {
+			Game::logger->Log("MasterServer", "Failed to move cdclient file.");
 			return EXIT_FAILURE;
 		}
 	}
 
 	//Connect to CDClient
 	try {
-		CDClientDatabase::Connect(cdclient_path);
+		CDClientDatabase::Connect((Game::assetManager->GetResPath() / "CDServer.sqlite").string());
 	} catch (CppSQLite3Exception& e) {
 		Game::logger->Log("WorldServer", "Unable to connect to CDServer SQLite Database");
 		Game::logger->Log("WorldServer", "Error: %s", e.errorMessage());
@@ -152,7 +159,7 @@ int main(int argc, char** argv) {
 		CDClientManager::Instance()->Initialize();
 	} catch (CppSQLite3Exception& e) {
 		Game::logger->Log("WorldServer", "Failed to initialize CDServer SQLite Database");
-		Game::logger->Log("WorldServer", "May be caused by corrupted file: %s", cdclient_path.c_str());
+		Game::logger->Log("WorldServer", "May be caused by corrupted file: %s", (Game::assetManager->GetResPath() / "CDServer.sqlite").string().c_str());
 		Game::logger->Log("WorldServer", "Error: %s", e.errorMessage());
 		Game::logger->Log("WorldServer", "Error Code: %i", e.errorCode());
 		return EXIT_FAILURE;

dNavigation/dNavMesh.cpp

@@ -43,7 +43,7 @@ dNavMesh::~dNavMesh() {
 
 void dNavMesh::LoadNavmesh() {
 
-	std::string path = "./res/maps/navmeshes/" + std::to_string(m_ZoneId) + ".bin";
+	std::string path = "./navmeshes/" + std::to_string(m_ZoneId) + ".bin";
 
 	if (!BinaryIO::DoesFileExist(path)) {
 		return;

dWorldServer/WorldServer.cpp

@@ -55,6 +55,7 @@
 #include "MasterPackets.h"
 #include "Player.h"
 #include "PropertyManagementComponent.h"
+#include "AssetManager.h"
 
 #include "ZCompression.h"
 
@@ -68,6 +69,8 @@ namespace Game {
 	dLocale* locale;
 	std::mt19937 randomEngine;
 
+	AssetManager* assetManager;
+
 	RakPeerInterface* chatServer;
 	SystemAddress chatSysAddr;
 }
@@ -142,9 +145,19 @@ int main(int argc, char** argv) {
 	Game::logger->SetLogDebugStatements(config.GetValue("log_debug_statements") == "1");
 	if (config.GetValue("disable_chat") == "1") chatDisabled = true;
 
+	try {
+		std::string client_path = config.GetValue("client_location");
+		if (client_path.empty()) client_path = "./res";
+		Game::assetManager = new AssetManager(config.GetValue("client_location"));
+	} catch (std::runtime_error& ex) {
+		Game::logger->Log("WorldServer", "Got an error while setting up assets: %s", ex.what());
+
+		return EXIT_FAILURE;
+	}
+
 	// Connect to CDClient
 	try {
-		CDClientDatabase::Connect("./res/CDServer.sqlite");
+		CDClientDatabase::Connect((Game::assetManager->GetResPath() / "CDServer.sqlite").string());
 	} catch (CppSQLite3Exception& e) {
 		Game::logger->Log("WorldServer", "Unable to connect to CDServer SQLite Database");
 		Game::logger->Log("WorldServer", "Error: %s", e.errorMessage());
@@ -189,7 +202,7 @@ int main(int argc, char** argv) {
 	ObjectIDManager::Instance()->Initialize();
 	UserManager::Instance()->Initialize();
 	LootGenerator::Instance();
-	Game::chatFilter = new dChatFilter("./res/chatplus_en_us", bool(std::stoi(config.GetValue("dont_generate_dcf"))));
+	Game::chatFilter = new dChatFilter(Game::assetManager->GetResPath().string() + "/chatplus_en_us", bool(std::stoi(config.GetValue("dont_generate_dcf"))));
 
 	Game::server = new dServer(masterIP, ourPort, instanceID, maxClients, false, true, Game::logger, masterIP, masterPort, ServerType::World, zoneID);
 
@@ -243,14 +256,14 @@ int main(int argc, char** argv) {
 			std::ifstream fileStream;
 
 			static const std::vector<std::string> aliases = {
-				"res/CDServers.fdb",
+				"CDServers.fdb",
-				"res/cdserver.fdb",
+				"cdserver.fdb",
-				"res/CDClient.fdb",
+				"CDClient.fdb",
-				"res/cdclient.fdb",
+				"cdclient.fdb",
 			};
 
 			for (const auto& file : aliases) {
-				fileStream.open(file, std::ios::binary | std::ios::in);
+				fileStream.open(Game::assetManager->GetResPath() / file, std::ios::binary | std::ios::in);
 				if (fileStream.is_open()) {
 					break;
 				}

dZoneManager/Level.cpp

@@ -13,17 +13,22 @@
 #include "EntityManager.h"
 #include "CDFeatureGatingTable.h"
 #include "CDClientManager.h"
+#include "AssetManager.h"
 
 Level::Level(Zone* parentZone, const std::string& filepath) {
 	m_ParentZone = parentZone;
-	std::ifstream file(filepath, std::ios_base::in | std::ios_base::binary);
+
-	if (file) {
+	auto buffer = Game::assetManager->GetFileAsBuffer(filepath.c_str());
-		ReadChunks(file);
+
-	} else {
+	if (!buffer.m_Success) {
 		Game::logger->Log("Level", "Failed to load %s", filepath.c_str());
+		return;
 	}
 
-	file.close();
+	std::istream file(&buffer);
+	ReadChunks(file);
+
+	buffer.close();
 }
 
 Level::~Level() {
@@ -41,7 +46,7 @@ const void Level::PrintAllObjects() {
 	}
 }
 
-void Level::ReadChunks(std::ifstream& file) {
+void Level::ReadChunks(std::istream& file) {
 	const uint32_t CHNK_HEADER = ('C' + ('H' << 8) + ('N' << 16) + ('K' << 24));
 
 	while (!file.eof()) {
@@ -139,7 +144,7 @@ void Level::ReadChunks(std::ifstream& file) {
 	}
 }
 
-void Level::ReadFileInfoChunk(std::ifstream& file, Header& header) {
+void Level::ReadFileInfoChunk(std::istream& file, Header& header) {
 	FileInfoChunk* fi = new FileInfoChunk;
 	BinaryIO::BinaryRead(file, fi->version);
 	BinaryIO::BinaryRead(file, fi->revision);
@@ -152,7 +157,7 @@ void Level::ReadFileInfoChunk(std::ifstream& file, Header& header) {
 	if (header.fileInfo->revision == 3452816845 && m_ParentZone->GetZoneID().GetMapID() == 1100) header.fileInfo->revision = 26;
 }
 
-void Level::ReadSceneObjectDataChunk(std::ifstream& file, Header& header) {
+void Level::ReadSceneObjectDataChunk(std::istream& file, Header& header) {
 	SceneObjectDataChunk* chunk = new SceneObjectDataChunk;
 	uint32_t objectsCount = 0;
 	BinaryIO::BinaryRead(file, objectsCount);

dZoneManager/Level.h

@@ -67,7 +67,7 @@ private:
 	Zone* m_ParentZone;
 
 	//private functions:
-	void ReadChunks(std::ifstream& file);
+	void ReadChunks(std::istream& file);
-	void ReadFileInfoChunk(std::ifstream& file, Header& header);
+	void ReadFileInfoChunk(std::istream& file, Header& header);
-	void ReadSceneObjectDataChunk(std::ifstream& file, Header& header);
+	void ReadSceneObjectDataChunk(std::istream& file, Header& header);
 };

dZoneManager/Zone.cpp

@@ -7,6 +7,7 @@
 #include "GeneralUtils.h"
 #include "BinaryIO.h"
 
+#include "AssetManager.h"
 #include "CDClientManager.h"
 #include "CDZoneTableTable.h"
 #include "Spawner.h"
@@ -40,7 +41,8 @@ void Zone::LoadZoneIntoMemory() {
 	m_ZonePath = m_ZoneFilePath.substr(0, m_ZoneFilePath.rfind('/') + 1);
 	if (m_ZoneFilePath == "ERR") return;
 
-	std::ifstream file(m_ZoneFilePath, std::ios::binary);
+	AssetMemoryBuffer buffer = Game::assetManager->GetFileAsBuffer(m_ZoneFilePath.c_str());
+	std::istream file(&buffer);
 	if (file) {
 		BinaryIO::BinaryRead(file, m_ZoneFileFormatVersion);
 
@@ -144,17 +146,13 @@ void Zone::LoadZoneIntoMemory() {
 				}
 
 			}
-
-
-			//m_PathData.resize(m_PathDataLength);
-			//file.read((char*)&m_PathData[0], m_PathDataLength);
 		}
 	} else {
 		Game::logger->Log("Zone", "Failed to open: %s", m_ZoneFilePath.c_str());
 	}
 	m_ZonePath = m_ZoneFilePath.substr(0, m_ZoneFilePath.rfind('/') + 1);
 
-	file.close();
+	buffer.close();
 }
 
 std::string Zone::GetFilePathForZoneID() {
@@ -162,7 +160,7 @@ std::string Zone::GetFilePathForZoneID() {
 	CDZoneTableTable* zoneTable = CDClientManager::Instance()->GetTable<CDZoneTableTable>("ZoneTable");
 	const CDZoneTable* zone = zoneTable->Query(this->GetZoneID().GetMapID());
 	if (zone != nullptr) {
-		std::string toReturn = "./res/maps/" + zone->zoneName;
+		std::string toReturn = "maps/" + zone->zoneName;
 		std::transform(toReturn.begin(), toReturn.end(), toReturn.begin(), ::tolower);
 		return toReturn;
 	}
@@ -222,7 +220,7 @@ const void Zone::PrintAllGameObjects() {
 	}
 }
 
-void Zone::LoadScene(std::ifstream& file) {
+void Zone::LoadScene(std::istream& file) {
 	SceneRef scene;
 	scene.level = nullptr;
 	LWOSCENEID lwoSceneID(LWOZONEID_INVALID, 0);
@@ -264,10 +262,17 @@ void Zone::LoadScene(std::ifstream& file) {
 
 std::vector<LUTriggers::Trigger*> Zone::LoadLUTriggers(std::string triggerFile, LWOSCENEID sceneID) {
 	std::vector<LUTriggers::Trigger*> lvlTriggers;
-	std::ifstream file(m_ZonePath + triggerFile);
+
+	auto buffer = Game::assetManager->GetFileAsBuffer((m_ZonePath + triggerFile).c_str());
+
+	if (!buffer.m_Success) return lvlTriggers;
+
+	std::istream file(&buffer);
 	std::stringstream data;
 	data << file.rdbuf();
 
+	buffer.close();
+
 	if (data.str().size() == 0) return lvlTriggers;
 
 	tinyxml2::XMLDocument* doc = new tinyxml2::XMLDocument();
@@ -336,7 +341,7 @@ const Path* Zone::GetPath(std::string name) const {
 	return nullptr;
 }
 
-void Zone::LoadSceneTransition(std::ifstream& file) {
+void Zone::LoadSceneTransition(std::istream& file) {
 	SceneTransition sceneTrans;
 	if (m_ZoneFileFormatVersion < Zone::ZoneFileFormatVersion::Auramar) {
 		uint8_t length;
@@ -355,14 +360,14 @@ void Zone::LoadSceneTransition(std::ifstream& file) {
 	m_SceneTransitions.push_back(sceneTrans);
 }
 
-SceneTransitionInfo Zone::LoadSceneTransitionInfo(std::ifstream& file) {
+SceneTransitionInfo Zone::LoadSceneTransitionInfo(std::istream& file) {
 	SceneTransitionInfo info;
 	BinaryIO::BinaryRead(file, info.sceneID);
 	BinaryIO::BinaryRead(file, info.position);
 	return info;
 }
 
-void Zone::LoadPath(std::ifstream& file) {
+void Zone::LoadPath(std::istream& file) {
 	// Currently only spawner (type 4) paths are supported
 	Path path = Path();
 

dZoneManager/Zone.h

@@ -225,9 +225,9 @@ private:
 	std::map<LWOSCENEID, uint32_t, mapCompareLwoSceneIDs> m_MapRevisions; //rhs is the revision!
 
 	//private ("helper") functions:
-	void LoadScene(std::ifstream& file);
+	void LoadScene(std::istream& file);
 	std::vector<LUTriggers::Trigger*> LoadLUTriggers(std::string triggerFile, LWOSCENEID sceneID);
-	void LoadSceneTransition(std::ifstream& file);
+	void LoadSceneTransition(std::istream& file);
-	SceneTransitionInfo LoadSceneTransitionInfo(std::ifstream& file);
+	SceneTransitionInfo LoadSceneTransitionInfo(std::istream& file);
-	void LoadPath(std::ifstream& file);
+	void LoadPath(std::istream& file);
 };

resources/sharedconfig.ini

@@ -21,3 +21,7 @@ max_clients=999
 
 # Where to put crashlogs
 dump_folder=
+
+# The location of the client
+# Either the folder with /res or with /client and /versions
+client_location=