diff --git a/CMakeLists.txt b/CMakeLists.txt index 33efbdc9..9437bc63 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -101,6 +101,17 @@ foreach(resource_file ${RESOURCE_FILES}) endif() endforeach() +# Copy navmesh data on first build and extract it +if (NOT EXISTS ${PROJECT_BINARY_DIR}/navmeshes/) + configure_file( + ${CMAKE_SOURCE_DIR}/resources/navmeshes.zip ${PROJECT_BINARY_DIR}/navmeshes.zip + COPYONLY + ) + + file(ARCHIVE_EXTRACT INPUT ${PROJECT_BINARY_DIR}/navmeshes.zip) + file(REMOVE ${PROJECT_BINARY_DIR}/navmeshes.zip) +endif() + # Copy vanity files on first build set(VANITY_FILES "CREDITS.md" "INFO.md" "TESTAMENT.md" "NPC.xml") foreach(file ${VANITY_FILES}) @@ -145,6 +156,7 @@ set(INCLUDED_DIRECTORIES "dGame/dEntity" "dGame/dPropertyBehaviors" "dGame/dUtilities" + "dCommon/dClient" "dPhysics" "dNavigation" "dNavigation/dTerrain" diff --git a/dChatServer/ChatServer.cpp b/dChatServer/ChatServer.cpp index 273921eb..5a60e494 100644 --- a/dChatServer/ChatServer.cpp +++ b/dChatServer/ChatServer.cpp @@ -12,6 +12,7 @@ #include "dMessageIdentifiers.h" #include "dChatFilter.h" #include "Diagnostics.h" +#include "AssetManager.h" #include "PlayerContainer.h" #include "ChatPacketHandler.h" @@ -22,6 +23,7 @@ namespace Game { dServer* server; dConfig* config; dChatFilter* chatFilter; + AssetManager* assetManager; } //RakNet includes: @@ -50,6 +52,16 @@ int main(int argc, char** argv) { Game::logger->SetLogToConsole(bool(std::stoi(config.GetValue("log_to_console")))); Game::logger->SetLogDebugStatements(config.GetValue("log_debug_statements") == "1"); + try { + std::string client_path = config.GetValue("client_location"); + if (client_path.empty()) client_path = "./res"; + Game::assetManager = new AssetManager(config.GetValue("client_location")); + } catch (std::runtime_error& ex) { + Game::logger->Log("ChatServer", "Got an error while setting up assets: %s", ex.what()); + + return EXIT_FAILURE; + } + //Connect to the MySQL Database std::string mysql_host = config.GetValue("mysql_host"); std::string mysql_database = config.GetValue("mysql_database"); @@ -87,7 +99,7 @@ int main(int argc, char** argv) { Game::server = new dServer(config.GetValue("external_ip"), ourPort, 0, maxClients, false, true, Game::logger, masterIP, masterPort, ServerType::Chat); - Game::chatFilter = new dChatFilter("./res/chatplus_en_us", bool(std::stoi(config.GetValue("dont_generate_dcf")))); + Game::chatFilter = new dChatFilter(Game::assetManager->GetResPath().string() + "/chatplus_en_us", bool(std::stoi(config.GetValue("dont_generate_dcf")))); //Run it until server gets a kill message from Master: auto t = std::chrono::high_resolution_clock::now(); diff --git a/dCommon/BinaryIO.cpp b/dCommon/BinaryIO.cpp index 7cb18331..22e4de60 100644 --- a/dCommon/BinaryIO.cpp +++ b/dCommon/BinaryIO.cpp @@ -10,7 +10,7 @@ void BinaryIO::WriteString(const std::string& stringToWrite, std::ofstream& outs } //For reading null-terminated strings -std::string BinaryIO::ReadString(std::ifstream& instream) { +std::string BinaryIO::ReadString(std::istream& instream) { std::string toReturn; char buffer; @@ -25,7 +25,7 @@ std::string BinaryIO::ReadString(std::ifstream& instream) { } //For reading strings of a specific size -std::string BinaryIO::ReadString(std::ifstream& instream, size_t size) { +std::string BinaryIO::ReadString(std::istream& instream, size_t size) { std::string toReturn; char buffer; @@ -37,7 +37,7 @@ std::string BinaryIO::ReadString(std::ifstream& instream, size_t size) { return toReturn; } -std::string BinaryIO::ReadWString(std::ifstream& instream) { +std::string BinaryIO::ReadWString(std::istream& instream) { size_t size; BinaryRead(instream, size); //toReturn.resize(size); diff --git a/dCommon/BinaryIO.h b/dCommon/BinaryIO.h index 1f9aaefd..a117ad0d 100644 --- a/dCommon/BinaryIO.h +++ b/dCommon/BinaryIO.h @@ -10,16 +10,15 @@ namespace BinaryIO { template std::istream& BinaryRead(std::istream& stream, T& value) { - if (!stream.good()) - printf("bla"); + if (!stream.good()) throw std::runtime_error("Failed to read from istream."); return stream.read(reinterpret_cast(&value), sizeof(T)); } void WriteString(const std::string& stringToWrite, std::ofstream& outstream); - std::string ReadString(std::ifstream& instream); - std::string ReadString(std::ifstream& instream, size_t size); - std::string ReadWString(std::ifstream& instream); + std::string ReadString(std::istream& instream); + std::string ReadString(std::istream& instream, size_t size); + std::string ReadWString(std::istream& instream); inline bool DoesFileExist(const std::string& name) { std::ifstream f(name.c_str()); diff --git a/dCommon/BrickByBrickFix.cpp b/dCommon/BrickByBrickFix.cpp index f0c4e824..15194bf9 100644 --- a/dCommon/BrickByBrickFix.cpp +++ b/dCommon/BrickByBrickFix.cpp @@ -49,10 +49,10 @@ uint32_t BrickByBrickFix::TruncateBrokenBrickByBrickXml() { } // Ignore the valgrind warning about uninitialized values. These are discarded later when we know the actual uncompressed size. - std::unique_ptr uncompressedChunk(new uint8_t[MAX_SD0_CHUNK_SIZE]); + std::unique_ptr uncompressedChunk(new uint8_t[ZCompression::MAX_SD0_CHUNK_SIZE]); int32_t err{}; int32_t actualUncompressedSize = ZCompression::Decompress( - compressedChunk.get(), chunkSize, uncompressedChunk.get(), MAX_SD0_CHUNK_SIZE, err); + compressedChunk.get(), chunkSize, uncompressedChunk.get(), ZCompression::MAX_SD0_CHUNK_SIZE, err); if (actualUncompressedSize != -1) { uint32_t previousSize = completeUncompressedModel.size(); diff --git a/dCommon/BrickByBrickFix.h b/dCommon/BrickByBrickFix.h index 0c7e314c..7450fb71 100644 --- a/dCommon/BrickByBrickFix.h +++ b/dCommon/BrickByBrickFix.h @@ -17,10 +17,4 @@ namespace BrickByBrickFix { * @return The number of BrickByBrick models that were updated */ uint32_t UpdateBrickByBrickModelsToSd0(); - - /** - * @brief Max size of an inflated sd0 zlib chunk - * - */ - constexpr uint32_t MAX_SD0_CHUNK_SIZE = 1024 * 256; }; diff --git a/dCommon/CMakeLists.txt b/dCommon/CMakeLists.txt index cb73bf6a..46102b74 100644 --- a/dCommon/CMakeLists.txt +++ b/dCommon/CMakeLists.txt @@ -17,6 +17,12 @@ set(DCOMMON_SOURCES "AMFFormat.cpp" "BrickByBrickFix.cpp" ) +add_subdirectory(dClient) + +foreach(file ${DCOMMON_DCLIENT_SOURCES}) + set(DCOMMON_SOURCES ${DCOMMON_SOURCES} "dClient/${file}") +endforeach() + include_directories(${PROJECT_SOURCE_DIR}/dCommon/) add_library(dCommon STATIC ${DCOMMON_SOURCES}) diff --git a/dCommon/Game.h b/dCommon/Game.h index f4862602..616c7fbf 100644 --- a/dCommon/Game.h +++ b/dCommon/Game.h @@ -10,6 +10,7 @@ class dChatFilter; class dConfig; class dLocale; class RakPeerInterface; +class AssetManager; struct SystemAddress; namespace Game { @@ -22,5 +23,6 @@ namespace Game { extern dLocale* locale; extern std::mt19937 randomEngine; extern RakPeerInterface* chatServer; + extern AssetManager* assetManager; extern SystemAddress chatSysAddr; } diff --git a/dCommon/GeneralUtils.cpp b/dCommon/GeneralUtils.cpp index 4a6c4739..24ea72a0 100644 --- a/dCommon/GeneralUtils.cpp +++ b/dCommon/GeneralUtils.cpp @@ -50,6 +50,7 @@ bool _IsSuffixChar(uint8_t c) { bool GeneralUtils::_NextUTF8Char(std::string_view& slice, uint32_t& out) { size_t rem = slice.length(); + if (slice.empty()) return false; const uint8_t* bytes = (const uint8_t*)&slice.front(); if (rem > 0) { uint8_t first = bytes[0]; diff --git a/dCommon/ZCompression.h b/dCommon/ZCompression.h index 22a5ff86..84e8a9b4 100644 --- a/dCommon/ZCompression.h +++ b/dCommon/ZCompression.h @@ -8,5 +8,11 @@ namespace ZCompression { int32_t Compress(const uint8_t* abSrc, int32_t nLenSrc, uint8_t* abDst, int32_t nLenDst); int32_t Decompress(const uint8_t* abSrc, int32_t nLenSrc, uint8_t* abDst, int32_t nLenDst, int32_t& nErr); + + /** + * @brief Max size of an inflated sd0 zlib chunk + * + */ + constexpr uint32_t MAX_SD0_CHUNK_SIZE = 1024 * 256; } diff --git a/dCommon/dClient/AssetManager.cpp b/dCommon/dClient/AssetManager.cpp new file mode 100644 index 00000000..3319bad7 --- /dev/null +++ b/dCommon/dClient/AssetManager.cpp @@ -0,0 +1,201 @@ +#include "AssetManager.h" + +#include + +AssetManager::AssetManager(const std::string& path) { + if (!std::filesystem::is_directory(path)) { + throw std::runtime_error("Attempted to load asset bundle (" + path + ") however it is not a valid directory."); + } + + m_Path = std::filesystem::path(path); + + if (std::filesystem::exists(m_Path / "client") && std::filesystem::exists(m_Path / "versions")) { + m_AssetBundleType = eAssetBundleType::Packed; + + m_RootPath = m_Path; + m_ResPath = (m_Path / "client" / "res"); + } else if (std::filesystem::exists(m_Path / ".." / "versions") && std::filesystem::exists(m_Path / "res")) { + m_AssetBundleType = eAssetBundleType::Packed; + + m_RootPath = (m_Path / ".."); + m_ResPath = (m_Path / "res"); + } else if (std::filesystem::exists(m_Path / "pack") && std::filesystem::exists(m_Path / ".." / ".." / "versions")) { + m_AssetBundleType = eAssetBundleType::Packed; + + m_RootPath = (m_Path / ".." / ".."); + m_ResPath = m_Path; + } else if (std::filesystem::exists(m_Path / "res" / "cdclient.fdb") && !std::filesystem::exists(m_Path / "res" / "pack")) { + m_AssetBundleType = eAssetBundleType::Unpacked; + + m_ResPath = (m_Path / "res"); + } else if (std::filesystem::exists(m_Path / "cdclient.fdb") && !std::filesystem::exists(m_Path / "pack")) { + m_AssetBundleType = eAssetBundleType::Unpacked; + + m_ResPath = m_Path; + } + + if (m_AssetBundleType == eAssetBundleType::None) { + throw std::runtime_error("Failed to identify client type, cannot read client data."); + } + + switch (m_AssetBundleType) { + case eAssetBundleType::Packed: { + this->LoadPackIndex(); + + this->UnpackRequiredAssets(); + + break; + } + } +} + +void AssetManager::LoadPackIndex() { + m_PackIndex = new PackIndex(m_RootPath); +} + +std::filesystem::path AssetManager::GetResPath() { + return m_ResPath; +} + +eAssetBundleType AssetManager::GetAssetBundleType() { + return m_AssetBundleType; +} + +bool AssetManager::HasFile(const char* name) { + auto fixedName = std::string(name); + std::transform(fixedName.begin(), fixedName.end(), fixedName.begin(), [](uint8_t c) { return std::tolower(c); }); + std::replace(fixedName.begin(), fixedName.end(), '/', '\\'); + + auto realPathName = fixedName; + + if (fixedName.rfind("client\\res\\", 0) != 0) { + fixedName = "client\\res\\" + fixedName; + } + + if (std::filesystem::exists(m_ResPath / realPathName)) { + return true; + } + + uint32_t crc = crc32b(0xFFFFFFFF, (uint8_t*)fixedName.c_str(), fixedName.size()); + crc = crc32b(crc, (Bytef*)"\0\0\0\0", 4); + + for (const auto& item : this->m_PackIndex->GetPackFileIndices()) { + if (item.m_Crc == crc) { + return true; + } + } + + return false; +} + +bool AssetManager::GetFile(const char* name, char** data, uint32_t* len) { + auto fixedName = std::string(name); + std::transform(fixedName.begin(), fixedName.end(), fixedName.begin(), [](uint8_t c) { return std::tolower(c); }); + std::replace(fixedName.begin(), fixedName.end(), '/', '\\'); + + auto realPathName = fixedName; + + if (fixedName.rfind("client\\res\\", 0) != 0) { + fixedName = "client\\res\\" + fixedName; + } + + if (std::filesystem::exists(m_ResPath / realPathName)) { + FILE* file; +#ifdef _WIN32 + fopen_s(&file, (m_ResPath / realPathName).string().c_str(), "rb"); +#elif __APPLE__ + // macOS has 64bit file IO by default + file = fopen((m_ResPath / realPathName).string().c_str(), "rb"); +#else + file = fopen64((m_ResPath / realPathName).string().c_str(), "rb"); +#endif + fseek(file, 0, SEEK_END); + *len = ftell(file); + *data = (char*)malloc(*len); + fseek(file, 0, SEEK_SET); + fread(*data, sizeof(uint8_t), *len, file); + fclose(file); + + return true; + } + + if (this->m_AssetBundleType == eAssetBundleType::Unpacked) return false; + + int32_t packIndex = -1; + uint32_t crc = crc32b(0xFFFFFFFF, (uint8_t*)fixedName.c_str(), fixedName.size()); + crc = crc32b(crc, (Bytef*)"\0\0\0\0", 4); + + for (const auto& item : this->m_PackIndex->GetPackFileIndices()) { + if (item.m_Crc == crc) { + packIndex = item.m_PackFileIndex; + crc = item.m_Crc; + break; + } + } + + if (packIndex == -1 || !crc) { + return false; + } + + auto packs = this->m_PackIndex->GetPacks(); + auto* pack = packs.at(packIndex); + + bool success = pack->ReadFileFromPack(crc, data, len); + + return success; +} + +AssetMemoryBuffer AssetManager::GetFileAsBuffer(const char* name) { + char* buf; + uint32_t len; + + bool success = this->GetFile(name, &buf, &len); + + return AssetMemoryBuffer(buf, len, success); +} + +void AssetManager::UnpackRequiredAssets() { + if (std::filesystem::exists(m_ResPath / "cdclient.fdb")) return; + + char* data; + uint32_t size; + + bool success = this->GetFile("cdclient.fdb", &data, &size); + + if (!success) { + Game::logger->Log("AssetManager", "Failed to extract required files from the packs."); + + delete data; + + return; + } + + std::ofstream cdclientOutput(m_ResPath / "cdclient.fdb", std::ios::out | std::ios::binary); + cdclientOutput.write(data, size); + cdclientOutput.close(); + + delete data; + + return; +} + +uint32_t AssetManager::crc32b(uint32_t base, uint8_t* message, size_t l) { + size_t i, j; + uint32_t crc, msb; + + crc = base; + for (i = 0; i < l; i++) { + // xor next byte to upper bits of crc + crc ^= (((unsigned int)message[i]) << 24); + for (j = 0; j < 8; j++) { // Do eight times. + msb = crc >> 31; + crc <<= 1; + crc ^= (0 - msb) & 0x04C11DB7; + } + } + return crc; // don't complement crc on output +} + +AssetManager::~AssetManager() { + delete m_PackIndex; +} diff --git a/dCommon/dClient/AssetManager.h b/dCommon/dClient/AssetManager.h new file mode 100644 index 00000000..87653845 --- /dev/null +++ b/dCommon/dClient/AssetManager.h @@ -0,0 +1,78 @@ +#pragma once + +#include +#include +#include +#include + +#include "Pack.h" +#include "PackIndex.h" + +enum class eAssetBundleType { + None, + Unpacked, + Packed +}; + +struct AssetMemoryBuffer : std::streambuf { + char* m_Base; + bool m_Success; + + AssetMemoryBuffer(char* base, std::ptrdiff_t n, bool success) { + m_Base = base; + m_Success = success; + if (!m_Success) return; + this->setg(base, base, base + n); + } + + pos_type seekpos(pos_type sp, std::ios_base::openmode which) override { + return seekoff(sp - pos_type(off_type(0)), std::ios_base::beg, which); + } + + pos_type seekoff(off_type off, + std::ios_base::seekdir dir, + std::ios_base::openmode which = std::ios_base::in) override { + if (dir == std::ios_base::cur) + gbump(off); + else if (dir == std::ios_base::end) + setg(eback(), egptr() + off, egptr()); + else if (dir == std::ios_base::beg) + setg(eback(), eback() + off, egptr()); + return gptr() - eback(); + } + + void close() { + delete m_Base; + } +}; + +class AssetManager { +public: + AssetManager(const std::string& path); + ~AssetManager(); + + std::filesystem::path GetResPath(); + eAssetBundleType GetAssetBundleType(); + + bool HasFile(const char* name); + bool GetFile(const char* name, char** data, uint32_t* len); + AssetMemoryBuffer GetFileAsBuffer(const char* name); + +private: + void LoadPackIndex(); + void UnpackRequiredAssets(); + + // Modified crc algorithm (mpeg2) + // Reference: https://stackoverflow.com/questions/54339800/how-to-modify-crc-32-to-crc-32-mpeg-2 + inline uint32_t crc32b(uint32_t base, uint8_t* message, size_t l); + + bool m_SuccessfullyLoaded; + + std::filesystem::path m_Path; + std::filesystem::path m_RootPath; + std::filesystem::path m_ResPath; + + eAssetBundleType m_AssetBundleType = eAssetBundleType::None; + + PackIndex* m_PackIndex; +}; diff --git a/dCommon/dClient/CMakeLists.txt b/dCommon/dClient/CMakeLists.txt new file mode 100644 index 00000000..69bb1712 --- /dev/null +++ b/dCommon/dClient/CMakeLists.txt @@ -0,0 +1,6 @@ +set(DCOMMON_DCLIENT_SOURCES + "PackIndex.cpp" + "Pack.cpp" + "AssetManager.cpp" + PARENT_SCOPE +) diff --git a/dCommon/dClient/Pack.cpp b/dCommon/dClient/Pack.cpp new file mode 100644 index 00000000..d7716bc9 --- /dev/null +++ b/dCommon/dClient/Pack.cpp @@ -0,0 +1,118 @@ +#include "Pack.h" + +#include "ZCompression.h" + +Pack::Pack(const std::filesystem::path& filePath) { + m_FilePath = filePath; + + if (!std::filesystem::exists(filePath)) { + return; + } + + m_FileStream = std::ifstream(filePath, std::ios::in | std::ios::binary); + + m_FileStream.read(m_Version, 7); + + m_FileStream.seekg(-8, std::ios::end); // move file pointer to 8 bytes before the end (location of the address of the record count) + + uint32_t recordCountPos = 0; + BinaryIO::BinaryRead(m_FileStream, recordCountPos); + + m_FileStream.seekg(recordCountPos, std::ios::beg); + + BinaryIO::BinaryRead(m_FileStream, m_RecordCount); + + for (int i = 0; i < m_RecordCount; i++) { + PackRecord record; + BinaryIO::BinaryRead(m_FileStream, record); + + m_Records.push_back(record); + } + + m_FileStream.close(); +} + +bool Pack::HasFile(uint32_t crc) { + for (const auto& record : m_Records) { + if (record.m_Crc == crc) { + return true; + } + } + + return false; +} + +bool Pack::ReadFileFromPack(uint32_t crc, char** data, uint32_t* len) { + // Time for some wacky C file reading for speed reasons + + PackRecord pkRecord{}; + + for (const auto& record : m_Records) { + if (record.m_Crc == crc) { + pkRecord = record; + break; + } + } + + if (pkRecord.m_Crc == 0) return false; + + size_t pos = 0; + pos += pkRecord.m_FilePointer; + + bool isCompressed = (pkRecord.m_IsCompressed & 0xff) > 0; + auto inPackSize = isCompressed ? pkRecord.m_CompressedSize : pkRecord.m_UncompressedSize; + + FILE* file; +#ifdef _WIN32 + fopen_s(&file, m_FilePath.string().c_str(), "rb"); +#elif __APPLE__ + // macOS has 64bit file IO by default + file = fopen(m_FilePath.string().c_str(), "rb"); +#else + file = fopen64(m_FilePath.string().c_str(), "rb"); +#endif + + fseek(file, pos, SEEK_SET); + + if (!isCompressed) { + char* tempData = (char*)malloc(pkRecord.m_UncompressedSize); + fread(tempData, sizeof(uint8_t), pkRecord.m_UncompressedSize, file); + + *data = tempData; + *len = pkRecord.m_UncompressedSize; + fclose(file); + + return true; + } + + pos += 5; // skip header + + fseek(file, pos, SEEK_SET); + + char* decompressedData = (char*)malloc(pkRecord.m_UncompressedSize); + uint32_t currentReadPos = 0; + + while (true) { + if (currentReadPos >= pkRecord.m_UncompressedSize) break; + + uint32_t size; + fread(&size, sizeof(uint32_t), 1, file); + pos += 4; // Move pointer position 4 to the right + + char* chunk = (char*)malloc(size); + fread(chunk, sizeof(int8_t), size, file); + pos += size; // Move pointer position the amount of bytes read to the right + + int32_t err; + currentReadPos += ZCompression::Decompress((uint8_t*)chunk, size, reinterpret_cast(decompressedData + currentReadPos), ZCompression::MAX_SD0_CHUNK_SIZE, err); + + free(chunk); + } + + *data = decompressedData; + *len = pkRecord.m_UncompressedSize; + + fclose(file); + + return true; +} diff --git a/dCommon/dClient/Pack.h b/dCommon/dClient/Pack.h new file mode 100644 index 00000000..3e95b00a --- /dev/null +++ b/dCommon/dClient/Pack.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include + +#pragma pack(push, 1) +struct PackRecord { + uint32_t m_Crc; + int32_t m_LowerCrc; + int32_t m_UpperCrc; + uint32_t m_UncompressedSize; + char m_UncompressedHash[32]; + uint32_t m_Padding1; + uint32_t m_CompressedSize; + char m_CompressedHash[32]; + uint32_t m_Padding2; + uint32_t m_FilePointer; + uint32_t m_IsCompressed; // u32 bool +}; +#pragma pack(pop) + +class Pack { +public: + Pack(const std::filesystem::path& filePath); + ~Pack() = default; + + bool HasFile(uint32_t crc); + bool ReadFileFromPack(uint32_t crc, char** data, uint32_t* len); +private: + std::ifstream m_FileStream; + std::filesystem::path m_FilePath; + + char m_Version[7]; + + uint32_t m_RecordCount; + std::vector m_Records; +}; diff --git a/dCommon/dClient/PackIndex.cpp b/dCommon/dClient/PackIndex.cpp new file mode 100644 index 00000000..49ce61d1 --- /dev/null +++ b/dCommon/dClient/PackIndex.cpp @@ -0,0 +1,50 @@ +#include "PackIndex.h" + + +PackIndex::PackIndex(const std::filesystem::path& filePath) { + m_FileStream = std::ifstream(filePath / "versions" / "primary.pki", std::ios::in | std::ios::binary); + + BinaryIO::BinaryRead(m_FileStream, m_Version); + BinaryIO::BinaryRead(m_FileStream, m_PackPathCount); + + for (int i = 0; i < m_PackPathCount; i++) { + uint32_t stringLen = 0; + BinaryIO::BinaryRead(m_FileStream, stringLen); + + std::string path; + + for (int j = 0; j < stringLen; j++) { + char inChar; + BinaryIO::BinaryRead(m_FileStream, inChar); + + path += inChar; + } + + m_PackPaths.push_back(path); + } + + BinaryIO::BinaryRead(m_FileStream, m_PackFileIndexCount); + + for (int i = 0; i < m_PackFileIndexCount; i++) { + PackFileIndex packFileIndex; + BinaryIO::BinaryRead(m_FileStream, packFileIndex); + + m_PackFileIndices.push_back(packFileIndex); + } + + Game::logger->Log("PackIndex", "Loaded pack catalog with %i pack files and %i files", m_PackPaths.size(), m_PackFileIndices.size()); + + for (const auto& item : m_PackPaths) { + auto* pack = new Pack(filePath / item); + + m_Packs.push_back(pack); + } + + m_FileStream.close(); +} + +PackIndex::~PackIndex() { + for (const auto* item : m_Packs) { + delete item; + } +} diff --git a/dCommon/dClient/PackIndex.h b/dCommon/dClient/PackIndex.h new file mode 100644 index 00000000..bf10b809 --- /dev/null +++ b/dCommon/dClient/PackIndex.h @@ -0,0 +1,40 @@ +#pragma once + +#include + +#include +#include +#include + +#include "Pack.h" + +#pragma pack(push, 1) +struct PackFileIndex { + uint32_t m_Crc; + int32_t m_LowerCrc; + int32_t m_UpperCrc; + uint32_t m_PackFileIndex; + uint32_t m_IsCompressed; // u32 bool? +}; +#pragma pack(pop) + +class PackIndex { +public: + PackIndex(const std::filesystem::path& filePath); + ~PackIndex(); + + const std::vector& GetPackPaths() { return m_PackPaths; } + const std::vector& GetPackFileIndices() { return m_PackFileIndices; } + const std::vector& GetPacks() { return m_Packs; } +private: + std::ifstream m_FileStream; + + uint32_t m_Version; + + uint32_t m_PackPathCount; + std::vector m_PackPaths; + uint32_t m_PackFileIndexCount; + std::vector m_PackFileIndices; + + std::vector m_Packs; +}; diff --git a/dGame/dInventory/Item.cpp b/dGame/dInventory/Item.cpp index 4d4f2686..655af84e 100644 --- a/dGame/dInventory/Item.cpp +++ b/dGame/dInventory/Item.cpp @@ -13,6 +13,7 @@ #include "PossessableComponent.h" #include "CharacterComponent.h" #include "eItemType.h" +#include "AssetManager.h" class Inventory; @@ -340,18 +341,23 @@ void Item::DisassembleModel() { std::string renderAsset = result.fieldIsNull(0) ? "" : std::string(result.getStringField(0)); std::vector renderAssetSplit = GeneralUtils::SplitString(renderAsset, '\\'); - std::string lxfmlPath = "res/BrickModels/" + GeneralUtils::SplitString(renderAssetSplit.back(), '.')[0] + ".lxfml"; - std::ifstream file(lxfmlPath); + std::string lxfmlPath = "BrickModels/" + GeneralUtils::SplitString(renderAssetSplit.back(), '.').at(0) + ".lxfml"; + auto buffer = Game::assetManager->GetFileAsBuffer(lxfmlPath.c_str()); + + std::istream file(&buffer); result.finalize(); if (!file.good()) { + buffer.close(); return; } std::stringstream data; data << file.rdbuf(); + buffer.close(); + if (data.str().empty()) { return; } diff --git a/dGame/dUtilities/BrickDatabase.cpp b/dGame/dUtilities/BrickDatabase.cpp index 7ff0febb..4e873278 100644 --- a/dGame/dUtilities/BrickDatabase.cpp +++ b/dGame/dUtilities/BrickDatabase.cpp @@ -3,6 +3,7 @@ #include "BrickDatabase.h" #include "Game.h" +#include "AssetManager.h" std::vector BrickDatabase::emptyCache{}; BrickDatabase* BrickDatabase::m_Address = nullptr; @@ -17,7 +18,8 @@ std::vector& BrickDatabase::GetBricks(const std::string& lxfmlPath) { return cached->second; } - std::ifstream file(lxfmlPath); + AssetMemoryBuffer buffer = Game::assetManager->GetFileAsBuffer(("client/" + lxfmlPath).c_str()); + std::istream file(&buffer); if (!file.good()) { return emptyCache; } @@ -25,9 +27,12 @@ std::vector& BrickDatabase::GetBricks(const std::string& lxfmlPath) { std::stringstream data; data << file.rdbuf(); if (data.str().empty()) { + buffer.close(); return emptyCache; } + buffer.close(); + auto* doc = new tinyxml2::XMLDocument(); if (doc->Parse(data.str().c_str(), data.str().size()) != 0) { delete doc; diff --git a/dGame/dUtilities/SlashCommandHandler.cpp b/dGame/dUtilities/SlashCommandHandler.cpp index c6f1da9f..e8f1659e 100644 --- a/dGame/dUtilities/SlashCommandHandler.cpp +++ b/dGame/dUtilities/SlashCommandHandler.cpp @@ -63,6 +63,7 @@ #include "GameConfig.h" #include "ScriptedActivityComponent.h" #include "LevelProgressionComponent.h" +#include "AssetManager.h" void SlashCommandHandler::HandleChatCommand(const std::u16string& command, Entity* entity, const SystemAddress& sysAddr) { std::string chatCommand; @@ -582,7 +583,8 @@ void SlashCommandHandler::HandleChatCommand(const std::u16string& command, Entit if (args[0].find("/") != std::string::npos) return; if (args[0].find("\\") != std::string::npos) return; - std::ifstream infile("./res/macros/" + args[0] + ".scm"); + auto buf = Game::assetManager->GetFileAsBuffer(("macros/" + args[0] + ".scm").c_str()); + std::istream infile(&buf); if (infile.good()) { std::string line; @@ -593,6 +595,8 @@ void SlashCommandHandler::HandleChatCommand(const std::u16string& command, Entit ChatPackets::SendSystemMessage(sysAddr, u"Unknown macro! Is the filename right?"); } + buf.close(); + return; } @@ -1904,10 +1908,7 @@ bool SlashCommandHandler::CheckIfAccessibleZone(const unsigned int zoneID) { CDZoneTableTable* zoneTable = CDClientManager::Instance()->GetTable("ZoneTable"); const CDZoneTable* zone = zoneTable->Query(zoneID); if (zone != nullptr) { - std::string zonePath = "./res/maps/" + zone->zoneName; - std::transform(zonePath.begin(), zonePath.end(), zonePath.begin(), ::tolower); - std::ifstream f(zonePath.c_str()); - return f.good(); + return Game::assetManager->HasFile(("maps/" + zone->zoneName).c_str()); } else { return false; } diff --git a/dMasterServer/MasterServer.cpp b/dMasterServer/MasterServer.cpp index cd54913a..5aac3743 100644 --- a/dMasterServer/MasterServer.cpp +++ b/dMasterServer/MasterServer.cpp @@ -25,6 +25,7 @@ #include "dConfig.h" #include "dLogger.h" #include "dServer.h" +#include "AssetManager.h" //RakNet includes: #include "RakNetDefines.h" @@ -44,6 +45,7 @@ namespace Game { dServer* server; InstanceManager* im; dConfig* config; + AssetManager* assetManager; } //namespace Game bool shutdownSequenceStarted = false; @@ -99,44 +101,49 @@ int main(int argc, char** argv) { return EXIT_FAILURE; } + try { + std::string client_path = config.GetValue("client_location"); + if (client_path.empty()) client_path = "./res"; + Game::assetManager = new AssetManager(config.GetValue("client_location")); + } catch (std::runtime_error& ex) { + Game::logger->Log("MasterServer", "Got an error while setting up assets: %s", ex.what()); + + return EXIT_FAILURE; + } + MigrationRunner::RunMigrations(); - //Check CDClient exists - const std::string cdclient_path = "./res/CDServer.sqlite"; - std::ifstream cdclient_fd(cdclient_path); - if (!cdclient_fd.good()) { - Game::logger->Log("WorldServer", "%s could not be opened. Looking for cdclient.fdb to convert to sqlite.", cdclient_path.c_str()); - cdclient_fd.close(); + // Check CDClient exists + if (!std::filesystem::exists(Game::assetManager->GetResPath() / "CDServer.sqlite")) { + Game::logger->Log("WorldServer", "CDServer.sqlite could not be opened. Looking for cdclient.fdb to convert to sqlite."); - const std::string cdclientFdbPath = "./res/cdclient.fdb"; - cdclient_fd.open(cdclientFdbPath); - if (!cdclient_fd.good()) { - Game::logger->Log( - "WorldServer", "%s could not be opened." - "Please move a cdclient.fdb or an already converted database to build/res.", cdclientFdbPath.c_str()); + if (!std::filesystem::exists(Game::assetManager->GetResPath() / "cdclient.fdb")) { + Game::logger->Log("WorldServer", "cdclient.fdb could not be opened. Please move a cdclient.fdb or an already converted database to build/res."); return EXIT_FAILURE; } - Game::logger->Log("WorldServer", "Found %s. Clearing cdserver migration_history then copying and converting to sqlite.", cdclientFdbPath.c_str()); + + Game::logger->Log("WorldServer", "Found cdclient.fdb. Clearing cdserver migration_history then copying and converting to sqlite."); auto stmt = Database::CreatePreppedStmt(R"#(DELETE FROM migration_history WHERE name LIKE "%cdserver%";)#"); stmt->executeUpdate(); delete stmt; - cdclient_fd.close(); - std::string res = "python3 ../thirdparty/docker-utils/utils/fdb_to_sqlite.py " + cdclientFdbPath; - int r = system(res.c_str()); - if (r != 0) { + std::string res = "python3 ../thirdparty/docker-utils/utils/fdb_to_sqlite.py " + (Game::assetManager->GetResPath() / "cdclient.fdb").string(); + + int result = system(res.c_str()); + if (result != 0) { Game::logger->Log("MasterServer", "Failed to convert fdb to sqlite"); return EXIT_FAILURE; } - if (std::rename("./cdclient.sqlite", "./res/CDServer.sqlite") != 0) { - Game::logger->Log("MasterServer", "failed to move cdclient file."); + + if (std::rename("./cdclient.sqlite", (Game::assetManager->GetResPath() / "CDServer.sqlite").string().c_str()) != 0) { + Game::logger->Log("MasterServer", "Failed to move cdclient file."); return EXIT_FAILURE; } } //Connect to CDClient try { - CDClientDatabase::Connect(cdclient_path); + CDClientDatabase::Connect((Game::assetManager->GetResPath() / "CDServer.sqlite").string()); } catch (CppSQLite3Exception& e) { Game::logger->Log("WorldServer", "Unable to connect to CDServer SQLite Database"); Game::logger->Log("WorldServer", "Error: %s", e.errorMessage()); @@ -152,7 +159,7 @@ int main(int argc, char** argv) { CDClientManager::Instance()->Initialize(); } catch (CppSQLite3Exception& e) { Game::logger->Log("WorldServer", "Failed to initialize CDServer SQLite Database"); - Game::logger->Log("WorldServer", "May be caused by corrupted file: %s", cdclient_path.c_str()); + Game::logger->Log("WorldServer", "May be caused by corrupted file: %s", (Game::assetManager->GetResPath() / "CDServer.sqlite").string().c_str()); Game::logger->Log("WorldServer", "Error: %s", e.errorMessage()); Game::logger->Log("WorldServer", "Error Code: %i", e.errorCode()); return EXIT_FAILURE; diff --git a/dNavigation/dNavMesh.cpp b/dNavigation/dNavMesh.cpp index b3c8a229..e5ba0129 100644 --- a/dNavigation/dNavMesh.cpp +++ b/dNavigation/dNavMesh.cpp @@ -43,7 +43,7 @@ dNavMesh::~dNavMesh() { void dNavMesh::LoadNavmesh() { - std::string path = "./res/maps/navmeshes/" + std::to_string(m_ZoneId) + ".bin"; + std::string path = "./navmeshes/" + std::to_string(m_ZoneId) + ".bin"; if (!BinaryIO::DoesFileExist(path)) { return; diff --git a/dWorldServer/WorldServer.cpp b/dWorldServer/WorldServer.cpp index 89243d59..23761b2b 100644 --- a/dWorldServer/WorldServer.cpp +++ b/dWorldServer/WorldServer.cpp @@ -55,6 +55,7 @@ #include "MasterPackets.h" #include "Player.h" #include "PropertyManagementComponent.h" +#include "AssetManager.h" #include "ZCompression.h" @@ -68,6 +69,8 @@ namespace Game { dLocale* locale; std::mt19937 randomEngine; + AssetManager* assetManager; + RakPeerInterface* chatServer; SystemAddress chatSysAddr; } @@ -142,9 +145,19 @@ int main(int argc, char** argv) { Game::logger->SetLogDebugStatements(config.GetValue("log_debug_statements") == "1"); if (config.GetValue("disable_chat") == "1") chatDisabled = true; + try { + std::string client_path = config.GetValue("client_location"); + if (client_path.empty()) client_path = "./res"; + Game::assetManager = new AssetManager(config.GetValue("client_location")); + } catch (std::runtime_error& ex) { + Game::logger->Log("WorldServer", "Got an error while setting up assets: %s", ex.what()); + + return EXIT_FAILURE; + } + // Connect to CDClient try { - CDClientDatabase::Connect("./res/CDServer.sqlite"); + CDClientDatabase::Connect((Game::assetManager->GetResPath() / "CDServer.sqlite").string()); } catch (CppSQLite3Exception& e) { Game::logger->Log("WorldServer", "Unable to connect to CDServer SQLite Database"); Game::logger->Log("WorldServer", "Error: %s", e.errorMessage()); @@ -189,7 +202,7 @@ int main(int argc, char** argv) { ObjectIDManager::Instance()->Initialize(); UserManager::Instance()->Initialize(); LootGenerator::Instance(); - Game::chatFilter = new dChatFilter("./res/chatplus_en_us", bool(std::stoi(config.GetValue("dont_generate_dcf")))); + Game::chatFilter = new dChatFilter(Game::assetManager->GetResPath().string() + "/chatplus_en_us", bool(std::stoi(config.GetValue("dont_generate_dcf")))); Game::server = new dServer(masterIP, ourPort, instanceID, maxClients, false, true, Game::logger, masterIP, masterPort, ServerType::World, zoneID); @@ -243,14 +256,14 @@ int main(int argc, char** argv) { std::ifstream fileStream; static const std::vector aliases = { - "res/CDServers.fdb", - "res/cdserver.fdb", - "res/CDClient.fdb", - "res/cdclient.fdb", + "CDServers.fdb", + "cdserver.fdb", + "CDClient.fdb", + "cdclient.fdb", }; for (const auto& file : aliases) { - fileStream.open(file, std::ios::binary | std::ios::in); + fileStream.open(Game::assetManager->GetResPath() / file, std::ios::binary | std::ios::in); if (fileStream.is_open()) { break; } diff --git a/dZoneManager/Level.cpp b/dZoneManager/Level.cpp index f12adc56..dd38d208 100644 --- a/dZoneManager/Level.cpp +++ b/dZoneManager/Level.cpp @@ -13,17 +13,22 @@ #include "EntityManager.h" #include "CDFeatureGatingTable.h" #include "CDClientManager.h" +#include "AssetManager.h" Level::Level(Zone* parentZone, const std::string& filepath) { m_ParentZone = parentZone; - std::ifstream file(filepath, std::ios_base::in | std::ios_base::binary); - if (file) { - ReadChunks(file); - } else { + + auto buffer = Game::assetManager->GetFileAsBuffer(filepath.c_str()); + + if (!buffer.m_Success) { Game::logger->Log("Level", "Failed to load %s", filepath.c_str()); + return; } - file.close(); + std::istream file(&buffer); + ReadChunks(file); + + buffer.close(); } Level::~Level() { @@ -41,7 +46,7 @@ const void Level::PrintAllObjects() { } } -void Level::ReadChunks(std::ifstream& file) { +void Level::ReadChunks(std::istream& file) { const uint32_t CHNK_HEADER = ('C' + ('H' << 8) + ('N' << 16) + ('K' << 24)); while (!file.eof()) { @@ -139,7 +144,7 @@ void Level::ReadChunks(std::ifstream& file) { } } -void Level::ReadFileInfoChunk(std::ifstream& file, Header& header) { +void Level::ReadFileInfoChunk(std::istream& file, Header& header) { FileInfoChunk* fi = new FileInfoChunk; BinaryIO::BinaryRead(file, fi->version); BinaryIO::BinaryRead(file, fi->revision); @@ -152,7 +157,7 @@ void Level::ReadFileInfoChunk(std::ifstream& file, Header& header) { if (header.fileInfo->revision == 3452816845 && m_ParentZone->GetZoneID().GetMapID() == 1100) header.fileInfo->revision = 26; } -void Level::ReadSceneObjectDataChunk(std::ifstream& file, Header& header) { +void Level::ReadSceneObjectDataChunk(std::istream& file, Header& header) { SceneObjectDataChunk* chunk = new SceneObjectDataChunk; uint32_t objectsCount = 0; BinaryIO::BinaryRead(file, objectsCount); diff --git a/dZoneManager/Level.h b/dZoneManager/Level.h index e724363f..83daeedb 100644 --- a/dZoneManager/Level.h +++ b/dZoneManager/Level.h @@ -67,7 +67,7 @@ private: Zone* m_ParentZone; //private functions: - void ReadChunks(std::ifstream& file); - void ReadFileInfoChunk(std::ifstream& file, Header& header); - void ReadSceneObjectDataChunk(std::ifstream& file, Header& header); + void ReadChunks(std::istream& file); + void ReadFileInfoChunk(std::istream& file, Header& header); + void ReadSceneObjectDataChunk(std::istream& file, Header& header); }; diff --git a/dZoneManager/Zone.cpp b/dZoneManager/Zone.cpp index b670849b..1fe47454 100644 --- a/dZoneManager/Zone.cpp +++ b/dZoneManager/Zone.cpp @@ -7,6 +7,7 @@ #include "GeneralUtils.h" #include "BinaryIO.h" +#include "AssetManager.h" #include "CDClientManager.h" #include "CDZoneTableTable.h" #include "Spawner.h" @@ -40,7 +41,8 @@ void Zone::LoadZoneIntoMemory() { m_ZonePath = m_ZoneFilePath.substr(0, m_ZoneFilePath.rfind('/') + 1); if (m_ZoneFilePath == "ERR") return; - std::ifstream file(m_ZoneFilePath, std::ios::binary); + AssetMemoryBuffer buffer = Game::assetManager->GetFileAsBuffer(m_ZoneFilePath.c_str()); + std::istream file(&buffer); if (file) { BinaryIO::BinaryRead(file, m_ZoneFileFormatVersion); @@ -144,17 +146,13 @@ void Zone::LoadZoneIntoMemory() { } } - - - //m_PathData.resize(m_PathDataLength); - //file.read((char*)&m_PathData[0], m_PathDataLength); } } else { Game::logger->Log("Zone", "Failed to open: %s", m_ZoneFilePath.c_str()); } m_ZonePath = m_ZoneFilePath.substr(0, m_ZoneFilePath.rfind('/') + 1); - file.close(); + buffer.close(); } std::string Zone::GetFilePathForZoneID() { @@ -162,7 +160,7 @@ std::string Zone::GetFilePathForZoneID() { CDZoneTableTable* zoneTable = CDClientManager::Instance()->GetTable("ZoneTable"); const CDZoneTable* zone = zoneTable->Query(this->GetZoneID().GetMapID()); if (zone != nullptr) { - std::string toReturn = "./res/maps/" + zone->zoneName; + std::string toReturn = "maps/" + zone->zoneName; std::transform(toReturn.begin(), toReturn.end(), toReturn.begin(), ::tolower); return toReturn; } @@ -222,7 +220,7 @@ const void Zone::PrintAllGameObjects() { } } -void Zone::LoadScene(std::ifstream& file) { +void Zone::LoadScene(std::istream& file) { SceneRef scene; scene.level = nullptr; LWOSCENEID lwoSceneID(LWOZONEID_INVALID, 0); @@ -264,10 +262,17 @@ void Zone::LoadScene(std::ifstream& file) { std::vector Zone::LoadLUTriggers(std::string triggerFile, LWOSCENEID sceneID) { std::vector lvlTriggers; - std::ifstream file(m_ZonePath + triggerFile); + + auto buffer = Game::assetManager->GetFileAsBuffer((m_ZonePath + triggerFile).c_str()); + + if (!buffer.m_Success) return lvlTriggers; + + std::istream file(&buffer); std::stringstream data; data << file.rdbuf(); + buffer.close(); + if (data.str().size() == 0) return lvlTriggers; tinyxml2::XMLDocument* doc = new tinyxml2::XMLDocument(); @@ -336,7 +341,7 @@ const Path* Zone::GetPath(std::string name) const { return nullptr; } -void Zone::LoadSceneTransition(std::ifstream& file) { +void Zone::LoadSceneTransition(std::istream& file) { SceneTransition sceneTrans; if (m_ZoneFileFormatVersion < Zone::ZoneFileFormatVersion::Auramar) { uint8_t length; @@ -355,14 +360,14 @@ void Zone::LoadSceneTransition(std::ifstream& file) { m_SceneTransitions.push_back(sceneTrans); } -SceneTransitionInfo Zone::LoadSceneTransitionInfo(std::ifstream& file) { +SceneTransitionInfo Zone::LoadSceneTransitionInfo(std::istream& file) { SceneTransitionInfo info; BinaryIO::BinaryRead(file, info.sceneID); BinaryIO::BinaryRead(file, info.position); return info; } -void Zone::LoadPath(std::ifstream& file) { +void Zone::LoadPath(std::istream& file) { // Currently only spawner (type 4) paths are supported Path path = Path(); diff --git a/dZoneManager/Zone.h b/dZoneManager/Zone.h index 50530273..f041b616 100644 --- a/dZoneManager/Zone.h +++ b/dZoneManager/Zone.h @@ -225,9 +225,9 @@ private: std::map m_MapRevisions; //rhs is the revision! //private ("helper") functions: - void LoadScene(std::ifstream& file); + void LoadScene(std::istream& file); std::vector LoadLUTriggers(std::string triggerFile, LWOSCENEID sceneID); - void LoadSceneTransition(std::ifstream& file); - SceneTransitionInfo LoadSceneTransitionInfo(std::ifstream& file); - void LoadPath(std::ifstream& file); + void LoadSceneTransition(std::istream& file); + SceneTransitionInfo LoadSceneTransitionInfo(std::istream& file); + void LoadPath(std::istream& file); }; diff --git a/resources/sharedconfig.ini b/resources/sharedconfig.ini index 1a377d28..847a6b7c 100644 --- a/resources/sharedconfig.ini +++ b/resources/sharedconfig.ini @@ -21,3 +21,7 @@ max_clients=999 # Where to put crashlogs dump_folder= + +# The location of the client +# Either the folder with /res or with /client and /versions +client_location=