From e58218cfbcc6b5a616dac59174f653aecf1ff2df Mon Sep 17 00:00:00 2001 From: David Markowitz <39972741+EmosewaMC@users.noreply.github.com> Date: Sat, 23 Dec 2023 09:24:16 -0800 Subject: [PATCH] chore: Speed up and cleanup level and zone loading; Add safer asset buffer reading (#1314) * Remove std::couts littered throughout the base * working End of optimizations for now going faster * Remove extraneous compare function std::less already does this in a map. * gaming * Update Zone.cpp * dlu is moving to bitbucket again * Update Level.cpp --------- Co-authored-by: Jettford --- dCommon/BinaryIO.cpp | 41 --- dCommon/BinaryIO.h | 53 +++- dCommon/FdbToSqlite.cpp | 8 +- dCommon/FdbToSqlite.h | 4 +- dCommon/dClient/AssetManager.cpp | 7 +- dCommon/dClient/AssetManager.h | 18 +- dCommon/dClient/PackIndex.cpp | 19 +- dGame/UserManager.cpp | 30 +- dGame/dInventory/Item.cpp | 13 +- dGame/dPropertyBehaviors/ControlBehaviors.cpp | 13 +- dGame/dUtilities/BrickDatabase.cpp | 12 +- dGame/dUtilities/SlashCommandHandler.cpp | 8 +- dMasterServer/MasterServer.cpp | 7 +- dZoneManager/Level.cpp | 151 +++------ dZoneManager/Level.h | 14 +- dZoneManager/Zone.cpp | 294 ++++++------------ dZoneManager/Zone.h | 9 +- dZoneManager/dZMCommon.h | 4 - dZoneManager/dZoneManager.cpp | 27 -- dZoneManager/dZoneManager.h | 2 - 20 files changed, 247 insertions(+), 487 deletions(-) diff --git a/dCommon/BinaryIO.cpp b/dCommon/BinaryIO.cpp index 22e4de60..b64fb3ba 100644 --- a/dCommon/BinaryIO.cpp +++ b/dCommon/BinaryIO.cpp @@ -1,14 +1,6 @@ #include "BinaryIO.h" #include -void BinaryIO::WriteString(const std::string& stringToWrite, std::ofstream& outstream) { - //BinaryWrite(outstream, uint32_t(stringToWrite.length())); - - for (size_t i = 0; i < size_t(stringToWrite.length()); ++i) { - BinaryIO::BinaryWrite(outstream, stringToWrite[i]); - } -} - //For reading null-terminated strings std::string BinaryIO::ReadString(std::istream& instream) { std::string toReturn; @@ -23,36 +15,3 @@ std::string BinaryIO::ReadString(std::istream& instream) { return toReturn; } - -//For reading strings of a specific size -std::string BinaryIO::ReadString(std::istream& instream, size_t size) { - std::string toReturn; - char buffer; - - for (size_t i = 0; i < size; ++i) { - BinaryIO::BinaryRead(instream, buffer); - toReturn += buffer; - } - - return toReturn; -} - -std::string BinaryIO::ReadWString(std::istream& instream) { - size_t size; - BinaryRead(instream, size); - //toReturn.resize(size); - std::string test; - unsigned char buf; - - for (size_t i = 0; i < size; ++i) { - //instream.ignore(1); - BinaryRead(instream, buf); - test += buf; - } - - //printf("%s\n", test.c_str()); - - //instream.read((char*)&toReturn[0], size * 2); - //std::string str(toReturn.begin(), toReturn.end()); - return test; -} diff --git a/dCommon/BinaryIO.h b/dCommon/BinaryIO.h index a117ad0d..d1e728ca 100644 --- a/dCommon/BinaryIO.h +++ b/dCommon/BinaryIO.h @@ -1,8 +1,17 @@ #pragma once + +#ifndef __BINARYIO__H__ +#define __BINARYIO__H__ + #include #include +#include + +#include "Game.h" +#include "Logger.h" namespace BinaryIO { + template std::ostream& BinaryWrite(std::ostream& stream, const T& value) { return stream.write(reinterpret_cast(&value), sizeof(T)); @@ -15,13 +24,51 @@ namespace BinaryIO { return stream.read(reinterpret_cast(&value), sizeof(T)); } - void WriteString(const std::string& stringToWrite, std::ofstream& outstream); + enum class ReadType : int8_t { + WideString = 0, + String = 1, + }; + + template + inline void ReadString(std::istream& stream, std::u16string& value) { + static_assert(std::is_integral::value, "SizeType must be an integral type."); + + SizeType size; + BinaryRead(stream, size); + + if (!stream.good()) throw std::runtime_error("Failed to read from istream."); + value.resize(size); + stream.read(reinterpret_cast(value.data()), size * sizeof(uint16_t)); + } + + template + inline void ReadString(std::istream& stream, std::string& value, ReadType readType) { + static_assert(std::is_integral::value, "SizeType must be an integral type."); + + SizeType size; + BinaryRead(stream, size); + + if (!stream.good()) throw std::runtime_error("Failed to read from istream."); + value.resize(size); + if (readType == ReadType::WideString) { + uint16_t wideChar; + + // Faster to do this than to read a u16string and convert it to a string since we only go through allocator once + for (SizeType i = 0; i < size; ++i) { + BinaryRead(stream, wideChar); + value[i] = static_cast(wideChar); + } + } else { + stream.read(value.data(), size); + } + } + std::string ReadString(std::istream& instream); - std::string ReadString(std::istream& instream, size_t size); - std::string ReadWString(std::istream& instream); inline bool DoesFileExist(const std::string& name) { std::ifstream f(name.c_str()); return f.good(); } } + +#endif //!__BINARYIO__H__ diff --git a/dCommon/FdbToSqlite.cpp b/dCommon/FdbToSqlite.cpp index 0b2c546c..d8409cd5 100644 --- a/dCommon/FdbToSqlite.cpp +++ b/dCommon/FdbToSqlite.cpp @@ -28,19 +28,17 @@ FdbToSqlite::Convert::Convert(std::string binaryOutPath) { this->m_BinaryOutPath = binaryOutPath; } -bool FdbToSqlite::Convert::ConvertDatabase(AssetMemoryBuffer& buffer) { +bool FdbToSqlite::Convert::ConvertDatabase(AssetStream& buffer) { if (m_ConversionStarted) return false; - std::istream cdClientBuffer(&buffer); - this->m_ConversionStarted = true; try { CDClientDatabase::Connect(m_BinaryOutPath + "/CDServer.sqlite"); CDClientDatabase::ExecuteQuery("BEGIN TRANSACTION;"); - int32_t numberOfTables = ReadInt32(cdClientBuffer); - ReadTables(numberOfTables, cdClientBuffer); + int32_t numberOfTables = ReadInt32(buffer); + ReadTables(numberOfTables, buffer); CDClientDatabase::ExecuteQuery("COMMIT;"); } catch (CppSQLite3Exception& e) { diff --git a/dCommon/FdbToSqlite.h b/dCommon/FdbToSqlite.h index 7aad2703..b0d20aee 100644 --- a/dCommon/FdbToSqlite.h +++ b/dCommon/FdbToSqlite.h @@ -7,7 +7,7 @@ #include #include -class AssetMemoryBuffer; +#include "AssetManager.h" enum class eSqliteDataType : int32_t; @@ -27,7 +27,7 @@ namespace FdbToSqlite { * * @return true if the database was converted properly, false otherwise. */ - bool ConvertDatabase(AssetMemoryBuffer& buffer); + bool ConvertDatabase(AssetStream& buffer); /** * @brief Reads a 32 bit int from the fdb file. diff --git a/dCommon/dClient/AssetManager.cpp b/dCommon/dClient/AssetManager.cpp index c672b335..00bae6ce 100644 --- a/dCommon/dClient/AssetManager.cpp +++ b/dCommon/dClient/AssetManager.cpp @@ -152,13 +152,12 @@ bool AssetManager::GetFile(const char* name, char** data, uint32_t* len) { return success; } -AssetMemoryBuffer AssetManager::GetFileAsBuffer(const char* name) { - char* buf; - uint32_t len; +AssetStream AssetManager::GetFile(const char* name) { + char* buf; uint32_t len; bool success = this->GetFile(name, &buf, &len); - return AssetMemoryBuffer(buf, len, success); + return AssetStream(buf, len, success); } uint32_t AssetManager::crc32b(uint32_t base, uint8_t* message, size_t l) { diff --git a/dCommon/dClient/AssetManager.h b/dCommon/dClient/AssetManager.h index d2543489..7e36b4b5 100644 --- a/dCommon/dClient/AssetManager.h +++ b/dCommon/dClient/AssetManager.h @@ -25,6 +25,10 @@ struct AssetMemoryBuffer : std::streambuf { this->setg(base, base, base + n); } + ~AssetMemoryBuffer() { + free(m_Base); + } + pos_type seekpos(pos_type sp, std::ios_base::openmode which) override { return seekoff(sp - pos_type(off_type(0)), std::ios_base::beg, which); } @@ -40,9 +44,17 @@ struct AssetMemoryBuffer : std::streambuf { setg(eback(), eback() + off, egptr()); return gptr() - eback(); } +}; - void close() { - free(m_Base); +struct AssetStream : std::istream { + AssetStream(char* base, std::ptrdiff_t n, bool success) : std::istream(new AssetMemoryBuffer(base, n, success)) {} + + ~AssetStream() { + delete rdbuf(); + } + + operator bool() { + return reinterpret_cast(rdbuf())->m_Success; } }; @@ -56,7 +68,7 @@ public: bool HasFile(const char* name); bool GetFile(const char* name, char** data, uint32_t* len); - AssetMemoryBuffer GetFileAsBuffer(const char* name); + AssetStream GetFile(const char* name); private: void LoadPackIndex(); diff --git a/dCommon/dClient/PackIndex.cpp b/dCommon/dClient/PackIndex.cpp index 55c26b48..0d7a7fe9 100644 --- a/dCommon/dClient/PackIndex.cpp +++ b/dCommon/dClient/PackIndex.cpp @@ -8,21 +8,10 @@ PackIndex::PackIndex(const std::filesystem::path& filePath) { BinaryIO::BinaryRead(m_FileStream, m_Version); BinaryIO::BinaryRead(m_FileStream, m_PackPathCount); - - for (int i = 0; i < m_PackPathCount; i++) { - uint32_t stringLen = 0; - BinaryIO::BinaryRead(m_FileStream, stringLen); - - std::string path; - - for (int j = 0; j < stringLen; j++) { - char inChar; - BinaryIO::BinaryRead(m_FileStream, inChar); - - path += inChar; - } - - m_PackPaths.push_back(path); + + m_PackPaths.resize(m_PackPathCount); + for (auto& item : m_PackPaths) { + BinaryIO::ReadString(m_FileStream, item, BinaryIO::ReadType::String); } BinaryIO::BinaryRead(m_FileStream, m_PackFileIndexCount); diff --git a/dGame/UserManager.cpp b/dGame/UserManager.cpp index 323bf29a..18387366 100644 --- a/dGame/UserManager.cpp +++ b/dGame/UserManager.cpp @@ -44,57 +44,53 @@ inline void StripCR(std::string& str) { void UserManager::Initialize() { std::string line; - AssetMemoryBuffer fnBuff = Game::assetManager->GetFileAsBuffer("names/minifigname_first.txt"); - if (!fnBuff.m_Success) { + auto fnStream = Game::assetManager->GetFile("names/minifigname_first.txt"); + if (!fnStream) { LOG("Failed to load %s", (Game::assetManager->GetResPath() / "names/minifigname_first.txt").string().c_str()); throw std::runtime_error("Aborting initialization due to missing minifigure name file."); } - std::istream fnStream = std::istream(&fnBuff); + while (std::getline(fnStream, line, '\n')) { std::string name = line; StripCR(name); m_FirstNames.push_back(name); } - fnBuff.close(); - AssetMemoryBuffer mnBuff = Game::assetManager->GetFileAsBuffer("names/minifigname_middle.txt"); - if (!mnBuff.m_Success) { + auto mnStream = Game::assetManager->GetFile("names/minifigname_middle.txt"); + if (!mnStream) { LOG("Failed to load %s", (Game::assetManager->GetResPath() / "names/minifigname_middle.txt").string().c_str()); throw std::runtime_error("Aborting initialization due to missing minifigure name file."); } - std::istream mnStream = std::istream(&mnBuff); + while (std::getline(mnStream, line, '\n')) { std::string name = line; StripCR(name); m_MiddleNames.push_back(name); } - mnBuff.close(); - AssetMemoryBuffer lnBuff = Game::assetManager->GetFileAsBuffer("names/minifigname_last.txt"); - if (!lnBuff.m_Success) { + auto lnStream = Game::assetManager->GetFile("names/minifigname_last.txt"); + if (!lnStream) { LOG("Failed to load %s", (Game::assetManager->GetResPath() / "names/minifigname_last.txt").string().c_str()); throw std::runtime_error("Aborting initialization due to missing minifigure name file."); } - std::istream lnStream = std::istream(&lnBuff); + while (std::getline(lnStream, line, '\n')) { std::string name = line; StripCR(name); m_LastNames.push_back(name); } - lnBuff.close(); - //Load our pre-approved names: - AssetMemoryBuffer chatListBuff = Game::assetManager->GetFileAsBuffer("chatplus_en_us.txt"); - if (!chatListBuff.m_Success) { + // Load our pre-approved names: + auto chatListStream = Game::assetManager->GetFile("chatplus_en_us.txt"); + if (!chatListStream) { LOG("Failed to load %s", (Game::assetManager->GetResPath() / "chatplus_en_us.txt").string().c_str()); throw std::runtime_error("Aborting initialization due to missing chat whitelist file."); } - std::istream chatListStream = std::istream(&chatListBuff); + while (std::getline(chatListStream, line, '\n')) { StripCR(line); m_PreapprovedNames.push_back(line); } - chatListBuff.close(); } UserManager::~UserManager() { diff --git a/dGame/dInventory/Item.cpp b/dGame/dInventory/Item.cpp index 438bec2f..9e4b46f6 100644 --- a/dGame/dInventory/Item.cpp +++ b/dGame/dInventory/Item.cpp @@ -423,25 +423,16 @@ void Item::DisassembleModel(uint32_t numToDismantle) { if (renderAssetSplit.empty()) return; std::string lxfmlPath = "BrickModels" + lxfmlFolderName + "/" + GeneralUtils::SplitString(renderAssetSplit.back(), '.').at(0) + ".lxfml"; - auto buffer = Game::assetManager->GetFileAsBuffer(lxfmlPath.c_str()); + auto file = Game::assetManager->GetFile(lxfmlPath.c_str()); - if (!buffer.m_Success) { + if (!file) { LOG("Failed to load %s to disassemble model into bricks, check that this file exists", lxfmlPath.c_str()); return; } - std::istream file(&buffer); - - if (!file.good()) { - buffer.close(); - return; - } - std::stringstream data; data << file.rdbuf(); - buffer.close(); - uint32_t fileSize; file.seekg(0, std::ios::end); fileSize = static_cast(file.tellg()); diff --git a/dGame/dPropertyBehaviors/ControlBehaviors.cpp b/dGame/dPropertyBehaviors/ControlBehaviors.cpp index 3dbd777a..b143cbc8 100644 --- a/dGame/dPropertyBehaviors/ControlBehaviors.cpp +++ b/dGame/dPropertyBehaviors/ControlBehaviors.cpp @@ -324,14 +324,9 @@ void ControlBehaviors::ProcessCommand(Entity* modelEntity, const SystemAddress& } ControlBehaviors::ControlBehaviors() { - auto blocksDefStreamBuffer = Game::assetManager->GetFileAsBuffer("ui\\ingame\\blocksdef.xml"); - if (!blocksDefStreamBuffer.m_Success) { - LOG("failed to open blocksdef"); - return; - } - std::istream blocksBuffer(&blocksDefStreamBuffer); - if (!blocksBuffer.good()) { - LOG("Blocks buffer is not good!"); + auto blocksBuffer = Game::assetManager->GetFile("ui\\ingame\\blocksdef.xml"); + if (!blocksBuffer) { + LOG("Failed to open blocksdef.xml"); return; } @@ -342,7 +337,7 @@ ControlBehaviors::ControlBehaviors() { std::string buffer{}; bool commentBlockStart = false; while (std::getline(blocksBuffer, read)) { - // tinyxml2 should handle comment blocks but the client has one that fails the processing. + // tinyxml2 should handle comment blocks but the client has one that fails the processing. // This preprocessing just removes all comments from the read file out of an abundance of caution. if (read.find("