Add support for packed clients (#802)

* First iteration of pack reader and interface

* Fix memory leak and remove logs

* Complete packed asset interface and begin on file loading replacement

* Implement proper BinaryIO error

* Improve AssetMemoryBuffer for reading and implement more reading

* Repair more file loading code and improve how navmeshes are loaded

* Missing checks implementation

* Revert addition of Manifest class and migration changes

* Resolved all feedback.
This commit is contained in:
Jett 2022-11-01 18:21:26 +00:00 committed by GitHub
parent 971e0fb3b6
commit 4a6f3e44ee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 690 additions and 81 deletions

View File

@ -101,6 +101,17 @@ foreach(resource_file ${RESOURCE_FILES})
endif()
endforeach()
# Copy navmesh data on first build and extract it
if (NOT EXISTS ${PROJECT_BINARY_DIR}/navmeshes/)
configure_file(
${CMAKE_SOURCE_DIR}/resources/navmeshes.zip ${PROJECT_BINARY_DIR}/navmeshes.zip
COPYONLY
)
file(ARCHIVE_EXTRACT INPUT ${PROJECT_BINARY_DIR}/navmeshes.zip)
file(REMOVE ${PROJECT_BINARY_DIR}/navmeshes.zip)
endif()
# Copy vanity files on first build
set(VANITY_FILES "CREDITS.md" "INFO.md" "TESTAMENT.md" "NPC.xml")
foreach(file ${VANITY_FILES})
@ -145,6 +156,7 @@ set(INCLUDED_DIRECTORIES
"dGame/dEntity"
"dGame/dPropertyBehaviors"
"dGame/dUtilities"
"dCommon/dClient"
"dPhysics"
"dNavigation"
"dNavigation/dTerrain"

View File

@ -12,6 +12,7 @@
#include "dMessageIdentifiers.h"
#include "dChatFilter.h"
#include "Diagnostics.h"
#include "AssetManager.h"
#include "PlayerContainer.h"
#include "ChatPacketHandler.h"
@ -22,6 +23,7 @@ namespace Game {
dServer* server;
dConfig* config;
dChatFilter* chatFilter;
AssetManager* assetManager;
}
//RakNet includes:
@ -50,6 +52,16 @@ int main(int argc, char** argv) {
Game::logger->SetLogToConsole(bool(std::stoi(config.GetValue("log_to_console"))));
Game::logger->SetLogDebugStatements(config.GetValue("log_debug_statements") == "1");
try {
std::string client_path = config.GetValue("client_location");
if (client_path.empty()) client_path = "./res";
Game::assetManager = new AssetManager(config.GetValue("client_location"));
} catch (std::runtime_error& ex) {
Game::logger->Log("ChatServer", "Got an error while setting up assets: %s", ex.what());
return EXIT_FAILURE;
}
//Connect to the MySQL Database
std::string mysql_host = config.GetValue("mysql_host");
std::string mysql_database = config.GetValue("mysql_database");
@ -87,7 +99,7 @@ int main(int argc, char** argv) {
Game::server = new dServer(config.GetValue("external_ip"), ourPort, 0, maxClients, false, true, Game::logger, masterIP, masterPort, ServerType::Chat);
Game::chatFilter = new dChatFilter("./res/chatplus_en_us", bool(std::stoi(config.GetValue("dont_generate_dcf"))));
Game::chatFilter = new dChatFilter(Game::assetManager->GetResPath().string() + "/chatplus_en_us", bool(std::stoi(config.GetValue("dont_generate_dcf"))));
//Run it until server gets a kill message from Master:
auto t = std::chrono::high_resolution_clock::now();

View File

@ -10,7 +10,7 @@ void BinaryIO::WriteString(const std::string& stringToWrite, std::ofstream& outs
}
//For reading null-terminated strings
std::string BinaryIO::ReadString(std::ifstream& instream) {
std::string BinaryIO::ReadString(std::istream& instream) {
std::string toReturn;
char buffer;
@ -25,7 +25,7 @@ std::string BinaryIO::ReadString(std::ifstream& instream) {
}
//For reading strings of a specific size
std::string BinaryIO::ReadString(std::ifstream& instream, size_t size) {
std::string BinaryIO::ReadString(std::istream& instream, size_t size) {
std::string toReturn;
char buffer;
@ -37,7 +37,7 @@ std::string BinaryIO::ReadString(std::ifstream& instream, size_t size) {
return toReturn;
}
std::string BinaryIO::ReadWString(std::ifstream& instream) {
std::string BinaryIO::ReadWString(std::istream& instream) {
size_t size;
BinaryRead(instream, size);
//toReturn.resize(size);

View File

@ -10,16 +10,15 @@ namespace BinaryIO {
template<typename T>
std::istream& BinaryRead(std::istream& stream, T& value) {
if (!stream.good())
printf("bla");
if (!stream.good()) throw std::runtime_error("Failed to read from istream.");
return stream.read(reinterpret_cast<char*>(&value), sizeof(T));
}
void WriteString(const std::string& stringToWrite, std::ofstream& outstream);
std::string ReadString(std::ifstream& instream);
std::string ReadString(std::ifstream& instream, size_t size);
std::string ReadWString(std::ifstream& instream);
std::string ReadString(std::istream& instream);
std::string ReadString(std::istream& instream, size_t size);
std::string ReadWString(std::istream& instream);
inline bool DoesFileExist(const std::string& name) {
std::ifstream f(name.c_str());

View File

@ -49,10 +49,10 @@ uint32_t BrickByBrickFix::TruncateBrokenBrickByBrickXml() {
}
// Ignore the valgrind warning about uninitialized values. These are discarded later when we know the actual uncompressed size.
std::unique_ptr<uint8_t[]> uncompressedChunk(new uint8_t[MAX_SD0_CHUNK_SIZE]);
std::unique_ptr<uint8_t[]> uncompressedChunk(new uint8_t[ZCompression::MAX_SD0_CHUNK_SIZE]);
int32_t err{};
int32_t actualUncompressedSize = ZCompression::Decompress(
compressedChunk.get(), chunkSize, uncompressedChunk.get(), MAX_SD0_CHUNK_SIZE, err);
compressedChunk.get(), chunkSize, uncompressedChunk.get(), ZCompression::MAX_SD0_CHUNK_SIZE, err);
if (actualUncompressedSize != -1) {
uint32_t previousSize = completeUncompressedModel.size();

View File

@ -17,10 +17,4 @@ namespace BrickByBrickFix {
* @return The number of BrickByBrick models that were updated
*/
uint32_t UpdateBrickByBrickModelsToSd0();
/**
* @brief Max size of an inflated sd0 zlib chunk
*
*/
constexpr uint32_t MAX_SD0_CHUNK_SIZE = 1024 * 256;
};

View File

@ -17,6 +17,12 @@ set(DCOMMON_SOURCES "AMFFormat.cpp"
"BrickByBrickFix.cpp"
)
add_subdirectory(dClient)
foreach(file ${DCOMMON_DCLIENT_SOURCES})
set(DCOMMON_SOURCES ${DCOMMON_SOURCES} "dClient/${file}")
endforeach()
include_directories(${PROJECT_SOURCE_DIR}/dCommon/)
add_library(dCommon STATIC ${DCOMMON_SOURCES})

View File

@ -10,6 +10,7 @@ class dChatFilter;
class dConfig;
class dLocale;
class RakPeerInterface;
class AssetManager;
struct SystemAddress;
namespace Game {
@ -22,5 +23,6 @@ namespace Game {
extern dLocale* locale;
extern std::mt19937 randomEngine;
extern RakPeerInterface* chatServer;
extern AssetManager* assetManager;
extern SystemAddress chatSysAddr;
}

View File

@ -50,6 +50,7 @@ bool _IsSuffixChar(uint8_t c) {
bool GeneralUtils::_NextUTF8Char(std::string_view& slice, uint32_t& out) {
size_t rem = slice.length();
if (slice.empty()) return false;
const uint8_t* bytes = (const uint8_t*)&slice.front();
if (rem > 0) {
uint8_t first = bytes[0];

View File

@ -8,5 +8,11 @@ namespace ZCompression {
int32_t Compress(const uint8_t* abSrc, int32_t nLenSrc, uint8_t* abDst, int32_t nLenDst);
int32_t Decompress(const uint8_t* abSrc, int32_t nLenSrc, uint8_t* abDst, int32_t nLenDst, int32_t& nErr);
/**
* @brief Max size of an inflated sd0 zlib chunk
*
*/
constexpr uint32_t MAX_SD0_CHUNK_SIZE = 1024 * 256;
}

View File

@ -0,0 +1,201 @@
#include "AssetManager.h"
#include <zlib.h>
AssetManager::AssetManager(const std::string& path) {
if (!std::filesystem::is_directory(path)) {
throw std::runtime_error("Attempted to load asset bundle (" + path + ") however it is not a valid directory.");
}
m_Path = std::filesystem::path(path);
if (std::filesystem::exists(m_Path / "client") && std::filesystem::exists(m_Path / "versions")) {
m_AssetBundleType = eAssetBundleType::Packed;
m_RootPath = m_Path;
m_ResPath = (m_Path / "client" / "res");
} else if (std::filesystem::exists(m_Path / ".." / "versions") && std::filesystem::exists(m_Path / "res")) {
m_AssetBundleType = eAssetBundleType::Packed;
m_RootPath = (m_Path / "..");
m_ResPath = (m_Path / "res");
} else if (std::filesystem::exists(m_Path / "pack") && std::filesystem::exists(m_Path / ".." / ".." / "versions")) {
m_AssetBundleType = eAssetBundleType::Packed;
m_RootPath = (m_Path / ".." / "..");
m_ResPath = m_Path;
} else if (std::filesystem::exists(m_Path / "res" / "cdclient.fdb") && !std::filesystem::exists(m_Path / "res" / "pack")) {
m_AssetBundleType = eAssetBundleType::Unpacked;
m_ResPath = (m_Path / "res");
} else if (std::filesystem::exists(m_Path / "cdclient.fdb") && !std::filesystem::exists(m_Path / "pack")) {
m_AssetBundleType = eAssetBundleType::Unpacked;
m_ResPath = m_Path;
}
if (m_AssetBundleType == eAssetBundleType::None) {
throw std::runtime_error("Failed to identify client type, cannot read client data.");
}
switch (m_AssetBundleType) {
case eAssetBundleType::Packed: {
this->LoadPackIndex();
this->UnpackRequiredAssets();
break;
}
}
}
void AssetManager::LoadPackIndex() {
m_PackIndex = new PackIndex(m_RootPath);
}
std::filesystem::path AssetManager::GetResPath() {
return m_ResPath;
}
eAssetBundleType AssetManager::GetAssetBundleType() {
return m_AssetBundleType;
}
bool AssetManager::HasFile(const char* name) {
auto fixedName = std::string(name);
std::transform(fixedName.begin(), fixedName.end(), fixedName.begin(), [](uint8_t c) { return std::tolower(c); });
std::replace(fixedName.begin(), fixedName.end(), '/', '\\');
auto realPathName = fixedName;
if (fixedName.rfind("client\\res\\", 0) != 0) {
fixedName = "client\\res\\" + fixedName;
}
if (std::filesystem::exists(m_ResPath / realPathName)) {
return true;
}
uint32_t crc = crc32b(0xFFFFFFFF, (uint8_t*)fixedName.c_str(), fixedName.size());
crc = crc32b(crc, (Bytef*)"\0\0\0\0", 4);
for (const auto& item : this->m_PackIndex->GetPackFileIndices()) {
if (item.m_Crc == crc) {
return true;
}
}
return false;
}
bool AssetManager::GetFile(const char* name, char** data, uint32_t* len) {
auto fixedName = std::string(name);
std::transform(fixedName.begin(), fixedName.end(), fixedName.begin(), [](uint8_t c) { return std::tolower(c); });
std::replace(fixedName.begin(), fixedName.end(), '/', '\\');
auto realPathName = fixedName;
if (fixedName.rfind("client\\res\\", 0) != 0) {
fixedName = "client\\res\\" + fixedName;
}
if (std::filesystem::exists(m_ResPath / realPathName)) {
FILE* file;
#ifdef _WIN32
fopen_s(&file, (m_ResPath / realPathName).string().c_str(), "rb");
#elif __APPLE__
// macOS has 64bit file IO by default
file = fopen((m_ResPath / realPathName).string().c_str(), "rb");
#else
file = fopen64((m_ResPath / realPathName).string().c_str(), "rb");
#endif
fseek(file, 0, SEEK_END);
*len = ftell(file);
*data = (char*)malloc(*len);
fseek(file, 0, SEEK_SET);
fread(*data, sizeof(uint8_t), *len, file);
fclose(file);
return true;
}
if (this->m_AssetBundleType == eAssetBundleType::Unpacked) return false;
int32_t packIndex = -1;
uint32_t crc = crc32b(0xFFFFFFFF, (uint8_t*)fixedName.c_str(), fixedName.size());
crc = crc32b(crc, (Bytef*)"\0\0\0\0", 4);
for (const auto& item : this->m_PackIndex->GetPackFileIndices()) {
if (item.m_Crc == crc) {
packIndex = item.m_PackFileIndex;
crc = item.m_Crc;
break;
}
}
if (packIndex == -1 || !crc) {
return false;
}
auto packs = this->m_PackIndex->GetPacks();
auto* pack = packs.at(packIndex);
bool success = pack->ReadFileFromPack(crc, data, len);
return success;
}
AssetMemoryBuffer AssetManager::GetFileAsBuffer(const char* name) {
char* buf;
uint32_t len;
bool success = this->GetFile(name, &buf, &len);
return AssetMemoryBuffer(buf, len, success);
}
void AssetManager::UnpackRequiredAssets() {
if (std::filesystem::exists(m_ResPath / "cdclient.fdb")) return;
char* data;
uint32_t size;
bool success = this->GetFile("cdclient.fdb", &data, &size);
if (!success) {
Game::logger->Log("AssetManager", "Failed to extract required files from the packs.");
delete data;
return;
}
std::ofstream cdclientOutput(m_ResPath / "cdclient.fdb", std::ios::out | std::ios::binary);
cdclientOutput.write(data, size);
cdclientOutput.close();
delete data;
return;
}
uint32_t AssetManager::crc32b(uint32_t base, uint8_t* message, size_t l) {
size_t i, j;
uint32_t crc, msb;
crc = base;
for (i = 0; i < l; i++) {
// xor next byte to upper bits of crc
crc ^= (((unsigned int)message[i]) << 24);
for (j = 0; j < 8; j++) { // Do eight times.
msb = crc >> 31;
crc <<= 1;
crc ^= (0 - msb) & 0x04C11DB7;
}
}
return crc; // don't complement crc on output
}
AssetManager::~AssetManager() {
delete m_PackIndex;
}

View File

@ -0,0 +1,78 @@
#pragma once
#include <string>
#include <vector>
#include <unordered_map>
#include <filesystem>
#include "Pack.h"
#include "PackIndex.h"
enum class eAssetBundleType {
None,
Unpacked,
Packed
};
struct AssetMemoryBuffer : std::streambuf {
char* m_Base;
bool m_Success;
AssetMemoryBuffer(char* base, std::ptrdiff_t n, bool success) {
m_Base = base;
m_Success = success;
if (!m_Success) return;
this->setg(base, base, base + n);
}
pos_type seekpos(pos_type sp, std::ios_base::openmode which) override {
return seekoff(sp - pos_type(off_type(0)), std::ios_base::beg, which);
}
pos_type seekoff(off_type off,
std::ios_base::seekdir dir,
std::ios_base::openmode which = std::ios_base::in) override {
if (dir == std::ios_base::cur)
gbump(off);
else if (dir == std::ios_base::end)
setg(eback(), egptr() + off, egptr());
else if (dir == std::ios_base::beg)
setg(eback(), eback() + off, egptr());
return gptr() - eback();
}
void close() {
delete m_Base;
}
};
class AssetManager {
public:
AssetManager(const std::string& path);
~AssetManager();
std::filesystem::path GetResPath();
eAssetBundleType GetAssetBundleType();
bool HasFile(const char* name);
bool GetFile(const char* name, char** data, uint32_t* len);
AssetMemoryBuffer GetFileAsBuffer(const char* name);
private:
void LoadPackIndex();
void UnpackRequiredAssets();
// Modified crc algorithm (mpeg2)
// Reference: https://stackoverflow.com/questions/54339800/how-to-modify-crc-32-to-crc-32-mpeg-2
inline uint32_t crc32b(uint32_t base, uint8_t* message, size_t l);
bool m_SuccessfullyLoaded;
std::filesystem::path m_Path;
std::filesystem::path m_RootPath;
std::filesystem::path m_ResPath;
eAssetBundleType m_AssetBundleType = eAssetBundleType::None;
PackIndex* m_PackIndex;
};

View File

@ -0,0 +1,6 @@
set(DCOMMON_DCLIENT_SOURCES
"PackIndex.cpp"
"Pack.cpp"
"AssetManager.cpp"
PARENT_SCOPE
)

118
dCommon/dClient/Pack.cpp Normal file
View File

@ -0,0 +1,118 @@
#include "Pack.h"
#include "ZCompression.h"
Pack::Pack(const std::filesystem::path& filePath) {
m_FilePath = filePath;
if (!std::filesystem::exists(filePath)) {
return;
}
m_FileStream = std::ifstream(filePath, std::ios::in | std::ios::binary);
m_FileStream.read(m_Version, 7);
m_FileStream.seekg(-8, std::ios::end); // move file pointer to 8 bytes before the end (location of the address of the record count)
uint32_t recordCountPos = 0;
BinaryIO::BinaryRead<uint32_t>(m_FileStream, recordCountPos);
m_FileStream.seekg(recordCountPos, std::ios::beg);
BinaryIO::BinaryRead<uint32_t>(m_FileStream, m_RecordCount);
for (int i = 0; i < m_RecordCount; i++) {
PackRecord record;
BinaryIO::BinaryRead<PackRecord>(m_FileStream, record);
m_Records.push_back(record);
}
m_FileStream.close();
}
bool Pack::HasFile(uint32_t crc) {
for (const auto& record : m_Records) {
if (record.m_Crc == crc) {
return true;
}
}
return false;
}
bool Pack::ReadFileFromPack(uint32_t crc, char** data, uint32_t* len) {
// Time for some wacky C file reading for speed reasons
PackRecord pkRecord{};
for (const auto& record : m_Records) {
if (record.m_Crc == crc) {
pkRecord = record;
break;
}
}
if (pkRecord.m_Crc == 0) return false;
size_t pos = 0;
pos += pkRecord.m_FilePointer;
bool isCompressed = (pkRecord.m_IsCompressed & 0xff) > 0;
auto inPackSize = isCompressed ? pkRecord.m_CompressedSize : pkRecord.m_UncompressedSize;
FILE* file;
#ifdef _WIN32
fopen_s(&file, m_FilePath.string().c_str(), "rb");
#elif __APPLE__
// macOS has 64bit file IO by default
file = fopen(m_FilePath.string().c_str(), "rb");
#else
file = fopen64(m_FilePath.string().c_str(), "rb");
#endif
fseek(file, pos, SEEK_SET);
if (!isCompressed) {
char* tempData = (char*)malloc(pkRecord.m_UncompressedSize);
fread(tempData, sizeof(uint8_t), pkRecord.m_UncompressedSize, file);
*data = tempData;
*len = pkRecord.m_UncompressedSize;
fclose(file);
return true;
}
pos += 5; // skip header
fseek(file, pos, SEEK_SET);
char* decompressedData = (char*)malloc(pkRecord.m_UncompressedSize);
uint32_t currentReadPos = 0;
while (true) {
if (currentReadPos >= pkRecord.m_UncompressedSize) break;
uint32_t size;
fread(&size, sizeof(uint32_t), 1, file);
pos += 4; // Move pointer position 4 to the right
char* chunk = (char*)malloc(size);
fread(chunk, sizeof(int8_t), size, file);
pos += size; // Move pointer position the amount of bytes read to the right
int32_t err;
currentReadPos += ZCompression::Decompress((uint8_t*)chunk, size, reinterpret_cast<uint8_t*>(decompressedData + currentReadPos), ZCompression::MAX_SD0_CHUNK_SIZE, err);
free(chunk);
}
*data = decompressedData;
*len = pkRecord.m_UncompressedSize;
fclose(file);
return true;
}

38
dCommon/dClient/Pack.h Normal file
View File

@ -0,0 +1,38 @@
#pragma once
#include <vector>
#include <string>
#include <filesystem>
#pragma pack(push, 1)
struct PackRecord {
uint32_t m_Crc;
int32_t m_LowerCrc;
int32_t m_UpperCrc;
uint32_t m_UncompressedSize;
char m_UncompressedHash[32];
uint32_t m_Padding1;
uint32_t m_CompressedSize;
char m_CompressedHash[32];
uint32_t m_Padding2;
uint32_t m_FilePointer;
uint32_t m_IsCompressed; // u32 bool
};
#pragma pack(pop)
class Pack {
public:
Pack(const std::filesystem::path& filePath);
~Pack() = default;
bool HasFile(uint32_t crc);
bool ReadFileFromPack(uint32_t crc, char** data, uint32_t* len);
private:
std::ifstream m_FileStream;
std::filesystem::path m_FilePath;
char m_Version[7];
uint32_t m_RecordCount;
std::vector<PackRecord> m_Records;
};

View File

@ -0,0 +1,50 @@
#include "PackIndex.h"
PackIndex::PackIndex(const std::filesystem::path& filePath) {
m_FileStream = std::ifstream(filePath / "versions" / "primary.pki", std::ios::in | std::ios::binary);
BinaryIO::BinaryRead<uint32_t>(m_FileStream, m_Version);
BinaryIO::BinaryRead<uint32_t>(m_FileStream, m_PackPathCount);
for (int i = 0; i < m_PackPathCount; i++) {
uint32_t stringLen = 0;
BinaryIO::BinaryRead<uint32_t>(m_FileStream, stringLen);
std::string path;
for (int j = 0; j < stringLen; j++) {
char inChar;
BinaryIO::BinaryRead<char>(m_FileStream, inChar);
path += inChar;
}
m_PackPaths.push_back(path);
}
BinaryIO::BinaryRead<uint32_t>(m_FileStream, m_PackFileIndexCount);
for (int i = 0; i < m_PackFileIndexCount; i++) {
PackFileIndex packFileIndex;
BinaryIO::BinaryRead<PackFileIndex>(m_FileStream, packFileIndex);
m_PackFileIndices.push_back(packFileIndex);
}
Game::logger->Log("PackIndex", "Loaded pack catalog with %i pack files and %i files", m_PackPaths.size(), m_PackFileIndices.size());
for (const auto& item : m_PackPaths) {
auto* pack = new Pack(filePath / item);
m_Packs.push_back(pack);
}
m_FileStream.close();
}
PackIndex::~PackIndex() {
for (const auto* item : m_Packs) {
delete item;
}
}

View File

@ -0,0 +1,40 @@
#pragma once
#include <cstdint>
#include <string>
#include <vector>
#include <filesystem>
#include "Pack.h"
#pragma pack(push, 1)
struct PackFileIndex {
uint32_t m_Crc;
int32_t m_LowerCrc;
int32_t m_UpperCrc;
uint32_t m_PackFileIndex;
uint32_t m_IsCompressed; // u32 bool?
};
#pragma pack(pop)
class PackIndex {
public:
PackIndex(const std::filesystem::path& filePath);
~PackIndex();
const std::vector<std::string>& GetPackPaths() { return m_PackPaths; }
const std::vector<PackFileIndex>& GetPackFileIndices() { return m_PackFileIndices; }
const std::vector<Pack*>& GetPacks() { return m_Packs; }
private:
std::ifstream m_FileStream;
uint32_t m_Version;
uint32_t m_PackPathCount;
std::vector<std::string> m_PackPaths;
uint32_t m_PackFileIndexCount;
std::vector<PackFileIndex> m_PackFileIndices;
std::vector<Pack*> m_Packs;
};

View File

@ -13,6 +13,7 @@
#include "PossessableComponent.h"
#include "CharacterComponent.h"
#include "eItemType.h"
#include "AssetManager.h"
class Inventory;
@ -340,18 +341,23 @@ void Item::DisassembleModel() {
std::string renderAsset = result.fieldIsNull(0) ? "" : std::string(result.getStringField(0));
std::vector<std::string> renderAssetSplit = GeneralUtils::SplitString(renderAsset, '\\');
std::string lxfmlPath = "res/BrickModels/" + GeneralUtils::SplitString(renderAssetSplit.back(), '.')[0] + ".lxfml";
std::ifstream file(lxfmlPath);
std::string lxfmlPath = "BrickModels/" + GeneralUtils::SplitString(renderAssetSplit.back(), '.').at(0) + ".lxfml";
auto buffer = Game::assetManager->GetFileAsBuffer(lxfmlPath.c_str());
std::istream file(&buffer);
result.finalize();
if (!file.good()) {
buffer.close();
return;
}
std::stringstream data;
data << file.rdbuf();
buffer.close();
if (data.str().empty()) {
return;
}

View File

@ -3,6 +3,7 @@
#include "BrickDatabase.h"
#include "Game.h"
#include "AssetManager.h"
std::vector<Brick> BrickDatabase::emptyCache{};
BrickDatabase* BrickDatabase::m_Address = nullptr;
@ -17,7 +18,8 @@ std::vector<Brick>& BrickDatabase::GetBricks(const std::string& lxfmlPath) {
return cached->second;
}
std::ifstream file(lxfmlPath);
AssetMemoryBuffer buffer = Game::assetManager->GetFileAsBuffer(("client/" + lxfmlPath).c_str());
std::istream file(&buffer);
if (!file.good()) {
return emptyCache;
}
@ -25,9 +27,12 @@ std::vector<Brick>& BrickDatabase::GetBricks(const std::string& lxfmlPath) {
std::stringstream data;
data << file.rdbuf();
if (data.str().empty()) {
buffer.close();
return emptyCache;
}
buffer.close();
auto* doc = new tinyxml2::XMLDocument();
if (doc->Parse(data.str().c_str(), data.str().size()) != 0) {
delete doc;

View File

@ -63,6 +63,7 @@
#include "GameConfig.h"
#include "ScriptedActivityComponent.h"
#include "LevelProgressionComponent.h"
#include "AssetManager.h"
void SlashCommandHandler::HandleChatCommand(const std::u16string& command, Entity* entity, const SystemAddress& sysAddr) {
std::string chatCommand;
@ -582,7 +583,8 @@ void SlashCommandHandler::HandleChatCommand(const std::u16string& command, Entit
if (args[0].find("/") != std::string::npos) return;
if (args[0].find("\\") != std::string::npos) return;
std::ifstream infile("./res/macros/" + args[0] + ".scm");
auto buf = Game::assetManager->GetFileAsBuffer(("macros/" + args[0] + ".scm").c_str());
std::istream infile(&buf);
if (infile.good()) {
std::string line;
@ -593,6 +595,8 @@ void SlashCommandHandler::HandleChatCommand(const std::u16string& command, Entit
ChatPackets::SendSystemMessage(sysAddr, u"Unknown macro! Is the filename right?");
}
buf.close();
return;
}
@ -1904,10 +1908,7 @@ bool SlashCommandHandler::CheckIfAccessibleZone(const unsigned int zoneID) {
CDZoneTableTable* zoneTable = CDClientManager::Instance()->GetTable<CDZoneTableTable>("ZoneTable");
const CDZoneTable* zone = zoneTable->Query(zoneID);
if (zone != nullptr) {
std::string zonePath = "./res/maps/" + zone->zoneName;
std::transform(zonePath.begin(), zonePath.end(), zonePath.begin(), ::tolower);
std::ifstream f(zonePath.c_str());
return f.good();
return Game::assetManager->HasFile(("maps/" + zone->zoneName).c_str());
} else {
return false;
}

View File

@ -25,6 +25,7 @@
#include "dConfig.h"
#include "dLogger.h"
#include "dServer.h"
#include "AssetManager.h"
//RakNet includes:
#include "RakNetDefines.h"
@ -44,6 +45,7 @@ namespace Game {
dServer* server;
InstanceManager* im;
dConfig* config;
AssetManager* assetManager;
} //namespace Game
bool shutdownSequenceStarted = false;
@ -99,44 +101,49 @@ int main(int argc, char** argv) {
return EXIT_FAILURE;
}
try {
std::string client_path = config.GetValue("client_location");
if (client_path.empty()) client_path = "./res";
Game::assetManager = new AssetManager(config.GetValue("client_location"));
} catch (std::runtime_error& ex) {
Game::logger->Log("MasterServer", "Got an error while setting up assets: %s", ex.what());
return EXIT_FAILURE;
}
MigrationRunner::RunMigrations();
//Check CDClient exists
const std::string cdclient_path = "./res/CDServer.sqlite";
std::ifstream cdclient_fd(cdclient_path);
if (!cdclient_fd.good()) {
Game::logger->Log("WorldServer", "%s could not be opened. Looking for cdclient.fdb to convert to sqlite.", cdclient_path.c_str());
cdclient_fd.close();
// Check CDClient exists
if (!std::filesystem::exists(Game::assetManager->GetResPath() / "CDServer.sqlite")) {
Game::logger->Log("WorldServer", "CDServer.sqlite could not be opened. Looking for cdclient.fdb to convert to sqlite.");
const std::string cdclientFdbPath = "./res/cdclient.fdb";
cdclient_fd.open(cdclientFdbPath);
if (!cdclient_fd.good()) {
Game::logger->Log(
"WorldServer", "%s could not be opened."
"Please move a cdclient.fdb or an already converted database to build/res.", cdclientFdbPath.c_str());
if (!std::filesystem::exists(Game::assetManager->GetResPath() / "cdclient.fdb")) {
Game::logger->Log("WorldServer", "cdclient.fdb could not be opened. Please move a cdclient.fdb or an already converted database to build/res.");
return EXIT_FAILURE;
}
Game::logger->Log("WorldServer", "Found %s. Clearing cdserver migration_history then copying and converting to sqlite.", cdclientFdbPath.c_str());
Game::logger->Log("WorldServer", "Found cdclient.fdb. Clearing cdserver migration_history then copying and converting to sqlite.");
auto stmt = Database::CreatePreppedStmt(R"#(DELETE FROM migration_history WHERE name LIKE "%cdserver%";)#");
stmt->executeUpdate();
delete stmt;
cdclient_fd.close();
std::string res = "python3 ../thirdparty/docker-utils/utils/fdb_to_sqlite.py " + cdclientFdbPath;
int r = system(res.c_str());
if (r != 0) {
std::string res = "python3 ../thirdparty/docker-utils/utils/fdb_to_sqlite.py " + (Game::assetManager->GetResPath() / "cdclient.fdb").string();
int result = system(res.c_str());
if (result != 0) {
Game::logger->Log("MasterServer", "Failed to convert fdb to sqlite");
return EXIT_FAILURE;
}
if (std::rename("./cdclient.sqlite", "./res/CDServer.sqlite") != 0) {
Game::logger->Log("MasterServer", "failed to move cdclient file.");
if (std::rename("./cdclient.sqlite", (Game::assetManager->GetResPath() / "CDServer.sqlite").string().c_str()) != 0) {
Game::logger->Log("MasterServer", "Failed to move cdclient file.");
return EXIT_FAILURE;
}
}
//Connect to CDClient
try {
CDClientDatabase::Connect(cdclient_path);
CDClientDatabase::Connect((Game::assetManager->GetResPath() / "CDServer.sqlite").string());
} catch (CppSQLite3Exception& e) {
Game::logger->Log("WorldServer", "Unable to connect to CDServer SQLite Database");
Game::logger->Log("WorldServer", "Error: %s", e.errorMessage());
@ -152,7 +159,7 @@ int main(int argc, char** argv) {
CDClientManager::Instance()->Initialize();
} catch (CppSQLite3Exception& e) {
Game::logger->Log("WorldServer", "Failed to initialize CDServer SQLite Database");
Game::logger->Log("WorldServer", "May be caused by corrupted file: %s", cdclient_path.c_str());
Game::logger->Log("WorldServer", "May be caused by corrupted file: %s", (Game::assetManager->GetResPath() / "CDServer.sqlite").string().c_str());
Game::logger->Log("WorldServer", "Error: %s", e.errorMessage());
Game::logger->Log("WorldServer", "Error Code: %i", e.errorCode());
return EXIT_FAILURE;

View File

@ -43,7 +43,7 @@ dNavMesh::~dNavMesh() {
void dNavMesh::LoadNavmesh() {
std::string path = "./res/maps/navmeshes/" + std::to_string(m_ZoneId) + ".bin";
std::string path = "./navmeshes/" + std::to_string(m_ZoneId) + ".bin";
if (!BinaryIO::DoesFileExist(path)) {
return;

View File

@ -55,6 +55,7 @@
#include "MasterPackets.h"
#include "Player.h"
#include "PropertyManagementComponent.h"
#include "AssetManager.h"
#include "ZCompression.h"
@ -68,6 +69,8 @@ namespace Game {
dLocale* locale;
std::mt19937 randomEngine;
AssetManager* assetManager;
RakPeerInterface* chatServer;
SystemAddress chatSysAddr;
}
@ -142,9 +145,19 @@ int main(int argc, char** argv) {
Game::logger->SetLogDebugStatements(config.GetValue("log_debug_statements") == "1");
if (config.GetValue("disable_chat") == "1") chatDisabled = true;
try {
std::string client_path = config.GetValue("client_location");
if (client_path.empty()) client_path = "./res";
Game::assetManager = new AssetManager(config.GetValue("client_location"));
} catch (std::runtime_error& ex) {
Game::logger->Log("WorldServer", "Got an error while setting up assets: %s", ex.what());
return EXIT_FAILURE;
}
// Connect to CDClient
try {
CDClientDatabase::Connect("./res/CDServer.sqlite");
CDClientDatabase::Connect((Game::assetManager->GetResPath() / "CDServer.sqlite").string());
} catch (CppSQLite3Exception& e) {
Game::logger->Log("WorldServer", "Unable to connect to CDServer SQLite Database");
Game::logger->Log("WorldServer", "Error: %s", e.errorMessage());
@ -189,7 +202,7 @@ int main(int argc, char** argv) {
ObjectIDManager::Instance()->Initialize();
UserManager::Instance()->Initialize();
LootGenerator::Instance();
Game::chatFilter = new dChatFilter("./res/chatplus_en_us", bool(std::stoi(config.GetValue("dont_generate_dcf"))));
Game::chatFilter = new dChatFilter(Game::assetManager->GetResPath().string() + "/chatplus_en_us", bool(std::stoi(config.GetValue("dont_generate_dcf"))));
Game::server = new dServer(masterIP, ourPort, instanceID, maxClients, false, true, Game::logger, masterIP, masterPort, ServerType::World, zoneID);
@ -243,14 +256,14 @@ int main(int argc, char** argv) {
std::ifstream fileStream;
static const std::vector<std::string> aliases = {
"res/CDServers.fdb",
"res/cdserver.fdb",
"res/CDClient.fdb",
"res/cdclient.fdb",
"CDServers.fdb",
"cdserver.fdb",
"CDClient.fdb",
"cdclient.fdb",
};
for (const auto& file : aliases) {
fileStream.open(file, std::ios::binary | std::ios::in);
fileStream.open(Game::assetManager->GetResPath() / file, std::ios::binary | std::ios::in);
if (fileStream.is_open()) {
break;
}

View File

@ -13,17 +13,22 @@
#include "EntityManager.h"
#include "CDFeatureGatingTable.h"
#include "CDClientManager.h"
#include "AssetManager.h"
Level::Level(Zone* parentZone, const std::string& filepath) {
m_ParentZone = parentZone;
std::ifstream file(filepath, std::ios_base::in | std::ios_base::binary);
if (file) {
ReadChunks(file);
} else {
auto buffer = Game::assetManager->GetFileAsBuffer(filepath.c_str());
if (!buffer.m_Success) {
Game::logger->Log("Level", "Failed to load %s", filepath.c_str());
return;
}
file.close();
std::istream file(&buffer);
ReadChunks(file);
buffer.close();
}
Level::~Level() {
@ -41,7 +46,7 @@ const void Level::PrintAllObjects() {
}
}
void Level::ReadChunks(std::ifstream& file) {
void Level::ReadChunks(std::istream& file) {
const uint32_t CHNK_HEADER = ('C' + ('H' << 8) + ('N' << 16) + ('K' << 24));
while (!file.eof()) {
@ -139,7 +144,7 @@ void Level::ReadChunks(std::ifstream& file) {
}
}
void Level::ReadFileInfoChunk(std::ifstream& file, Header& header) {
void Level::ReadFileInfoChunk(std::istream& file, Header& header) {
FileInfoChunk* fi = new FileInfoChunk;
BinaryIO::BinaryRead(file, fi->version);
BinaryIO::BinaryRead(file, fi->revision);
@ -152,7 +157,7 @@ void Level::ReadFileInfoChunk(std::ifstream& file, Header& header) {
if (header.fileInfo->revision == 3452816845 && m_ParentZone->GetZoneID().GetMapID() == 1100) header.fileInfo->revision = 26;
}
void Level::ReadSceneObjectDataChunk(std::ifstream& file, Header& header) {
void Level::ReadSceneObjectDataChunk(std::istream& file, Header& header) {
SceneObjectDataChunk* chunk = new SceneObjectDataChunk;
uint32_t objectsCount = 0;
BinaryIO::BinaryRead(file, objectsCount);

View File

@ -67,7 +67,7 @@ private:
Zone* m_ParentZone;
//private functions:
void ReadChunks(std::ifstream& file);
void ReadFileInfoChunk(std::ifstream& file, Header& header);
void ReadSceneObjectDataChunk(std::ifstream& file, Header& header);
void ReadChunks(std::istream& file);
void ReadFileInfoChunk(std::istream& file, Header& header);
void ReadSceneObjectDataChunk(std::istream& file, Header& header);
};

View File

@ -7,6 +7,7 @@
#include "GeneralUtils.h"
#include "BinaryIO.h"
#include "AssetManager.h"
#include "CDClientManager.h"
#include "CDZoneTableTable.h"
#include "Spawner.h"
@ -40,7 +41,8 @@ void Zone::LoadZoneIntoMemory() {
m_ZonePath = m_ZoneFilePath.substr(0, m_ZoneFilePath.rfind('/') + 1);
if (m_ZoneFilePath == "ERR") return;
std::ifstream file(m_ZoneFilePath, std::ios::binary);
AssetMemoryBuffer buffer = Game::assetManager->GetFileAsBuffer(m_ZoneFilePath.c_str());
std::istream file(&buffer);
if (file) {
BinaryIO::BinaryRead(file, m_ZoneFileFormatVersion);
@ -144,17 +146,13 @@ void Zone::LoadZoneIntoMemory() {
}
}
//m_PathData.resize(m_PathDataLength);
//file.read((char*)&m_PathData[0], m_PathDataLength);
}
} else {
Game::logger->Log("Zone", "Failed to open: %s", m_ZoneFilePath.c_str());
}
m_ZonePath = m_ZoneFilePath.substr(0, m_ZoneFilePath.rfind('/') + 1);
file.close();
buffer.close();
}
std::string Zone::GetFilePathForZoneID() {
@ -162,7 +160,7 @@ std::string Zone::GetFilePathForZoneID() {
CDZoneTableTable* zoneTable = CDClientManager::Instance()->GetTable<CDZoneTableTable>("ZoneTable");
const CDZoneTable* zone = zoneTable->Query(this->GetZoneID().GetMapID());
if (zone != nullptr) {
std::string toReturn = "./res/maps/" + zone->zoneName;
std::string toReturn = "maps/" + zone->zoneName;
std::transform(toReturn.begin(), toReturn.end(), toReturn.begin(), ::tolower);
return toReturn;
}
@ -222,7 +220,7 @@ const void Zone::PrintAllGameObjects() {
}
}
void Zone::LoadScene(std::ifstream& file) {
void Zone::LoadScene(std::istream& file) {
SceneRef scene;
scene.level = nullptr;
LWOSCENEID lwoSceneID(LWOZONEID_INVALID, 0);
@ -264,10 +262,17 @@ void Zone::LoadScene(std::ifstream& file) {
std::vector<LUTriggers::Trigger*> Zone::LoadLUTriggers(std::string triggerFile, LWOSCENEID sceneID) {
std::vector<LUTriggers::Trigger*> lvlTriggers;
std::ifstream file(m_ZonePath + triggerFile);
auto buffer = Game::assetManager->GetFileAsBuffer((m_ZonePath + triggerFile).c_str());
if (!buffer.m_Success) return lvlTriggers;
std::istream file(&buffer);
std::stringstream data;
data << file.rdbuf();
buffer.close();
if (data.str().size() == 0) return lvlTriggers;
tinyxml2::XMLDocument* doc = new tinyxml2::XMLDocument();
@ -336,7 +341,7 @@ const Path* Zone::GetPath(std::string name) const {
return nullptr;
}
void Zone::LoadSceneTransition(std::ifstream& file) {
void Zone::LoadSceneTransition(std::istream& file) {
SceneTransition sceneTrans;
if (m_ZoneFileFormatVersion < Zone::ZoneFileFormatVersion::Auramar) {
uint8_t length;
@ -355,14 +360,14 @@ void Zone::LoadSceneTransition(std::ifstream& file) {
m_SceneTransitions.push_back(sceneTrans);
}
SceneTransitionInfo Zone::LoadSceneTransitionInfo(std::ifstream& file) {
SceneTransitionInfo Zone::LoadSceneTransitionInfo(std::istream& file) {
SceneTransitionInfo info;
BinaryIO::BinaryRead(file, info.sceneID);
BinaryIO::BinaryRead(file, info.position);
return info;
}
void Zone::LoadPath(std::ifstream& file) {
void Zone::LoadPath(std::istream& file) {
// Currently only spawner (type 4) paths are supported
Path path = Path();

View File

@ -225,9 +225,9 @@ private:
std::map<LWOSCENEID, uint32_t, mapCompareLwoSceneIDs> m_MapRevisions; //rhs is the revision!
//private ("helper") functions:
void LoadScene(std::ifstream& file);
void LoadScene(std::istream& file);
std::vector<LUTriggers::Trigger*> LoadLUTriggers(std::string triggerFile, LWOSCENEID sceneID);
void LoadSceneTransition(std::ifstream& file);
SceneTransitionInfo LoadSceneTransitionInfo(std::ifstream& file);
void LoadPath(std::ifstream& file);
void LoadSceneTransition(std::istream& file);
SceneTransitionInfo LoadSceneTransitionInfo(std::istream& file);
void LoadPath(std::istream& file);
};

View File

@ -21,3 +21,7 @@ max_clients=999
# Where to put crashlogs
dump_folder=
# The location of the client
# Either the folder with /res or with /client and /versions
client_location=