Mirrored_Repos/DarkflameServer
1
0
Fork 0
mirror of https://github.com/DarkflameUniverse/DarkflameServer.git synced 2025-08-04 09:44:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Address Docker issues and remove need to extract cdclient.fdb (#895)

Browse Source 
* Implement a server res directory

* Only convert if neither exist

* Remove unzip, Update RegEx

* readme updates

Run setup after setting working dir

Address several docker issues

Revert "Run setup after setting working dir"

This reverts commit fd2fb9228e82a350204c1ef61f7ba059479bb12f.

Fix docker

* Remove extra submodules

* Rework logic

* Switch if block

* Remove need to extract fdb from client

* Change log name

* Update FdbToSqlite.cpp
This commit is contained in:
David Markowitz
2023-01-06 21:04:20 -08:00
committed by GitHub
parent 7fcc8a6e84
commit bad3845d83
14 changed files with 109 additions and 243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
dCommon/FdbToSqlite.cpp
View File

@@ -10,6 +10,7 @@
#include "GeneralUtils.h"
#include "Game.h"
#include "dLogger.h"
#include "AssetManager.h"
#include "eSqliteDataType.h"
@@ -23,26 +24,23 @@ std::map<eSqliteDataType, std::string> FdbToSqlite::Convert::m_SqliteType = {
{ eSqliteDataType::TEXT_8, "text_8"}
};
FdbToSqlite::Convert::Convert(std::string basePath, std::string binaryOutPath) {
this->m_BasePath = basePath;
FdbToSqlite::Convert::Convert(std::string binaryOutPath) {
this->m_BinaryOutPath = binaryOutPath;
m_Fdb.open(m_BasePath + "/cdclient.fdb", std::ios::binary);
}
FdbToSqlite::Convert::~Convert() {
this->m_Fdb.close();
}
bool FdbToSqlite::Convert::ConvertDatabase() {
bool FdbToSqlite::Convert::ConvertDatabase(AssetMemoryBuffer& buffer) {
if (m_ConversionStarted) return false;
std::istream cdClientBuffer(&buffer);
this->m_ConversionStarted = true;
try {
CDClientDatabase::Connect(m_BinaryOutPath + "/CDServer.sqlite");
CDClientDatabase::ExecuteQuery("BEGIN TRANSACTION;");
int32_t numberOfTables = ReadInt32();
ReadTables(numberOfTables);
int32_t numberOfTables = ReadInt32(cdClientBuffer);
ReadTables(numberOfTables, cdClientBuffer);
CDClientDatabase::ExecuteQuery("COMMIT;");
} catch (CppSQLite3Exception& e) {
@@ -53,130 +51,130 @@ bool FdbToSqlite::Convert::ConvertDatabase() {
return true;
}
int32_t FdbToSqlite::Convert::ReadInt32() {
int32_t FdbToSqlite::Convert::ReadInt32(std::istream& cdClientBuffer) {
int32_t nextInt{};
BinaryIO::BinaryRead(m_Fdb, nextInt);
BinaryIO::BinaryRead(cdClientBuffer, nextInt);
return nextInt;
}
int64_t FdbToSqlite::Convert::ReadInt64() {
int32_t prevPosition = SeekPointer();
int64_t FdbToSqlite::Convert::ReadInt64(std::istream& cdClientBuffer) {
int32_t prevPosition = SeekPointer(cdClientBuffer);
int64_t value{};
BinaryIO::BinaryRead(m_Fdb, value);
BinaryIO::BinaryRead(cdClientBuffer, value);
m_Fdb.seekg(prevPosition);
cdClientBuffer.seekg(prevPosition);
return value;
}
std::string FdbToSqlite::Convert::ReadString() {
int32_t prevPosition = SeekPointer();
std::string FdbToSqlite::Convert::ReadString(std::istream& cdClientBuffer) {
int32_t prevPosition = SeekPointer(cdClientBuffer);
auto readString = BinaryIO::ReadString(m_Fdb);
auto readString = BinaryIO::ReadString(cdClientBuffer);
m_Fdb.seekg(prevPosition);
cdClientBuffer.seekg(prevPosition);
return readString;
}
int32_t FdbToSqlite::Convert::SeekPointer() {
int32_t FdbToSqlite::Convert::SeekPointer(std::istream& cdClientBuffer) {
int32_t position{};
BinaryIO::BinaryRead(m_Fdb, position);
int32_t prevPosition = m_Fdb.tellg();
m_Fdb.seekg(position);
BinaryIO::BinaryRead(cdClientBuffer, position);
int32_t prevPosition = cdClientBuffer.tellg();
cdClientBuffer.seekg(position);
return prevPosition;
}
std::string FdbToSqlite::Convert::ReadColumnHeader() {
int32_t prevPosition = SeekPointer();
std::string FdbToSqlite::Convert::ReadColumnHeader(std::istream& cdClientBuffer) {
int32_t prevPosition = SeekPointer(cdClientBuffer);
int32_t numberOfColumns = ReadInt32();
std::string tableName = ReadString();
int32_t numberOfColumns = ReadInt32(cdClientBuffer);
std::string tableName = ReadString(cdClientBuffer);
auto columns = ReadColumns(numberOfColumns);
auto columns = ReadColumns(numberOfColumns, cdClientBuffer);
std::string newTable = "CREATE TABLE IF NOT EXISTS '" + tableName + "' (" + columns + ");";
CDClientDatabase::ExecuteDML(newTable);
m_Fdb.seekg(prevPosition);
cdClientBuffer.seekg(prevPosition);
return tableName;
}
void FdbToSqlite::Convert::ReadTables(int32_t& numberOfTables) {
int32_t prevPosition = SeekPointer();
void FdbToSqlite::Convert::ReadTables(int32_t& numberOfTables, std::istream& cdClientBuffer) {
int32_t prevPosition = SeekPointer(cdClientBuffer);
for (int32_t i = 0; i < numberOfTables; i++) {
auto columnHeader = ReadColumnHeader();
ReadRowHeader(columnHeader);
auto columnHeader = ReadColumnHeader(cdClientBuffer);
ReadRowHeader(columnHeader, cdClientBuffer);
}
m_Fdb.seekg(prevPosition);
cdClientBuffer.seekg(prevPosition);
}
std::string FdbToSqlite::Convert::ReadColumns(int32_t& numberOfColumns) {
std::string FdbToSqlite::Convert::ReadColumns(int32_t& numberOfColumns, std::istream& cdClientBuffer) {
std::stringstream columnsToCreate;
int32_t prevPosition = SeekPointer();
int32_t prevPosition = SeekPointer(cdClientBuffer);
std::string name{};
eSqliteDataType dataType{};
for (int32_t i = 0; i < numberOfColumns; i++) {
if (i != 0) columnsToCreate << ", ";
dataType = static_cast<eSqliteDataType>(ReadInt32());
name = ReadString();
dataType = static_cast<eSqliteDataType>(ReadInt32(cdClientBuffer));
name = ReadString(cdClientBuffer);
columnsToCreate << "'" << name << "' " << FdbToSqlite::Convert::m_SqliteType[dataType];
}
m_Fdb.seekg(prevPosition);
cdClientBuffer.seekg(prevPosition);
return columnsToCreate.str();
}
void FdbToSqlite::Convert::ReadRowHeader(std::string& tableName) {
int32_t prevPosition = SeekPointer();
void FdbToSqlite::Convert::ReadRowHeader(std::string& tableName, std::istream& cdClientBuffer) {
int32_t prevPosition = SeekPointer(cdClientBuffer);
int32_t numberOfAllocatedRows = ReadInt32();
int32_t numberOfAllocatedRows = ReadInt32(cdClientBuffer);
if (numberOfAllocatedRows != 0) assert((numberOfAllocatedRows & (numberOfAllocatedRows - 1)) == 0); // assert power of 2 allocation size
ReadRows(numberOfAllocatedRows, tableName);
ReadRows(numberOfAllocatedRows, tableName, cdClientBuffer);
m_Fdb.seekg(prevPosition);
cdClientBuffer.seekg(prevPosition);
}
void FdbToSqlite::Convert::ReadRows(int32_t& numberOfAllocatedRows, std::string& tableName) {
int32_t prevPosition = SeekPointer();
void FdbToSqlite::Convert::ReadRows(int32_t& numberOfAllocatedRows, std::string& tableName, std::istream& cdClientBuffer) {
int32_t prevPosition = SeekPointer(cdClientBuffer);
int32_t rowid = 0;
for (int32_t row = 0; row < numberOfAllocatedRows; row++) {
int32_t rowPointer = ReadInt32();
int32_t rowPointer = ReadInt32(cdClientBuffer);
if (rowPointer == -1) rowid++;
else ReadRow(rowPointer, tableName);
else ReadRow(rowPointer, tableName, cdClientBuffer);
}
m_Fdb.seekg(prevPosition);
cdClientBuffer.seekg(prevPosition);
}
void FdbToSqlite::Convert::ReadRow(int32_t& position, std::string& tableName) {
int32_t prevPosition = m_Fdb.tellg();
m_Fdb.seekg(position);
void FdbToSqlite::Convert::ReadRow(int32_t& position, std::string& tableName, std::istream& cdClientBuffer) {
int32_t prevPosition = cdClientBuffer.tellg();
cdClientBuffer.seekg(position);
while (true) {
ReadRowInfo(tableName);
int32_t linked = ReadInt32();
ReadRowInfo(tableName, cdClientBuffer);
int32_t linked = ReadInt32(cdClientBuffer);
if (linked == -1) break;
m_Fdb.seekg(linked);
cdClientBuffer.seekg(linked);
}
m_Fdb.seekg(prevPosition);
cdClientBuffer.seekg(prevPosition);
}
void FdbToSqlite::Convert::ReadRowInfo(std::string& tableName) {
int32_t prevPosition = SeekPointer();
void FdbToSqlite::Convert::ReadRowInfo(std::string& tableName, std::istream& cdClientBuffer) {
int32_t prevPosition = SeekPointer(cdClientBuffer);
int32_t numberOfColumns = ReadInt32();
ReadRowValues(numberOfColumns, tableName);
int32_t numberOfColumns = ReadInt32(cdClientBuffer);
ReadRowValues(numberOfColumns, tableName, cdClientBuffer);
m_Fdb.seekg(prevPosition);
cdClientBuffer.seekg(prevPosition);
}
void FdbToSqlite::Convert::ReadRowValues(int32_t& numberOfColumns, std::string& tableName) {
int32_t prevPosition = SeekPointer();
void FdbToSqlite::Convert::ReadRowValues(int32_t& numberOfColumns, std::string& tableName, std::istream& cdClientBuffer) {
int32_t prevPosition = SeekPointer(cdClientBuffer);
int32_t emptyValue{};
int32_t intValue{};
@@ -190,26 +188,26 @@ void FdbToSqlite::Convert::ReadRowValues(int32_t& numberOfColumns, std::string&
for (int32_t i = 0; i < numberOfColumns; i++) {
if (i != 0) insertedRow << ", "; // Only append comma and space after first entry in row.
switch (static_cast<eSqliteDataType>(ReadInt32())) {
switch (static_cast<eSqliteDataType>(ReadInt32(cdClientBuffer))) {
case eSqliteDataType::NONE:
BinaryIO::BinaryRead(m_Fdb, emptyValue);
BinaryIO::BinaryRead(cdClientBuffer, emptyValue);
assert(emptyValue == 0);
insertedRow << "NULL";
break;
case eSqliteDataType::INT32:
intValue = ReadInt32();
intValue = ReadInt32(cdClientBuffer);
insertedRow << intValue;
break;
case eSqliteDataType::REAL:
BinaryIO::BinaryRead(m_Fdb, floatValue);
BinaryIO::BinaryRead(cdClientBuffer, floatValue);
insertedRow << std::fixed << std::setprecision(34) << floatValue; // maximum precision of floating point number
break;
case eSqliteDataType::TEXT_4:
case eSqliteDataType::TEXT_8: {
stringValue = ReadString();
stringValue = ReadString(cdClientBuffer);
size_t position = 0;
// Need to escape quote with a double of ".
@@ -225,12 +223,12 @@ void FdbToSqlite::Convert::ReadRowValues(int32_t& numberOfColumns, std::string&
}
case eSqliteDataType::INT_BOOL:
BinaryIO::BinaryRead(m_Fdb, boolValue);
BinaryIO::BinaryRead(cdClientBuffer, boolValue);
insertedRow << static_cast<bool>(boolValue);
break;
case eSqliteDataType::INT64:
int64Value = ReadInt64();
int64Value = ReadInt64(cdClientBuffer);
insertedRow << std::to_string(int64Value);
break;
@@ -245,5 +243,5 @@ void FdbToSqlite::Convert::ReadRowValues(int32_t& numberOfColumns, std::string&
auto copiedString = insertedRow.str();
CDClientDatabase::ExecuteDML(copiedString);
m_Fdb.seekg(prevPosition);
cdClientBuffer.seekg(prevPosition);
}

40
dCommon/FdbToSqlite.h
View File

@@ -7,6 +7,8 @@
#include <iosfwd>
#include <map>
class AssetMemoryBuffer;
enum class eSqliteDataType : int32_t;
namespace FdbToSqlite {
@@ -18,33 +20,28 @@ namespace FdbToSqlite {
* @param inputFile The file which ends in .fdb to be converted
* @param binaryPath The base path where the file will be saved
*/
Convert(std::string inputFile, std::string binaryOutPath);
/**
* Destroy the convert object and close the fdb file.
*/
~Convert();
Convert(std::string binaryOutPath);
/**
* Converts the input file to sqlite. Calling multiple times is safe.
*
* @return true if the database was converted properly, false otherwise.
*/
bool ConvertDatabase();
bool ConvertDatabase(AssetMemoryBuffer& buffer);
/**
* @brief Reads a 32 bit int from the fdb file.
*
* @return The read value
*/
int32_t ReadInt32();
int32_t ReadInt32(std::istream& cdClientBuffer);
/**
* @brief Reads a 64 bit integer from the fdb file.
*
* @return The read value
*/
int64_t ReadInt64();
int64_t ReadInt64(std::istream& cdClientBuffer);
/**
* @brief Reads a string from the fdb file.
@@ -53,28 +50,28 @@ namespace FdbToSqlite {
*
* TODO This needs to be translated to latin-1!
*/
std::string ReadString();
std::string ReadString(std::istream& cdClientBuffer);
/**
* @brief Seeks to a pointer position.
*
* @return The previous position before the seek
*/
int32_t SeekPointer();
int32_t SeekPointer(std::istream& cdClientBuffer);
/**
* @brief Reads a column header from the fdb file and creates the table in the database
*
* @return The table name
*/
std::string ReadColumnHeader();
std::string ReadColumnHeader(std::istream& cdClientBuffer);
/**
* @brief Read the tables from the fdb file.
*
* @param numberOfTables The number of tables to read
*/
void ReadTables(int32_t& numberOfTables);
void ReadTables(int32_t& numberOfTables, std::istream& cdClientBuffer);
/**
* @brief Reads the columns from the fdb file.
@@ -82,14 +79,14 @@ namespace FdbToSqlite {
* @param numberOfColumns The number of columns to read
* @return All columns of the table formatted for a sql query
*/
std::string ReadColumns(int32_t& numberOfColumns);
std::string ReadColumns(int32_t& numberOfColumns, std::istream& cdClientBuffer);
/**
* @brief Reads the row header from the fdb file.
*
* @param tableName The tables name
*/
void ReadRowHeader(std::string& tableName);
void ReadRowHeader(std::string& tableName, std::istream& cdClientBuffer);
/**
* @brief Read the rows from the fdb file.,
@@ -97,7 +94,7 @@ namespace FdbToSqlite {
* @param numberOfAllocatedRows The number of rows that were allocated. Always a power of 2!
* @param tableName The tables name.
*/
void ReadRows(int32_t& numberOfAllocatedRows, std::string& tableName);
void ReadRows(int32_t& numberOfAllocatedRows, std::string& tableName, std::istream& cdClientBuffer);
/**
* @brief Reads a row from the fdb file.
@@ -105,14 +102,14 @@ namespace FdbToSqlite {
* @param position The position to seek in the fdb to
* @param tableName The tables name
*/
void ReadRow(int32_t& position, std::string& tableName);
void ReadRow(int32_t& position, std::string& tableName, std::istream& cdClientBuffer);
/**
* @brief Reads the row info from the fdb file.
*
* @param tableName The tables name
*/
void ReadRowInfo(std::string& tableName);
void ReadRowInfo(std::string& tableName, std::istream& cdClientBuffer);
/**
* @brief Reads each row and its values from the fdb file and inserts them into the database
@@ -120,7 +117,7 @@ namespace FdbToSqlite {
* @param numberOfColumns The number of columns to read in
* @param tableName The tables name
*/
void ReadRowValues(int32_t& numberOfColumns, std::string& tableName);
void ReadRowValues(int32_t& numberOfColumns, std::string& tableName, std::istream& cdClientBuffer);
private:
/**
@@ -132,11 +129,6 @@ namespace FdbToSqlite {
* Base path of the folder containing the fdb file
*/
std::string m_BasePath{};
/**
* ifstream containing the fdb file
*/
std::ifstream m_Fdb{};
/**
* Whether or not a conversion was started. If one was started, do not attempt to convert the file again.

28
dCommon/dClient/AssetManager.cpp
View File

@@ -45,9 +45,6 @@ AssetManager::AssetManager(const std::filesystem::path& path) {
switch (m_AssetBundleType) {
case eAssetBundleType::Packed: {
this->LoadPackIndex();
this->UnpackRequiredAssets();
break;
}
}
@@ -160,31 +157,6 @@ AssetMemoryBuffer AssetManager::GetFileAsBuffer(const char* name) {
return AssetMemoryBuffer(buf, len, success);
}
void AssetManager::UnpackRequiredAssets() {
if (std::filesystem::exists(m_ResPath / "cdclient.fdb")) return;
char* data;
uint32_t size;
bool success = this->GetFile("cdclient.fdb", &data, &size);
if (!success) {
Game::logger->Log("AssetManager", "Failed to extract required files from the packs.");
delete data;
return;
}
std::ofstream cdclientOutput(m_ResPath / "cdclient.fdb", std::ios::out | std::ios::binary);
cdclientOutput.write(data, size);
cdclientOutput.close();
delete data;
return;
}
uint32_t AssetManager::crc32b(uint32_t base, uint8_t* message, size_t l) {
size_t i, j;
uint32_t crc, msb;

1
dCommon/dClient/AssetManager.h
View File

@@ -60,7 +60,6 @@ public:
private:
void LoadPackIndex();
void UnpackRequiredAssets();
// Modified crc algorithm (mpeg2)
// Reference: https://stackoverflow.com/questions/54339800/how-to-modify-crc-32-to-crc-32-mpeg-2