From 2ba3103a0c69252976f5114c67197be38753fabb Mon Sep 17 00:00:00 2001 From: David Markowitz <39972741+EmosewaMC@users.noreply.github.com> Date: Mon, 5 Dec 2022 00:57:58 -0800 Subject: [PATCH] Implement FDB to SQLite (#872) --- dCommon/CMakeLists.txt | 1 + dCommon/FdbToSqlite.cpp | 248 +++++++++++++++++++++++++++++++ dCommon/FdbToSqlite.h | 49 ++++++ dCommon/dEnums/eSqliteDataType.h | 16 ++ dDatabase/MigrationRunner.cpp | 2 + dMasterServer/MasterServer.cpp | 14 +- 6 files changed, 319 insertions(+), 11 deletions(-) create mode 100644 dCommon/FdbToSqlite.cpp create mode 100644 dCommon/FdbToSqlite.h create mode 100644 dCommon/dEnums/eSqliteDataType.h diff --git a/dCommon/CMakeLists.txt b/dCommon/CMakeLists.txt index 8d02186b..549acfb2 100644 --- a/dCommon/CMakeLists.txt +++ b/dCommon/CMakeLists.txt @@ -16,6 +16,7 @@ set(DCOMMON_SOURCES "AMFFormat.cpp" "ZCompression.cpp" "BrickByBrickFix.cpp" "BinaryPathFinder.cpp" + "FdbToSqlite.cpp" ) add_subdirectory(dClient) diff --git a/dCommon/FdbToSqlite.cpp b/dCommon/FdbToSqlite.cpp new file mode 100644 index 00000000..d98d4962 --- /dev/null +++ b/dCommon/FdbToSqlite.cpp @@ -0,0 +1,248 @@ +#include "FdbToSqlite.h" + +#include +#include +#include +#include + +#include "BinaryIO.h" +#include "CDClientDatabase.h" +#include "GeneralUtils.h" +#include "Game.h" +#include "dLogger.h" + +#include "eSqliteDataType.h" + +std::map FdbToSqlite::Convert::sqliteType = { + { eSqliteDataType::NONE, "none"}, + { eSqliteDataType::INT32, "int32"}, + { eSqliteDataType::REAL, "real"}, + { eSqliteDataType::TEXT_4, "text_4"}, + { eSqliteDataType::INT_BOOL, "int_bool"}, + { eSqliteDataType::INT64, "int64"}, + { eSqliteDataType::TEXT_8, "text_8"} +}; + +FdbToSqlite::Convert::Convert(std::string basePath) { + this->basePath = basePath; +} + +bool FdbToSqlite::Convert::ConvertDatabase() { + fdb.open(basePath + "/cdclient.fdb", std::ios::binary); + + try { + CDClientDatabase::Connect(basePath + "/CDServer.sqlite"); + + CDClientDatabase::ExecuteQuery("BEGIN TRANSACTION;"); + + int32_t numberOfTables = ReadInt32(); + ReadTables(numberOfTables); + + CDClientDatabase::ExecuteQuery("COMMIT;"); + } catch (CppSQLite3Exception& e) { + Game::logger->Log("FdbToSqlite", "Encountered error %s converting FDB to SQLite", e.errorMessage()); + return false; + } + + fdb.close(); + return true; +} + +int32_t FdbToSqlite::Convert::ReadInt32() { + uint32_t numberOfTables{}; + BinaryIO::BinaryRead(fdb, numberOfTables); + return numberOfTables; +} + +int64_t FdbToSqlite::Convert::ReadInt64() { + int32_t prevPosition = SeekPointer(); + + int64_t value{}; + BinaryIO::BinaryRead(fdb, value); + + fdb.seekg(prevPosition); + return value; +} + +std::string FdbToSqlite::Convert::ReadString() { + int32_t prevPosition = SeekPointer(); + + auto readString = BinaryIO::ReadString(fdb); + + fdb.seekg(prevPosition); + return readString; +} + +int32_t FdbToSqlite::Convert::SeekPointer() { + int32_t position{}; + BinaryIO::BinaryRead(fdb, position); + int32_t prevPosition = fdb.tellg(); + fdb.seekg(position); + return prevPosition; +} + +std::string FdbToSqlite::Convert::ReadColumnHeader() { + int32_t prevPosition = SeekPointer(); + + int32_t numberOfColumns = ReadInt32(); + std::string tableName = ReadString(); + + auto columns = ReadColumns(numberOfColumns); + std::string newTable = "CREATE TABLE IF NOT EXISTS '" + tableName + "' (" + columns + ");"; + CDClientDatabase::ExecuteDML(newTable); + + fdb.seekg(prevPosition); + + return tableName; +} + +void FdbToSqlite::Convert::ReadTables(int32_t& numberOfTables) { + int32_t prevPosition = SeekPointer(); + + for (int32_t i = 0; i < numberOfTables; i++) { + auto columnHeader = ReadColumnHeader(); + ReadRowHeader(columnHeader); + } + + fdb.seekg(prevPosition); +} + +std::string FdbToSqlite::Convert::ReadColumns(int32_t& numberOfColumns) { + std::stringstream columnsToCreate; + int32_t prevPosition = SeekPointer(); + + std::string name{}; + eSqliteDataType dataType{}; + for (int32_t i = 0; i < numberOfColumns; i++) { + if (i != 0) columnsToCreate << ", "; + dataType = static_cast(ReadInt32()); + name = ReadString(); + columnsToCreate << "'" << name << "' " << FdbToSqlite::Convert::sqliteType[dataType]; + } + + fdb.seekg(prevPosition); + return columnsToCreate.str(); +} + +void FdbToSqlite::Convert::ReadRowHeader(std::string& tableName) { + int32_t prevPosition = SeekPointer(); + + int32_t numberOfAllocatedRows = ReadInt32(); + if (numberOfAllocatedRows != 0) assert((numberOfAllocatedRows & (numberOfAllocatedRows - 1)) == 0); // assert power of 2 allocation size + ReadRows(numberOfAllocatedRows, tableName); + + fdb.seekg(prevPosition); +} + +void FdbToSqlite::Convert::ReadRows(int32_t& numberOfAllocatedRows, std::string& tableName) { + int32_t prevPosition = SeekPointer(); + + int32_t rowid = 0; + for (int32_t row = 0; row < numberOfAllocatedRows; row++) { + int32_t rowPointer = ReadInt32(); + if (rowPointer == -1) rowid++; + else ReadRow(rowid, rowPointer, tableName); + } + + fdb.seekg(prevPosition); +} + +void FdbToSqlite::Convert::ReadRow(int32_t& rowid, int32_t& position, std::string& tableName) { + int32_t prevPosition = fdb.tellg(); + fdb.seekg(position); + + while (true) { + ReadRowInfo(tableName); + int32_t linked = ReadInt32(); + + rowid += 1; + + if (linked == -1) break; + + fdb.seekg(linked); + } + + fdb.seekg(prevPosition); +} + +void FdbToSqlite::Convert::ReadRowInfo(std::string& tableName) { + int32_t prevPosition = SeekPointer(); + + int32_t numberOfColumns = ReadInt32(); + ReadRowValues(numberOfColumns, tableName); + + fdb.seekg(prevPosition); +} + +void FdbToSqlite::Convert::ReadRowValues(int32_t& numberOfColumns, std::string& tableName) { + int32_t prevPosition = SeekPointer(); + + int32_t emptyValue{}; + int32_t intValue{}; + float_t floatValue{}; + std::string stringValue{}; + int32_t boolValue{}; + int64_t int64Value{}; + bool insertedFirstEntry = false; + std::stringstream insertedRow; + insertedRow << "INSERT INTO " << tableName << " values ("; + + for (int32_t i = 0; i < numberOfColumns; i++) { + if (i != 0) insertedRow << ", "; // Only append comma and space after first entry in row. + switch (static_cast(ReadInt32())) { + case eSqliteDataType::NONE: + BinaryIO::BinaryRead(fdb, emptyValue); + assert(emptyValue == 0); + insertedRow << "\"\""; + break; + + case eSqliteDataType::INT32: + intValue = ReadInt32(); + insertedRow << intValue; + break; + + case eSqliteDataType::REAL: + BinaryIO::BinaryRead(fdb, floatValue); + insertedRow << std::fixed << std::setprecision(34) << floatValue; // maximum precision of floating point number + break; + + case eSqliteDataType::TEXT_4: + case eSqliteDataType::TEXT_8: { + stringValue = ReadString(); + size_t position = 0; + + // Need to escape quote with a double of ". + while (position < stringValue.size()) { + if (stringValue.at(position) == '\"') { + stringValue.insert(position, "\""); + position++; + } + position++; + } + insertedRow << "\"" << stringValue << "\""; + break; + } + + case eSqliteDataType::INT_BOOL: + BinaryIO::BinaryRead(fdb, boolValue); + insertedRow << static_cast(boolValue); + break; + + case eSqliteDataType::INT64: + int64Value = ReadInt64(); + insertedRow << std::to_string(int64Value); + break; + + default: + throw std::invalid_argument("Unsupported SQLite type encountered."); + break; + + } + } + + insertedRow << ");"; + + auto copiedString = insertedRow.str(); + CDClientDatabase::ExecuteDML(copiedString); + fdb.seekg(prevPosition); +} diff --git a/dCommon/FdbToSqlite.h b/dCommon/FdbToSqlite.h new file mode 100644 index 00000000..a9611220 --- /dev/null +++ b/dCommon/FdbToSqlite.h @@ -0,0 +1,49 @@ +#ifndef __FDBTOSQLITE__H__ +#define __FDBTOSQLITE__H__ + +#pragma once + +#include +#include +#include + +enum class eSqliteDataType : int32_t; + +namespace FdbToSqlite { + class Convert { + public: + Convert(std::string inputFile); + + bool ConvertDatabase(); + + int32_t ReadInt32(); + + int64_t ReadInt64(); + + std::string ReadString(); + + int32_t SeekPointer(); + + std::string ReadColumnHeader(); + + void ReadTables(int32_t& numberOfTables); + + std::string ReadColumns(int32_t& numberOfColumns); + + void ReadRowHeader(std::string& tableName); + + void ReadRows(int32_t& numberOfAllocatedRows, std::string& tableName); + + void ReadRow(int32_t& rowid, int32_t& position, std::string& tableName); + + void ReadRowInfo(std::string& tableName); + + void ReadRowValues(int32_t& numberOfColumns, std::string& tableName); + private: + static std::map sqliteType; + std::string basePath{}; + std::ifstream fdb{}; + }; // class FdbToSqlite +}; //! namespace FdbToSqlite + +#endif //!__FDBTOSQLITE__H__ diff --git a/dCommon/dEnums/eSqliteDataType.h b/dCommon/dEnums/eSqliteDataType.h new file mode 100644 index 00000000..26e1233b --- /dev/null +++ b/dCommon/dEnums/eSqliteDataType.h @@ -0,0 +1,16 @@ +#ifndef __ESQLITEDATATYPE__H__ +#define __ESQLITEDATATYPE__H__ + +#include + +enum class eSqliteDataType : int32_t { + NONE = 0, + INT32, + REAL = 3, + TEXT_4, + INT_BOOL, + INT64, + TEXT_8 = 8 +}; + +#endif //!__ESQLITEDATATYPE__H__ diff --git a/dDatabase/MigrationRunner.cpp b/dDatabase/MigrationRunner.cpp index 31fb9148..d39201b2 100644 --- a/dDatabase/MigrationRunner.cpp +++ b/dDatabase/MigrationRunner.cpp @@ -136,6 +136,7 @@ void MigrationRunner::RunSQLiteMigrations() { // Doing these 1 migration at a time since one takes a long time and some may think it is crashing. // This will at the least guarentee that the full migration needs to be run in order to be counted as "migrated". Game::logger->Log("MigrationRunner", "Executing migration: %s. This may take a while. Do not shut down server.", migration.name.c_str()); + CDClientDatabase::ExecuteQuery("BEGIN TRANSACTION;"); for (const auto& dml : GeneralUtils::SplitString(migration.data, ';')) { if (dml.empty()) continue; try { @@ -150,6 +151,7 @@ void MigrationRunner::RunSQLiteMigrations() { cdstmt.bind((int32_t) 1, migration.name.c_str()); cdstmt.execQuery().finalize(); cdstmt.finalize(); + CDClientDatabase::ExecuteQuery("COMMIT;"); } Game::logger->Log("MigrationRunner", "CDServer database is up to date."); diff --git a/dMasterServer/MasterServer.cpp b/dMasterServer/MasterServer.cpp index 0a387d8e..0329d9a0 100644 --- a/dMasterServer/MasterServer.cpp +++ b/dMasterServer/MasterServer.cpp @@ -40,6 +40,7 @@ #include "ObjectIDManager.h" #include "PacketUtils.h" #include "dMessageIdentifiers.h" +#include "FdbToSqlite.h" namespace Game { dLogger* logger; @@ -126,18 +127,9 @@ int main(int argc, char** argv) { return EXIT_FAILURE; } - Game::logger->Log("WorldServer", "Found cdclient.fdb. Clearing cdserver migration_history then copying and converting to sqlite."); - auto stmt = Database::CreatePreppedStmt(R"#(DELETE FROM migration_history WHERE name LIKE "%cdserver%";)#"); - stmt->executeUpdate(); - delete stmt; + Game::logger->Log("WorldServer", "Found cdclient.fdb. Converting to SQLite"); - std::string res = "python3 " - + (BinaryPathFinder::GetBinaryDir() / "../thirdparty/docker-utils/utils/fdb_to_sqlite.py").string() - + " --sqlite_path " + (Game::assetManager->GetResPath() / "CDServer.sqlite").string() - + " " + (Game::assetManager->GetResPath() / "cdclient.fdb").string(); - - int result = system(res.c_str()); - if (result != 0) { + if (FdbToSqlite::Convert(Game::assetManager->GetResPath().string()).ConvertDatabase() == false) { Game::logger->Log("MasterServer", "Failed to convert fdb to sqlite"); return EXIT_FAILURE; }