Implement FDB to SQLite (#872)

dCommon/CMakeLists.txt

@@ -16,6 +16,7 @@ set(DCOMMON_SOURCES "AMFFormat.cpp"
 		"ZCompression.cpp"
 		"BrickByBrickFix.cpp"
 		"BinaryPathFinder.cpp"
+		"FdbToSqlite.cpp"
 )
 
 add_subdirectory(dClient)

dCommon/FdbToSqlite.cpp

@@ -0,0 +1,248 @@
+#include "FdbToSqlite.h"
+
+#include <map>
+#include <fstream>
+#include <cassert>
+#include <iomanip>
+
+#include "BinaryIO.h"
+#include "CDClientDatabase.h"
+#include "GeneralUtils.h"
+#include "Game.h"
+#include "dLogger.h"
+
+#include "eSqliteDataType.h"
+
+std::map<eSqliteDataType, std::string> FdbToSqlite::Convert::sqliteType = {
+			{ eSqliteDataType::NONE, "none"},
+			{ eSqliteDataType::INT32, "int32"},
+			{ eSqliteDataType::REAL, "real"},
+			{ eSqliteDataType::TEXT_4, "text_4"},
+			{ eSqliteDataType::INT_BOOL, "int_bool"},
+			{ eSqliteDataType::INT64, "int64"},
+			{ eSqliteDataType::TEXT_8, "text_8"}
+};
+
+FdbToSqlite::Convert::Convert(std::string basePath) {
+	this->basePath = basePath;
+}
+
+bool FdbToSqlite::Convert::ConvertDatabase() {
+	fdb.open(basePath + "/cdclient.fdb", std::ios::binary);
+
+	try {
+		CDClientDatabase::Connect(basePath + "/CDServer.sqlite");
+
+		CDClientDatabase::ExecuteQuery("BEGIN TRANSACTION;");
+
+		int32_t numberOfTables = ReadInt32();
+		ReadTables(numberOfTables);
+
+		CDClientDatabase::ExecuteQuery("COMMIT;");
+	} catch (CppSQLite3Exception& e) {
+		Game::logger->Log("FdbToSqlite", "Encountered error %s converting FDB to SQLite", e.errorMessage());
+		return false;
+	}
+
+	fdb.close();
+	return true;
+}
+
+int32_t FdbToSqlite::Convert::ReadInt32() {
+	uint32_t numberOfTables{};
+	BinaryIO::BinaryRead(fdb, numberOfTables);
+	return numberOfTables;
+}
+
+int64_t FdbToSqlite::Convert::ReadInt64() {
+	int32_t prevPosition = SeekPointer();
+
+	int64_t value{};
+	BinaryIO::BinaryRead(fdb, value);
+
+	fdb.seekg(prevPosition);
+	return value;
+}
+
+std::string FdbToSqlite::Convert::ReadString() {
+	int32_t prevPosition = SeekPointer();
+
+	auto readString = BinaryIO::ReadString(fdb);
+
+	fdb.seekg(prevPosition);
+	return readString;
+}
+
+int32_t FdbToSqlite::Convert::SeekPointer() {
+	int32_t position{};
+	BinaryIO::BinaryRead(fdb, position);
+	int32_t prevPosition = fdb.tellg();
+	fdb.seekg(position);
+	return prevPosition;
+}
+
+std::string FdbToSqlite::Convert::ReadColumnHeader() {
+	int32_t prevPosition = SeekPointer();
+
+	int32_t numberOfColumns = ReadInt32();
+	std::string tableName = ReadString();
+
+	auto columns = ReadColumns(numberOfColumns);
+	std::string newTable = "CREATE TABLE IF NOT EXISTS '" + tableName + "' (" + columns + ");";
+	CDClientDatabase::ExecuteDML(newTable);
+
+	fdb.seekg(prevPosition);
+
+	return tableName;
+}
+
+void FdbToSqlite::Convert::ReadTables(int32_t& numberOfTables) {
+	int32_t prevPosition = SeekPointer();
+
+	for (int32_t i = 0; i < numberOfTables; i++) {
+		auto columnHeader = ReadColumnHeader();
+		ReadRowHeader(columnHeader);
+	}
+
+	fdb.seekg(prevPosition);
+}
+
+std::string FdbToSqlite::Convert::ReadColumns(int32_t& numberOfColumns) {
+	std::stringstream columnsToCreate;
+	int32_t prevPosition = SeekPointer();
+
+	std::string name{};
+	eSqliteDataType dataType{};
+	for (int32_t i = 0; i < numberOfColumns; i++) {
+		if (i != 0) columnsToCreate << ", ";
+		dataType = static_cast<eSqliteDataType>(ReadInt32());
+		name = ReadString();
+		columnsToCreate << "'" << name << "' " << FdbToSqlite::Convert::sqliteType[dataType];
+	}
+
+	fdb.seekg(prevPosition);
+	return columnsToCreate.str();
+}
+
+void FdbToSqlite::Convert::ReadRowHeader(std::string& tableName) {
+	int32_t prevPosition = SeekPointer();
+
+	int32_t numberOfAllocatedRows = ReadInt32();
+	if (numberOfAllocatedRows != 0) assert((numberOfAllocatedRows & (numberOfAllocatedRows - 1)) == 0);  // assert power of 2 allocation size
+	ReadRows(numberOfAllocatedRows, tableName);
+
+	fdb.seekg(prevPosition);
+}
+
+void FdbToSqlite::Convert::ReadRows(int32_t& numberOfAllocatedRows, std::string& tableName) {
+	int32_t prevPosition = SeekPointer();
+
+	int32_t rowid = 0;
+	for (int32_t row = 0; row < numberOfAllocatedRows; row++) {
+		int32_t rowPointer = ReadInt32();
+		if (rowPointer == -1) rowid++;
+		else ReadRow(rowid, rowPointer, tableName);
+	}
+
+	fdb.seekg(prevPosition);
+}
+
+void FdbToSqlite::Convert::ReadRow(int32_t& rowid, int32_t& position, std::string& tableName) {
+	int32_t prevPosition = fdb.tellg();
+	fdb.seekg(position);
+
+	while (true) {
+		ReadRowInfo(tableName);
+		int32_t linked = ReadInt32();
+
+		rowid += 1;
+
+		if (linked == -1) break;
+
+		fdb.seekg(linked);
+	}
+
+	fdb.seekg(prevPosition);
+}
+
+void FdbToSqlite::Convert::ReadRowInfo(std::string& tableName) {
+	int32_t prevPosition = SeekPointer();
+
+	int32_t numberOfColumns = ReadInt32();
+	ReadRowValues(numberOfColumns, tableName);
+
+	fdb.seekg(prevPosition);
+}
+
+void FdbToSqlite::Convert::ReadRowValues(int32_t& numberOfColumns, std::string& tableName) {
+	int32_t prevPosition = SeekPointer();
+
+	int32_t emptyValue{};
+	int32_t intValue{};
+	float_t floatValue{};
+	std::string stringValue{};
+	int32_t boolValue{};
+	int64_t int64Value{};
+	bool insertedFirstEntry = false;
+	std::stringstream insertedRow;
+	insertedRow << "INSERT INTO " << tableName << " values (";
+
+	for (int32_t i = 0; i < numberOfColumns; i++) {
+		if (i != 0) insertedRow << ", "; // Only append comma and space after first entry in row.
+		switch (static_cast<eSqliteDataType>(ReadInt32())) {
+		case eSqliteDataType::NONE:
+			BinaryIO::BinaryRead(fdb, emptyValue);
+			assert(emptyValue == 0);
+			insertedRow << "\"\"";
+			break;
+
+		case eSqliteDataType::INT32:
+			intValue = ReadInt32();
+			insertedRow << intValue;
+			break;
+
+		case eSqliteDataType::REAL:
+			BinaryIO::BinaryRead(fdb, floatValue);
+			insertedRow << std::fixed << std::setprecision(34) << floatValue; // maximum precision of floating point number
+			break;
+
+		case eSqliteDataType::TEXT_4:
+		case eSqliteDataType::TEXT_8: {
+			stringValue = ReadString();
+			size_t position = 0;
+
+			// Need to escape quote with a double of ".
+			while (position < stringValue.size()) {
+				if (stringValue.at(position) == '\"') {
+					stringValue.insert(position, "\"");
+					position++;
+				}
+				position++;
+			}
+			insertedRow << "\"" << stringValue << "\"";
+			break;
+		}
+
+		case eSqliteDataType::INT_BOOL:
+			BinaryIO::BinaryRead(fdb, boolValue);
+			insertedRow << static_cast<bool>(boolValue);
+			break;
+
+		case eSqliteDataType::INT64:
+			int64Value = ReadInt64();
+			insertedRow << std::to_string(int64Value);
+			break;
+
+		default:
+			throw std::invalid_argument("Unsupported SQLite type encountered.");
+			break;
+
+		}
+	}
+
+	insertedRow << ");";
+
+	auto copiedString = insertedRow.str();
+	CDClientDatabase::ExecuteDML(copiedString);
+	fdb.seekg(prevPosition);
+}

dCommon/FdbToSqlite.h

@@ -0,0 +1,49 @@
+#ifndef __FDBTOSQLITE__H__
+#define __FDBTOSQLITE__H__
+
+#pragma once
+
+#include <cstdint>
+#include <iosfwd>
+#include <map>
+
+enum class eSqliteDataType : int32_t;
+
+namespace FdbToSqlite {
+	class Convert {
+	public:
+		Convert(std::string inputFile);
+
+		bool ConvertDatabase();
+
+		int32_t ReadInt32();
+
+		int64_t ReadInt64();
+
+		std::string ReadString();
+
+		int32_t SeekPointer();
+
+		std::string ReadColumnHeader();
+
+		void ReadTables(int32_t& numberOfTables);
+
+		std::string ReadColumns(int32_t& numberOfColumns);
+
+		void ReadRowHeader(std::string& tableName);
+
+		void ReadRows(int32_t& numberOfAllocatedRows, std::string& tableName);
+
+		void ReadRow(int32_t& rowid, int32_t& position, std::string& tableName);
+
+		void ReadRowInfo(std::string& tableName);
+
+		void ReadRowValues(int32_t& numberOfColumns, std::string& tableName);
+	private:
+		static std::map<eSqliteDataType, std::string> sqliteType;
+		std::string basePath{};
+		std::ifstream fdb{};
+	}; // class FdbToSqlite
+}; //! namespace FdbToSqlite
+
+#endif  //!__FDBTOSQLITE__H__

dCommon/dEnums/eSqliteDataType.h

@@ -0,0 +1,16 @@
+#ifndef __ESQLITEDATATYPE__H__
+#define __ESQLITEDATATYPE__H__
+
+#include <cstdint>
+
+enum class eSqliteDataType : int32_t {
+	NONE = 0,
+	INT32,
+	REAL = 3,
+	TEXT_4,
+	INT_BOOL,
+	INT64,
+	TEXT_8 = 8
+};
+
+#endif  //!__ESQLITEDATATYPE__H__

dDatabase/MigrationRunner.cpp

@@ -136,6 +136,7 @@ void MigrationRunner::RunSQLiteMigrations() {
 		// Doing these 1 migration at a time since one takes a long time and some may think it is crashing.
 		// This will at the least guarentee that the full migration needs to be run in order to be counted as "migrated".
 		Game::logger->Log("MigrationRunner", "Executing migration: %s.  This may take a while.  Do not shut down server.", migration.name.c_str());
+		CDClientDatabase::ExecuteQuery("BEGIN TRANSACTION;");
 		for (const auto& dml : GeneralUtils::SplitString(migration.data, ';')) {
 			if (dml.empty()) continue;
 			try {
@@ -150,6 +151,7 @@ void MigrationRunner::RunSQLiteMigrations() {
 		cdstmt.bind((int32_t) 1, migration.name.c_str());
 		cdstmt.execQuery().finalize();
 		cdstmt.finalize();
+		CDClientDatabase::ExecuteQuery("COMMIT;");
 	}
 
 	Game::logger->Log("MigrationRunner", "CDServer database is up to date.");

dMasterServer/MasterServer.cpp

@@ -40,6 +40,7 @@
 #include "ObjectIDManager.h"
 #include "PacketUtils.h"
 #include "dMessageIdentifiers.h"
+#include "FdbToSqlite.h"
 
 namespace Game {
 	dLogger* logger;
@@ -126,18 +127,9 @@ int main(int argc, char** argv) {
 			return EXIT_FAILURE;
 		}
 
-		Game::logger->Log("WorldServer", "Found cdclient.fdb.  Clearing cdserver migration_history then copying and converting to sqlite.");
-		auto stmt = Database::CreatePreppedStmt(R"#(DELETE FROM migration_history WHERE name LIKE "%cdserver%";)#");
-		stmt->executeUpdate();
-		delete stmt;
+		Game::logger->Log("WorldServer", "Found cdclient.fdb.  Converting to SQLite");
 
-		std::string res = "python3 "
-			+ (BinaryPathFinder::GetBinaryDir() / "../thirdparty/docker-utils/utils/fdb_to_sqlite.py").string()
-			+ " --sqlite_path " + (Game::assetManager->GetResPath() / "CDServer.sqlite").string()
-			+ " " + (Game::assetManager->GetResPath() / "cdclient.fdb").string();
-
-		int result = system(res.c_str());
-		if (result != 0) {
+		if (FdbToSqlite::Convert(Game::assetManager->GetResPath().string()).ConvertDatabase() == false) {
 			Game::logger->Log("MasterServer", "Failed to convert fdb to sqlite");
 			return EXIT_FAILURE;
 		}