Start moving migration stuff!

dDatabase/CMakeLists.txt

@@ -1,7 +1,6 @@
 set(DDATABASE_SOURCES "CDClientDatabase.cpp"
 		"CDClientManager.cpp"
-		"Database.cpp"
-		"MigrationRunner.cpp")
+		"Database.cpp")
 
 add_subdirectory(Tables)
 
@@ -15,5 +14,11 @@ foreach (file ${DDATABASE_DATABASES_SOURCES})
 	set(DDATABASE_SOURCES ${DDATABASE_SOURCES} "Databases/${file}")
 endforeach()
 
+add_subdirectory(Databases/Migrations)
+
+foreach (file ${DDATABASE_DATABASES_MIGRATIONS_SOURCES})
+	set(DDATABASE_SOURCES ${DDATABASE_SOURCES} "Databases/Migrations/${file}")
+endforeach()
+
 add_library(dDatabase STATIC ${DDATABASE_SOURCES})
 target_link_libraries(dDatabase sqlite3 mariadbConnCpp)

dDatabase/Databases/Migrations/CMakeLists.txt

@@ -0,0 +1 @@
+set(DDATABASE_DATABASES_MIGRATIONS_SOURCES "MigrationManager.cpp" "MySQLMigration.cpp")

dDatabase/Databases/Migrations/MigrationManager.cpp

@@ -0,0 +1,90 @@
+#include "MigrationManager.h"
+
+#include <fstream>
+#include <string>
+
+#include "dLogger.h"
+#include "GeneralUtils.h"
+#include "CDClientDatabase.h"
+#include "BinaryPathFinder.h"
+
+Migration Migration::LoadMigration(const std::string& type, const std::string& dbType, const std::string& name) {
+	Migration migration{};
+	std::ifstream file(BinaryPathFinder::GetBinaryDir() / "migrations/" / type / dbType / name);
+
+	if (file.is_open()) {
+		std::string line;
+		std::string total = "";
+
+		while (std::getline(file, line)) {
+			total += line;
+		}
+
+		file.close();
+
+		migration.name = ;
+		migration.data = total;
+	}
+
+	return migration;
+}
+
+void MigrationManager::RunSQLiteMigrations() {
+	auto cdstmt = CDClientDatabase::CreatePreppedStmt("CREATE TABLE IF NOT EXISTS migration_history (name TEXT NOT NULL, date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP);");
+	cdstmt.execQuery().finalize();
+	cdstmt.finalize();
+
+	for (const auto& entry : GeneralUtils::GetSqlFileNamesFromFolder((BinaryPathFinder::GetBinaryDir() / "migrations/cdserver/").string())) {
+		auto migration = Migration::LoadMigration("cdserver", "", entry);
+
+		if (migration.data.empty()) continue;
+
+		// Check if there is an entry in the migration history table on the cdclient database.
+		cdstmt = CDClientDatabase::CreatePreppedStmt("SELECT name FROM migration_history WHERE name = ?;");
+		cdstmt.bind((int32_t)1, migration.name.c_str());
+		auto cdres = cdstmt.execQuery();
+		bool doExit = !cdres.eof();
+		cdres.finalize();
+		cdstmt.finalize();
+
+		if (doExit) continue;
+
+		// Check first if there is entry in the migration history table on the main database.
+		stmt = Database::CreatePreppedStmt("SELECT name FROM migration_history WHERE name = ?;");
+		stmt->setString(1, migration.name.c_str());
+		auto* res = stmt->executeQuery();
+		doExit = res->next();
+		delete res;
+		delete stmt;
+		if (doExit) {
+			// Insert into cdclient database if there is an entry in the main database but not the cdclient database.
+			cdstmt = CDClientDatabase::CreatePreppedStmt("INSERT INTO migration_history (name) VALUES (?);");
+			cdstmt.bind((int32_t)1, migration.name.c_str());
+			cdstmt.execQuery().finalize();
+			cdstmt.finalize();
+			continue;
+		}
+
+		// Doing these 1 migration at a time since one takes a long time and some may think it is crashing.
+		// This will at the least guarentee that the full migration needs to be run in order to be counted as "migrated".
+		Game::logger->Log("MigrationRunner", "Executing migration: %s.  This may take a while.  Do not shut down server.", migration.name.c_str());
+		CDClientDatabase::ExecuteQuery("BEGIN TRANSACTION;");
+		for (const auto& dml : GeneralUtils::SplitString(migration.data, ';')) {
+			if (dml.empty()) continue;
+			try {
+				CDClientDatabase::ExecuteDML(dml.c_str());
+			} catch (CppSQLite3Exception& e) {
+				Game::logger->Log("MigrationRunner", "Encountered error running DML command: (%i) : %s", e.errorCode(), e.errorMessage());
+			}
+		}
+
+		// Insert into cdclient database.
+		cdstmt = CDClientDatabase::CreatePreppedStmt("INSERT INTO migration_history (name) VALUES (?);");
+		cdstmt.bind((int32_t)1, migration.name.c_str());
+		cdstmt.execQuery().finalize();
+		cdstmt.finalize();
+		CDClientDatabase::ExecuteQuery("COMMIT;");
+	}
+
+	Game::logger->Log("MigrationRunner", "CDServer database is up to date.");
+}

dDatabase/Databases/Migrations/MigrationManager.h

@@ -0,0 +1,17 @@
+#pragma once
+
+class DatabaseBase;
+
+struct Migration {
+	std::string data;
+	std::string name;
+
+	static Migration LoadMigration(const std::string& type, const std::string& dbType, const std::string& name);
+};
+
+class MigrationManager {
+public:
+	void RunMigrations(DatabaseBase* db);
+	void RunSQLiteMigrations();
+private:
+};

dDatabase/Databases/Migrations/MySQLMigration.cpp

@@ -0,0 +1,241 @@
+#include "MySQLMigration.h"
+
+#include <istream>
+
+#include "CDClientDatabase.h"
+#include "Game.h"
+#include "GeneralUtils.h"
+#include "dLogger.h"
+#include "BinaryPathFinder.h"
+#include "ZCompression.h"
+#include "tinyxml2.h"
+
+#include "../MySQL.h"
+
+void MigrationRunner::RunMigrations(DatabaseBase* db) {
+	auto database = dynamic_cast<MySQLDatabase*>(db);
+
+	auto* stmt = database->CreatePreppedStmt("CREATE TABLE IF NOT EXISTS migration_history (name TEXT NOT NULL, date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP());");
+	stmt->execute();
+	delete stmt;
+
+	sql::SQLString finalSQL = "";
+	bool runSd0Migrations = false;
+	for (const auto& entry : GeneralUtils::GetSqlFileNamesFromFolder((BinaryPathFinder::GetBinaryDir() / "./migrations/dlu/mysql").string())) {
+		auto migration = Migration::LoadMigration("dlu", "mysql", entry);
+
+		if (migration.data.empty()) {
+			continue;
+		}
+
+		stmt = database->CreatePreppedStmt("SELECT name FROM migration_history WHERE name = ?;");
+		stmt->setString(1, migration.name.c_str());
+		auto* res = stmt->executeQuery();
+		bool doExit = res->next();
+		delete res;
+		delete stmt;
+		if (doExit) continue;
+
+		Game::logger->Log("MigrationRunner", "Running migration: %s", migration.name.c_str());
+		if (migration.name == "dlu/5_brick_model_sd0.sql") {
+			runSd0Migrations = true;
+		} else {
+			finalSQL.append(migration.data.c_str());
+		}
+
+		stmt = database->CreatePreppedStmt("INSERT INTO migration_history (name) VALUES (?);");
+		stmt->setString(1, migration.name.c_str());
+		stmt->execute();
+		delete stmt;
+	}
+
+	if (finalSQL.empty() && !runSd0Migrations) {
+		Game::logger->Log("MigrationRunner", "Server database is up to date.");
+		return;
+	}
+
+	if (!finalSQL.empty()) {
+		auto migration = GeneralUtils::SplitString(static_cast<std::string>(finalSQL), ';');
+		std::unique_ptr<sql::Statement> simpleStatement(Database::CreateStmt());
+		for (auto& query : migration) {
+			try {
+				if (query.empty()) continue;
+				simpleStatement->execute(query.c_str());
+			} catch (sql::SQLException& e) {
+				Game::logger->Log("MigrationRunner", "Encountered error running migration: %s", e.what());
+			}
+		}
+	}
+
+	// Do this last on the off chance none of the other migrations have been run yet.
+	if (runSd0Migrations) {
+		uint32_t numberOfUpdatedModels = UpdateBrickByBrickModelsToSd0();
+		Game::logger->Log("MasterServer", "%i models were updated from zlib to sd0.", numberOfUpdatedModels);
+		
+		uint32_t numberOfTruncatedModels = TruncateBrokenBrickByBrickXml();
+		Game::logger->Log("MasterServer", "%i models were truncated from the database.", numberOfTruncatedModels);
+	}
+}
+
+
+uint32_t TruncateBrokenBrickByBrickXml(MySQLDatabase* database) {
+	uint32_t modelsTruncated{};
+	auto modelsToTruncate = GetModelsFromDatabase(database);
+	
+	bool previousCommitValue = database->GetAutoCommit();
+	database->SetAutoCommit(false);
+	
+	while (modelsToTruncate->next()) {
+		std::unique_ptr<sql::PreparedStatement> ugcModelToDelete(database->CreatePreppedStmt("DELETE FROM ugc WHERE ugc.id = ?;"));
+		std::unique_ptr<sql::PreparedStatement> pcModelToDelete(database->CreatePreppedStmt("DELETE FROM properties_contents WHERE ugc_id = ?;"));
+		std::string completeUncompressedModel{};
+		uint32_t chunkCount{};
+		uint64_t modelId = modelsToTruncate->getInt(1);
+		std::unique_ptr<sql::Blob> modelAsSd0(modelsToTruncate->getBlob(2));
+		// Check that header is sd0 by checking for the sd0 magic.
+		if (CheckSd0Magic(modelAsSd0.get())) {
+			while (true) {
+				uint32_t chunkSize{};
+				modelAsSd0->read(reinterpret_cast<char*>(&chunkSize), sizeof(uint32_t)); // Extract chunk size from istream
+
+				// Check if good here since if at the end of an sd0 file, this will have eof flagged.
+				if (!modelAsSd0->good()) break;
+
+				std::unique_ptr<uint8_t[]> compressedChunk(new uint8_t[chunkSize]);
+				for (uint32_t i = 0; i < chunkSize; i++) {
+					compressedChunk[i] = modelAsSd0->get();
+				}
+
+				// Ignore the valgrind warning about uninitialized values.  These are discarded later when we know the actual uncompressed size.
+				std::unique_ptr<uint8_t[]> uncompressedChunk(new uint8_t[ZCompression::MAX_SD0_CHUNK_SIZE]);
+				int32_t err{};
+				int32_t actualUncompressedSize = ZCompression::Decompress(
+					compressedChunk.get(), chunkSize, uncompressedChunk.get(), ZCompression::MAX_SD0_CHUNK_SIZE, err);
+
+				if (actualUncompressedSize != -1) {
+					uint32_t previousSize = completeUncompressedModel.size();
+					completeUncompressedModel.append((char*)uncompressedChunk.get());
+					completeUncompressedModel.resize(previousSize + actualUncompressedSize);
+				} else {
+					Game::logger->Log("BrickByBrickFix", "Failed to inflate chunk %i for model %llu.  Error: %i", chunkCount, modelId, err);
+					break;
+				}
+				chunkCount++;
+			}
+			std::unique_ptr<tinyxml2::XMLDocument> document = std::make_unique<tinyxml2::XMLDocument>();
+			if (!document) {
+				Game::logger->Log("BrickByBrickFix", "Failed to initialize tinyxml document.  Aborting.");
+				return 0;
+			}
+
+			if (!(document->Parse(completeUncompressedModel.c_str(), completeUncompressedModel.size()) == tinyxml2::XML_SUCCESS)) {
+				if (completeUncompressedModel.find(
+					"</LXFML>",
+					completeUncompressedModel.length() >= 15 ? completeUncompressedModel.length() - 15 : 0) == std::string::npos
+					) {
+					Game::logger->Log("BrickByBrickFix",
+						"Brick-by-brick model %llu will be deleted!", modelId);
+					ugcModelToDelete->setInt64(1, modelsToTruncate->getInt64(1));
+					pcModelToDelete->setInt64(1, modelsToTruncate->getInt64(1));
+					ugcModelToDelete->execute();
+					pcModelToDelete->execute();
+					modelsTruncated++;
+				}
+			}
+		} else {
+			Game::logger->Log("BrickByBrickFix",
+				"Brick-by-brick model %llu will be deleted!", modelId);
+			ugcModelToDelete->setInt64(1, modelsToTruncate->getInt64(1));
+			pcModelToDelete->setInt64(1, modelsToTruncate->getInt64(1));
+			ugcModelToDelete->execute();
+			pcModelToDelete->execute();
+			modelsTruncated++;
+		}
+	}
+
+	database->Commit();
+	database->SetAutoCommit(previousCommitValue);
+	return modelsTruncated;
+}
+
+/**
+ * @brief Updates all current models in the database to have the Segmented Data 0 (SD0) format.
+ * Any models that do not start with zlib and best compression magic will not be updated.
+ *
+ * @return The number of models updated to SD0
+ */
+uint32_t UpdateBrickByBrickModelsToSd0(MySQLDatabase* database) {
+	uint32_t updatedModels = 0;
+	auto modelsToUpdate = GetModelsFromDatabase();
+	
+	auto previousAutoCommitState = database->GetAutoCommit();
+	database->SetAutoCommit(false);
+	
+	std::unique_ptr<sql::PreparedStatement> insertionStatement(database->CreatePreppedStmt("UPDATE ugc SET lxfml = ? WHERE id = ?;"));
+	while (modelsToUpdate->next()) {
+		int64_t modelId = modelsToUpdate->getInt64(1);
+		std::unique_ptr<sql::Blob> oldLxfml(modelsToUpdate->getBlob(2));
+		// Check if the stored blob starts with zlib magic (0x78 0xDA - best compression of zlib)
+		// If it does, convert it to sd0.
+		if (oldLxfml->get() == 0x78 && oldLxfml->get() == 0xDA) {
+
+			// Get and save size of zlib compressed chunk.
+			oldLxfml->seekg(0, std::ios::end);
+			uint32_t oldLxfmlSize = static_cast<uint32_t>(oldLxfml->tellg());
+			oldLxfml->seekg(0);
+
+			// Allocate 9 extra bytes.  5 for sd0 magic, 4 for the only zlib compressed size.
+			uint32_t oldLxfmlSizeWithHeader = oldLxfmlSize + 9;
+			std::unique_ptr<char[]> sd0ConvertedModel(new char[oldLxfmlSizeWithHeader]);
+
+			WriteSd0Magic(sd0ConvertedModel.get(), oldLxfmlSize);
+			for (uint32_t i = 9; i < oldLxfmlSizeWithHeader; i++) {
+				sd0ConvertedModel.get()[i] = oldLxfml->get();
+			}
+
+			std::string outputString(sd0ConvertedModel.get(), oldLxfmlSizeWithHeader);
+			std::istringstream outputStringStream(outputString);
+
+			insertionStatement->setBlob(1, static_cast<std::istream*>(&outputStringStream));
+			insertionStatement->setInt64(2, modelId);
+			try {
+				insertionStatement->executeUpdate();
+				Game::logger->Log("BrickByBrickFix", "Updated model %i to sd0", modelId);
+				updatedModels++;
+			} catch (sql::SQLException exception) {
+				Game::logger->Log(
+					"BrickByBrickFix",
+					"Failed to update model %i.  This model should be inspected manually to see why."
+					"The database error is %s", modelId, exception.what());
+			}
+		}
+	}
+	
+	database->Commit();
+	database->SetAutoCommit(previousAutoCommitState);
+	return updatedModels;
+}
+
+std::unique_ptr<sql::ResultSet> GetModelsFromDatabase(MySQLDatabase* database) {
+	std::unique_ptr<sql::PreparedStatement> modelsRawDataQuery(database->CreatePreppedStmt("SELECT id, lxfml FROM ugc;"));
+	return std::unique_ptr<sql::ResultSet>(modelsRawDataQuery->executeQuery());
+}
+
+/**
+ * @brief Writes sd0 magic at the front of a char*
+ *
+ * @param input the char* to write at the front of
+ * @param chunkSize The size of the first chunk to write the size of
+ */
+void WriteSd0Magic(char* input, uint32_t chunkSize) {
+	input[0] = 's';
+	input[1] = 'd';
+	input[2] = '0';
+	input[3] = 0x01;
+	input[4] = 0xFF;
+	*reinterpret_cast<uint32_t*>(input + 5) = chunkSize; // Write the integer to the character array
+}
+
+bool CheckSd0Magic(sql::Blob* streamToCheck) {
+	return streamToCheck->get() == 's' && streamToCheck->get() == 'd' && streamToCheck->get() == '0' && streamToCheck->get() == 0x01 && streamToCheck->get() == 0xFF;
+}

dDatabase/Databases/Migrations/MySQLMigration.h

@@ -0,0 +1,13 @@
+#pragma once
+
+#include <string>
+
+#include "MigrationManager.h"
+
+class MySQLMigrationManager : public MigrationManager {
+public:
+	void RunMigrations(DatabaseBase* db) override;
+	void RunSQLiteMigrations() override;
+private:
+
+};

dDatabase/MigrationRunner.cpp

@@ -1,158 +0,0 @@
-#include "MigrationRunner.h"
-
-#include "BrickByBrickFix.h"
-#include "CDClientDatabase.h"
-#include "Database.h"
-#include "Game.h"
-#include "GeneralUtils.h"
-#include "dLogger.h"
-#include "BinaryPathFinder.h"
-
-#include <istream>
-
-Migration LoadMigration(std::string path) {
-	Migration migration{};
-	std::ifstream file(BinaryPathFinder::GetBinaryDir() / "migrations/" / path);
-
-	if (file.is_open()) {
-		std::string line;
-		std::string total = "";
-
-		while (std::getline(file, line)) {
-			total += line;
-		}
-
-		file.close();
-
-		migration.name = path;
-		migration.data = total;
-	}
-
-	return migration;
-}
-
-void MigrationRunner::RunMigrations() {
-	auto* stmt = Database::CreatePreppedStmt("CREATE TABLE IF NOT EXISTS migration_history (name TEXT NOT NULL, date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP());");
-	stmt->execute();
-	delete stmt;
-
-	sql::SQLString finalSQL = "";
-	bool runSd0Migrations = false;
-	for (const auto& entry : GeneralUtils::GetSqlFileNamesFromFolder((BinaryPathFinder::GetBinaryDir() / "./migrations/dlu/").string())) {
-		auto migration = LoadMigration("dlu/" + entry);
-
-		if (migration.data.empty()) {
-			continue;
-		}
-
-		stmt = Database::CreatePreppedStmt("SELECT name FROM migration_history WHERE name = ?;");
-		stmt->setString(1, migration.name.c_str());
-		auto* res = stmt->executeQuery();
-		bool doExit = res->next();
-		delete res;
-		delete stmt;
-		if (doExit) continue;
-
-		Game::logger->Log("MigrationRunner", "Running migration: %s", migration.name.c_str());
-		if (migration.name == "dlu/5_brick_model_sd0.sql") {
-			runSd0Migrations = true;
-		} else {
-			finalSQL.append(migration.data.c_str());
-		}
-
-		stmt = Database::CreatePreppedStmt("INSERT INTO migration_history (name) VALUES (?);");
-		stmt->setString(1, migration.name.c_str());
-		stmt->execute();
-		delete stmt;
-	}
-
-	if (finalSQL.empty() && !runSd0Migrations) {
-		Game::logger->Log("MigrationRunner", "Server database is up to date.");
-		return;
-	}
-
-	if (!finalSQL.empty()) {
-		auto migration = GeneralUtils::SplitString(static_cast<std::string>(finalSQL), ';');
-		std::unique_ptr<sql::Statement> simpleStatement(Database::CreateStmt());
-		for (auto& query : migration) {
-			try {
-				if (query.empty()) continue;
-				simpleStatement->execute(query.c_str());
-			} catch (sql::SQLException& e) {
-				Game::logger->Log("MigrationRunner", "Encountered error running migration: %s", e.what());
-			}
-		}
-	}
-
-	// Do this last on the off chance none of the other migrations have been run yet.
-	if (runSd0Migrations) {
-		uint32_t numberOfUpdatedModels = BrickByBrickFix::UpdateBrickByBrickModelsToSd0();
-		Game::logger->Log("MasterServer", "%i models were updated from zlib to sd0.", numberOfUpdatedModels);
-		uint32_t numberOfTruncatedModels = BrickByBrickFix::TruncateBrokenBrickByBrickXml();
-		Game::logger->Log("MasterServer", "%i models were truncated from the database.", numberOfTruncatedModels);
-	}
-}
-
-void MigrationRunner::RunSQLiteMigrations() {
-	auto cdstmt = CDClientDatabase::CreatePreppedStmt("CREATE TABLE IF NOT EXISTS migration_history (name TEXT NOT NULL, date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP);");
-	cdstmt.execQuery().finalize();
-	cdstmt.finalize();
-
-	auto* stmt = Database::CreatePreppedStmt("CREATE TABLE IF NOT EXISTS migration_history (name TEXT NOT NULL, date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP());");
-	stmt->execute();
-	delete stmt;
-
-	for (const auto& entry : GeneralUtils::GetSqlFileNamesFromFolder((BinaryPathFinder::GetBinaryDir() / "migrations/cdserver/").string())) {
-		auto migration = LoadMigration("cdserver/" + entry);
-
-		if (migration.data.empty()) continue;
-
-		// Check if there is an entry in the migration history table on the cdclient database.
-		cdstmt = CDClientDatabase::CreatePreppedStmt("SELECT name FROM migration_history WHERE name = ?;");
-		cdstmt.bind((int32_t) 1, migration.name.c_str());
-		auto cdres = cdstmt.execQuery();
-		bool doExit = !cdres.eof();
-		cdres.finalize();
-		cdstmt.finalize();
-
-		if (doExit) continue;
-
-		// Check first if there is entry in the migration history table on the main database.
-		stmt = Database::CreatePreppedStmt("SELECT name FROM migration_history WHERE name = ?;");
-		stmt->setString(1, migration.name.c_str());
-		auto* res = stmt->executeQuery();
-		doExit = res->next();
-		delete res;
-		delete stmt;
-		if (doExit) {
-			// Insert into cdclient database if there is an entry in the main database but not the cdclient database.
-			cdstmt = CDClientDatabase::CreatePreppedStmt("INSERT INTO migration_history (name) VALUES (?);");
-			cdstmt.bind((int32_t) 1, migration.name.c_str());
-			cdstmt.execQuery().finalize();
-			cdstmt.finalize();
-			continue;
-		}
-
-		// Doing these 1 migration at a time since one takes a long time and some may think it is crashing.
-		// This will at the least guarentee that the full migration needs to be run in order to be counted as "migrated".
-		Game::logger->Log("MigrationRunner", "Executing migration: %s.  This may take a while.  Do not shut down server.", migration.name.c_str());
-		CDClientDatabase::ExecuteQuery("BEGIN TRANSACTION;");
-		for (const auto& dml : GeneralUtils::SplitString(migration.data, ';')) {
-			if (dml.empty()) continue;
-			try {
-				CDClientDatabase::ExecuteDML(dml.c_str());
-			} catch (CppSQLite3Exception& e) {
-				Game::logger->Log("MigrationRunner", "Encountered error running DML command: (%i) : %s", e.errorCode(), e.errorMessage());
-			}
-		}
-
-		// Insert into cdclient database.
-		cdstmt = CDClientDatabase::CreatePreppedStmt("INSERT INTO migration_history (name) VALUES (?);");
-		cdstmt.bind((int32_t) 1, migration.name.c_str());
-		cdstmt.execQuery().finalize();
-		cdstmt.finalize();
-		CDClientDatabase::ExecuteQuery("COMMIT;");
-	}
-
-	Game::logger->Log("MigrationRunner", "CDServer database is up to date.");
-}

dDatabase/MigrationRunner.h

@@ -1,13 +0,0 @@
-#pragma once
-
-#include <string>
-
-struct Migration {
-	std::string data;
-	std::string name;
-};
-
-namespace MigrationRunner {
-	void RunMigrations();
-	void RunSQLiteMigrations();
-};