Add automatic cdclient migration runner support and setup (#789)

* Add automatic migrations for CDServer

Add support to automatically migrate and update CDServers with new migrations.  Also adds support to simplify the setup process by simply putting the fdb in the res folder and letting the server convert it to sqlite.

This reduces the amount of back and forth when setting up a server.

* Remove transaction language

* Add DML execution
`poggers`
Add a way to execute DML commands through the sqlite connection on the server.

* Make DML Commands more robust

On the off chance the server is shutdown before the whole migration is run, lets just not add it to our "finished list" until the whole file is done.

* Update README

CMakeLists.txt

@@ -108,13 +108,25 @@ foreach(file ${VANITY_FILES})
 endforeach()
 
 # Move our migrations for MasterServer to run
-file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/migrations/)
+file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/migrations/dlu/)
 file(GLOB SQL_FILES ${CMAKE_SOURCE_DIR}/migrations/dlu/*.sql)
 foreach(file ${SQL_FILES})
 	get_filename_component(file ${file} NAME)
-	if (NOT EXISTS ${PROJECT_BINARY_DIR}/migrations/${file})
+	if (NOT EXISTS ${PROJECT_BINARY_DIR}/migrations/dlu/${file})
 		configure_file(
-			${CMAKE_SOURCE_DIR}/migrations/dlu/${file} ${PROJECT_BINARY_DIR}/migrations/${file}
+			${CMAKE_SOURCE_DIR}/migrations/dlu/${file} ${PROJECT_BINARY_DIR}/migrations/dlu/${file}
+			COPYONLY
+		)
+	endif()
+endforeach()
+
+file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/migrations/cdserver/)
+file(GLOB SQL_FILES ${CMAKE_SOURCE_DIR}/migrations/cdserver/*.sql)
+foreach(file ${SQL_FILES})
+	get_filename_component(file ${file} NAME)
+	if (NOT EXISTS ${PROJECT_BINARY_DIR}/migrations/cdserver/${file})
+		configure_file(
+			${CMAKE_SOURCE_DIR}/migrations/cdserver/${file} ${PROJECT_BINARY_DIR}/migrations/cdserver/${file}
 			COPYONLY
 		)
 	endif()

README.md

@@ -196,9 +196,10 @@ certutil -hashfile <file> SHA256
 * Copy over or create symlinks from `locale.xml` in your client `locale` directory to the `build/locale` directory
 
 #### Client database
-* Use `fdb_to_sqlite.py` in lcdr's utilities on `res/cdclient.fdb` in the unpacked client to convert the client database to `cdclient.sqlite`
+* Move the file `res/cdclient.fdb` from the unpacked client to the `build/res` folder on the server.
-* Move and rename `cdclient.sqlite` into `build/res/CDServer.sqlite`
+* The server will automatically copy and convert the file from fdb to sqlite should `CDServer.sqlite` not already exist.
-* Run each SQL file in the order at which they appear [here](migrations/cdserver/) on the SQLite database
+* You can also convert the database manually using `fdb_to_sqlite.py` using lcdr's utilities.  Just make sure to rename the file to `CDServer.sqlite` instead of `cdclient.sqlite`.
+* Migrations to the database are automatically run on server start.  When migrations are needed to be ran, the server may take a bit longer to start.
 
 ### Database
 Darkflame Universe utilizes a MySQL/MariaDB database for account and character information.
@@ -229,7 +230,7 @@ Your build directory should now look like this:
 * **locale/**
   * locale.xml
 * **res/**
-  * CDServer.sqlite
+  * cdclient.fdb
   * chatplus_en_us.txt
   * **macros/**
     * ...

dDatabase/CDClientDatabase.cpp

@@ -14,6 +14,11 @@ CppSQLite3Query CDClientDatabase::ExecuteQuery(const std::string& query) {
 	return conn->execQuery(query.c_str());
 }
 
+//! Updates the CDClient file with Data Manipulation Language (DML) commands.
+int CDClientDatabase::ExecuteDML(const std::string& query) {
+	return conn->execDML(query.c_str());
+}
+
 //! Makes prepared statements
 CppSQLite3Statement CDClientDatabase::CreatePreppedStmt(const std::string& query) {
 	return conn->compileStatement(query.c_str());

dDatabase/CDClientDatabase.h

@@ -40,6 +40,14 @@ namespace CDClientDatabase {
 	 */
 	CppSQLite3Query ExecuteQuery(const std::string& query);
 
+	//! Updates the CDClient file with Data Manipulation Language (DML) commands.
+	/*!
+	  \param query The DML command to run.  DML command can be multiple queries in one string but only
+		the last one will return its number of updated rows.
+	  \return The number of updated rows.
+	*/
+	int ExecuteDML(const std::string& query);
+
 	//! Queries the CDClient and parses arguments
 	/*!
 	  \param query The query with formatted arguments

dDatabase/MigrationRunner.cpp

@@ -1,11 +1,34 @@
 #include "MigrationRunner.h"
 
 #include "BrickByBrickFix.h"
+#include "CDClientDatabase.h"
+#include "Database.h"
+#include "Game.h"
 #include "GeneralUtils.h"
+#include "dLogger.h"
 
-#include <fstream>
+#include <istream>
-#include <algorithm>
+
-#include <thread>
+Migration LoadMigration(std::string path) {
+	Migration migration{};
+	std::ifstream file("./migrations/" + path);
+
+	if (file.is_open()) {
+		std::string line;
+		std::string total = "";
+
+		while (std::getline(file, line)) {
+			total += line;
+		}
+
+		file.close();
+
+		migration.name = path;
+		migration.data = total;
+	}
+
+	return migration;
+}
 
 void MigrationRunner::RunMigrations() {
 	auto* stmt = Database::CreatePreppedStmt("CREATE TABLE IF NOT EXISTS migration_history (name TEXT NOT NULL, date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP());");
@@ -13,17 +36,14 @@ void MigrationRunner::RunMigrations() {
 	delete stmt;
 
 	sql::SQLString finalSQL = "";
-	Migration checkMigration{};
 	bool runSd0Migrations = false;
-	for (const auto& entry : GeneralUtils::GetFileNamesFromFolder("./migrations/")) {
+	for (const auto& entry : GeneralUtils::GetFileNamesFromFolder("./migrations/dlu/")) {
-		auto migration = LoadMigration(entry);
+		auto migration = LoadMigration("dlu/" + entry);
 
 		if (migration.data.empty()) {
 			continue;
 		}
 
-		checkMigration = migration;
-
 		stmt = Database::CreatePreppedStmt("SELECT name FROM migration_history WHERE name = ?;");
 		stmt->setString(1, migration.name);
 		auto* res = stmt->executeQuery();
@@ -40,7 +60,7 @@ void MigrationRunner::RunMigrations() {
 		}
 
 		stmt = Database::CreatePreppedStmt("INSERT INTO migration_history (name) VALUES (?);");
-		stmt->setString(1, entry);
+		stmt->setString(1, migration.name);
 		stmt->execute();
 		delete stmt;
 	}
@@ -72,23 +92,39 @@ void MigrationRunner::RunMigrations() {
 	}
 }
 
-Migration MigrationRunner::LoadMigration(std::string path) {
+void MigrationRunner::RunSQLiteMigrations() {
-	Migration migration{};
+	auto* stmt = Database::CreatePreppedStmt("CREATE TABLE IF NOT EXISTS migration_history (name TEXT NOT NULL, date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP());");
-	std::ifstream file("./migrations/" + path);
+	stmt->execute();
+	delete stmt;
 
-	if (file.is_open()) {
+	for (const auto& entry : GeneralUtils::GetFileNamesFromFolder("./migrations/cdserver/")) {
-		std::string line;
+		auto migration = LoadMigration("cdserver/" + entry);
-		std::string total = "";
 
-		while (std::getline(file, line)) {
+		if (migration.data.empty()) continue;
-			total += line;
+
+		stmt = Database::CreatePreppedStmt("SELECT name FROM migration_history WHERE name = ?;");
+		stmt->setString(1, migration.name);
+		auto* res = stmt->executeQuery();
+		bool doExit = res->next();
+		delete res;
+		delete stmt;
+		if (doExit) continue;
+
+		// Doing these 1 migration at a time since one takes a long time and some may think it is crashing.
+		// This will at the least guarentee that the full migration needs to be run in order to be counted as "migrated".
+		Game::logger->Log("MigrationRunner", "Executing migration: %s.  This may take a while.  Do not shut down server.", migration.name.c_str());
+		for (const auto& dml : GeneralUtils::SplitString(migration.data, ';')) {
+			if (dml.empty()) continue;
+			try {
+				CDClientDatabase::ExecuteDML(dml.c_str());
+			} catch (CppSQLite3Exception& e) {
+				Game::logger->Log("MigrationRunner", "Encountered error running DML command: (%i) : %s", e.errorCode(), e.errorMessage());
+			}
 		}
-
+		stmt = Database::CreatePreppedStmt("INSERT INTO migration_history (name) VALUES (?);");
-		file.close();
+		stmt->setString(1, migration.name);
-
+		stmt->execute();
-		migration.name = path;
+		delete stmt;
-		migration.data = total;
 	}
-
+	Game::logger->Log("MigrationRunner", "CDServer database is up to date.");
-	return migration;
 }

dDatabase/MigrationRunner.h

@@ -1,19 +1,13 @@
 #pragma once
 
-#include "Database.h"
+#include <string>
-
-#include "dCommonVars.h"
-#include "Game.h"
-#include "dCommonVars.h"
-#include "dLogger.h"
 
 struct Migration {
 	std::string data;
 	std::string name;
 };
 
-class MigrationRunner {
+namespace MigrationRunner {
-public:
+	void RunMigrations();
-	static void RunMigrations();
+	void RunSQLiteMigrations();
-	static Migration LoadMigration(std::string path);
 };

dMasterServer/MasterServer.cpp

@@ -105,10 +105,34 @@ int main(int argc, char** argv) {
 	const std::string cdclient_path = "./res/CDServer.sqlite";
 	std::ifstream cdclient_fd(cdclient_path);
 	if (!cdclient_fd.good()) {
-		Game::logger->Log("WorldServer", "%s could not be opened", cdclient_path.c_str());
+		Game::logger->Log("WorldServer", "%s could not be opened.  Looking for cdclient.fdb to convert to sqlite.", cdclient_path.c_str());
-		return EXIT_FAILURE;
+		cdclient_fd.close();
+
+		const std::string cdclientFdbPath = "./res/cdclient.fdb";
+		cdclient_fd.open(cdclientFdbPath);
+		if (!cdclient_fd.good()) {
+			Game::logger->Log(
+				"WorldServer", "%s could not be opened."
+				"Please move a cdclient.fdb or an already converted database to build/res.", cdclientFdbPath.c_str());
+			return EXIT_FAILURE;
+		}
+		Game::logger->Log("WorldServer", "Found %s.  Clearing cdserver migration_history then copying and converting to sqlite.", cdclientFdbPath.c_str());
+		auto stmt = Database::CreatePreppedStmt(R"#(DELETE FROM migration_history WHERE name LIKE "%cdserver%";)#");
+		stmt->executeUpdate();
+		delete stmt;
+		cdclient_fd.close();
+
+		std::string res = "python3 ../thirdparty/docker-utils/utils/fdb_to_sqlite.py " + cdclientFdbPath;
+		int r = system(res.c_str());
+		if (r != 0) {
+			Game::logger->Log("MasterServer", "Failed to convert fdb to sqlite");
+			return EXIT_FAILURE;
+		}
+		if (std::rename("./cdclient.sqlite", "./res/CDServer.sqlite") != 0) {
+			Game::logger->Log("MasterServer", "failed to move cdclient file.");
+			return EXIT_FAILURE;
+		}
 	}
-	cdclient_fd.close();
 
 	//Connect to CDClient
 	try {
@@ -120,6 +144,9 @@ int main(int argc, char** argv) {
 		return EXIT_FAILURE;
 	}
 
+	// Run migrations should any need to be run.
+	MigrationRunner::RunSQLiteMigrations();
+
 	//Get CDClient initial information
 	try {
 		CDClientManager::Instance()->Initialize();

migrations/cdserver/0_nt_footrace.sql

@@ -1,6 +1,2 @@
-BEGIN TRANSACTION;
-
 UPDATE ComponentsRegistry SET component_id = 1901 WHERE id = 12916 AND component_type = 39;
 INSERT INTO ActivityRewards (objectTemplate, ActivityRewardIndex, activityRating, LootMatrixIndex, CurrencyIndex, ChallengeRating, description) VALUES (1901, 166, -1, 598, 1, 4, 'NT Foot Race');
-
-COMMIT;