Add Sd0Conversion

Add brick by brick conversion commands with 2 parameters to tell the program what to do with the data.

Add zlib -> sd0 conversion.  Files look good at a glance but should be tested in game to ensure stability.  Tests to come.

dCommon/BrickByBrickFix.cpp

@@ -0,0 +1,88 @@
+#include <sstream>
+#include <memory>
+
+#include "BrickByBrickFix.h"
+#include "Database.h"
+#include "dLogger.h"
+#include "Game.h"
+
+//! Forward declarations
+
+std::unique_ptr<sql::ResultSet> GetModelsFromDatabase();
+void WriteSd0Magic(char* input);
+
+uint32_t BrickByBrickFix::TruncateBrokenBrickByBrickXml() {
+	auto modelsToTruncate = GetModelsFromDatabase();
+	if (modelsToTruncate->next()) {
+		// Decompress with zlib and attempt to convert to xml.  If that fails this model is broken.
+	}
+	return 0;
+}
+
+uint32_t BrickByBrickFix::UpdateBrickByBrickModelsToSd0() {
+	uint32_t updatedModels = 0;
+	auto modelsToUpdate = GetModelsFromDatabase();
+	auto previousAutoCommitState = Database::GetAutoCommit();
+	Database::SetAutoCommit(false);
+	std::unique_ptr<sql::PreparedStatement> insertionStatement(Database::CreatePreppedStmt("UPDATE ugc SET lxfml = ? WHERE id = ?;"));
+	while (modelsToUpdate->next()) {
+		uint32_t modelId = modelsToUpdate->getInt(1);
+		auto oldLxfml = modelsToUpdate->getBlob(2);
+		// Check if the stored blob starts with zlib magic (0x78).  If it does, convert it to sd0.
+		if (oldLxfml->get() == 0x78) {
+			Game::logger->Log("BrickByBrickFix", "Updating model %i", modelId);
+
+			// Get and save size of zlib compressed chunk.
+			oldLxfml->seekg(0, std::ios::end);
+			uint32_t oldLxfmlSize = static_cast<uint32_t>(oldLxfml->tellg());
+			oldLxfml->seekg(0);
+
+			// Allocate 9 extra bytes.  5 for sd0 magic, 4 for the only zlib compressed size.
+			uint32_t oldLxfmlSizeWithHeader = oldLxfmlSize + 9;
+			char* sd0ConvertedModel = static_cast<char*>(std::malloc(oldLxfmlSizeWithHeader));
+
+			WriteSd0Magic(sd0ConvertedModel);
+			// zlib compressed chunk size
+			sd0ConvertedModel[5] = static_cast<uint8_t>(oldLxfmlSize);
+			sd0ConvertedModel[6] = static_cast<uint8_t>(oldLxfmlSize >> 8U);
+			sd0ConvertedModel[7] = static_cast<uint8_t>(oldLxfmlSize >> 16U);
+			sd0ConvertedModel[8] = static_cast<uint8_t>(oldLxfmlSize >> 24U);
+			for (uint32_t i = 9; i < oldLxfmlSizeWithHeader; i++) {
+				sd0ConvertedModel[i] = oldLxfml->get();
+			}
+
+			std::string outputString(sd0ConvertedModel, oldLxfmlSizeWithHeader);
+			std::istringstream outputStringStream(outputString);
+
+			insertionStatement->setBlob(1, static_cast<std::istream*>(&outputStringStream));
+			insertionStatement->setInt(2, modelId);
+			try {
+				insertionStatement->executeUpdate();
+				Game::logger->Log("BrickByBrickFix", "Updated model %i", modelId);
+				updatedModels++;
+			} catch (sql::SQLException exception) {
+				Game::logger->Log(
+					"BrickByBrickFix",
+					"Failed to update model %i.  This model should be inspected manually to see why."
+					"The database error is %s", modelId, exception.what());
+			}
+			free(sd0ConvertedModel);
+		}
+	}
+	Database::Commit();
+	Database::SetAutoCommit(previousAutoCommitState);
+	return updatedModels;
+}
+
+std::unique_ptr<sql::ResultSet> GetModelsFromDatabase() {
+	std::unique_ptr<sql::PreparedStatement> modelsRawDataQuery(Database::CreatePreppedStmt("SELECT id, lxfml FROM ugc;"));
+	return std::unique_ptr<sql::ResultSet>(modelsRawDataQuery->executeQuery());
+}
+
+void WriteSd0Magic(char* input) {
+	input[0] = 's';
+	input[1] = 'd';
+	input[2] = '0';
+	input[3] = 0x01;
+	input[4] = 0xFF;
+}

dCommon/BrickByBrickFix.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <cstdint>
+
+class BrickByBrickFix {
+	public:
+		/**
+		 * @brief Deletes all broken BrickByBrick models that have invalid XML
+		 *
+		 * @return The number of BrickByBrick models that were truncated
+		 */
+		static uint32_t TruncateBrokenBrickByBrickXml();
+
+		/**
+		 * @brief Updates all BrickByBrick models in the database to be
+		 * in the sd0 format as opposed to a zlib compressed format.
+		 *
+		 * @return The number of BrickByBrick models that were updated
+		 */
+		static uint32_t UpdateBrickByBrickModelsToSd0();
+};

dCommon/CMakeLists.txt

@@ -13,13 +13,15 @@ set(DCOMMON_SOURCES "AMFFormat.cpp"
 		"NiQuaternion.cpp"
 		"SHA512.cpp"
 		"Type.cpp"
-		"ZCompression.cpp")
+		"ZCompression.cpp"
+		"BrickByBrickFix.cpp"
+)
 
 include_directories(${PROJECT_SOURCE_DIR}/dCommon/)
 
 add_library(dCommon STATIC ${DCOMMON_SOURCES})
 
-target_link_libraries(dCommon bcrypt)
+target_link_libraries(dCommon bcrypt dDatabase)
 
 if (UNIX)
 	find_package(ZLIB REQUIRED)

dDatabase/Database.cpp

@@ -83,3 +83,13 @@ sql::PreparedStatement* Database::CreatePreppedStmt(const std::string& query) {
 void Database::Commit() {
 	Database::con->commit();
 }
+
+bool Database::GetAutoCommit() {
+	// TODO This should not just access a pointer.  A future PR should update this
+	// to check for null and throw an error if the connection is not valid.
+	return con->getAutoCommit();
+}
+
+void Database::SetAutoCommit(bool value) {
+	Database::con->setAutoCommit(value);
+}

dDatabase/Database.h

@@ -23,6 +23,8 @@ public:
 	static sql::Statement* CreateStmt();
 	static sql::PreparedStatement* CreatePreppedStmt(const std::string& query);
 	static void Commit();
+	static bool GetAutoCommit();
+	static void SetAutoCommit(bool value);
 
 	static std::string GetDatabase() { return database; }
 	static sql::Properties GetProperties() { return props; }

dMasterServer/MasterServer.cpp

@@ -20,6 +20,7 @@
 #include "CDClientManager.h"
 #include "Database.h"
 #include "MigrationRunner.h"
+#include "BrickByBrickFix.h"
 #include "Diagnostics.h"
 #include "dCommonVars.h"
 #include "dConfig.h"
@@ -203,6 +204,24 @@ int main(int argc, char** argv) {
 		return EXIT_SUCCESS;
 	}
 
+	// We were given a brick by brick command
+	if (argc == 3 && strcmp(argv[1], "--brick-by-brick") == 0) {
+		// Truncates invalid model lxfml from the database and
+		// removes their respective models from the database.
+		if (strcmp(argv[2], "TruncateBrokenModels") == 0) {
+			uint32_t numberOfTruncatedModels = BrickByBrickFix::TruncateBrokenBrickByBrickXml();
+			Game::logger->Log("MasterServer", "%i models were truncated from the database.", numberOfTruncatedModels);
+		}
+
+		// Updates old Brick-by-Brick models to use sd0 compression
+		// as opposed to zlib compression
+		if (strcmp(argv[2], "UpdateOldModels") == 0) {
+			uint32_t numberOfUpdatedModels = BrickByBrickFix::UpdateBrickByBrickModelsToSd0();
+			Game::logger->Log("MasterServer", "%i models were updated from zlib to sd0.", numberOfUpdatedModels);
+		}
+		FinalizeShutdown();
+	}
+
 	int maxClients = 999;
 	int ourPort = 1000;
 	if (config.GetValue("max_clients") != "") maxClients = std::stoi(config.GetValue("max_clients"));
@@ -825,9 +844,9 @@ void ShutdownSequence() {
 int FinalizeShutdown() {
 	//Delete our objects here:
 	Database::Destroy("MasterServer");
-	delete Game::im;
-	delete Game::server;
-	delete Game::logger;
+	if (Game::im) delete Game::im;
+	if (Game::server) delete Game::server;
+	if (Game::logger) delete Game::logger;
 
 	exit(EXIT_SUCCESS);
 	return EXIT_SUCCESS;