Merge branch 'main' into npc-pathing

.github/workflows/build-and-test.yml

@@ -10,6 +10,7 @@ jobs:
   build-and-test:
     name: Build & Test (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
+    continue-on-error: true
     strategy:
       matrix:
         os: [ windows-2022, ubuntu-20.04, macos-11 ]

CMakeLists.txt

@@ -90,7 +90,7 @@ make_directory(${CMAKE_BINARY_DIR}/locale)
 make_directory(${CMAKE_BINARY_DIR}/logs)
 
 # Copy resource files on first build
-set(RESOURCE_FILES "authconfig.ini" "chatconfig.ini" "worldconfig.ini" "masterconfig.ini" "blacklist.dcf")
+set(RESOURCE_FILES "sharedconfig.ini" "authconfig.ini" "chatconfig.ini" "worldconfig.ini" "masterconfig.ini" "blacklist.dcf")
 foreach(resource_file ${RESOURCE_FILES})
 	if (NOT EXISTS ${PROJECT_BINARY_DIR}/${resource_file})
 		configure_file(
@@ -108,13 +108,25 @@ foreach(file ${VANITY_FILES})
 endforeach()
 
 # Move our migrations for MasterServer to run
-file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/migrations/)
+file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/migrations/dlu/)
 file(GLOB SQL_FILES ${CMAKE_SOURCE_DIR}/migrations/dlu/*.sql)
 foreach(file ${SQL_FILES})
 	get_filename_component(file ${file} NAME)
-	if (NOT EXISTS ${PROJECT_BINARY_DIR}/migrations/${file})
+	if (NOT EXISTS ${PROJECT_BINARY_DIR}/migrations/dlu/${file})
 		configure_file(
-			${CMAKE_SOURCE_DIR}/migrations/dlu/${file} ${PROJECT_BINARY_DIR}/migrations/${file}
+			${CMAKE_SOURCE_DIR}/migrations/dlu/${file} ${PROJECT_BINARY_DIR}/migrations/dlu/${file}
+			COPYONLY
+		)
+	endif()
+endforeach()
+
+file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/migrations/cdserver/)
+file(GLOB SQL_FILES ${CMAKE_SOURCE_DIR}/migrations/cdserver/*.sql)
+foreach(file ${SQL_FILES})
+	get_filename_component(file ${file} NAME)
+	if (NOT EXISTS ${PROJECT_BINARY_DIR}/migrations/cdserver/${file})
+		configure_file(
+			${CMAKE_SOURCE_DIR}/migrations/cdserver/${file} ${PROJECT_BINARY_DIR}/migrations/cdserver/${file}
 			COPYONLY
 		)
 	endif()

README.md

@@ -196,23 +196,25 @@ certutil -hashfile <file> SHA256
 * Copy over or create symlinks from `locale.xml` in your client `locale` directory to the `build/locale` directory
 
 #### Client database
-* Use `fdb_to_sqlite.py` in lcdr's utilities on `res/cdclient.fdb` in the unpacked client to convert the client database to `cdclient.sqlite`
+* Move the file `res/cdclient.fdb` from the unpacked client to the `build/res` folder on the server.
-* Move and rename `cdclient.sqlite` into `build/res/CDServer.sqlite`
+* The server will automatically copy and convert the file from fdb to sqlite should `CDServer.sqlite` not already exist.
-* Run each SQL file in the order at which they appear [here](migrations/cdserver/) on the SQLite database
+* You can also convert the database manually using `fdb_to_sqlite.py` using lcdr's utilities.  Just make sure to rename the file to `CDServer.sqlite` instead of `cdclient.sqlite`.
+* Migrations to the database are automatically run on server start.  When migrations are needed to be ran, the server may take a bit longer to start.
 
 ### Database
 Darkflame Universe utilizes a MySQL/MariaDB database for account and character information.
 
 Initial setup can vary drastically based on which operating system or distribution you are running; there are instructions out there for most setups, follow those and come back here when you have a database up and running.
-* Create a database for Darkflame Universe to use
+
+* All that you need to do is create a database to connect to.  As long as the server can connect to the database, the schema will always be kept up to date when you start the server.
 
 #### Configuration
 
-After the server has been built there should be four `ini` files in the build director: `authconfig.ini`, `chatconfig.ini`, `masterconfig.ini`, and `worldconfig.ini`. Go through them and fill in the database credentials and configure other settings if necessary.
+After the server has been built there should be four `ini` files in the build director: `sharedconfig.ini`, `authconfig.ini`, `chatconfig.ini`, `masterconfig.ini`, and `worldconfig.ini`. Go through them and fill in the database credentials and configure other settings if necessary.
 
-#### Setup and Migrations
+#### Migrations
 
-Use the command `./MasterServer -m` to setup the tables in the database. The first time this command is run on a database, the tables will be up to date with the most recent version. To update your database tables, run this command again. Multiple invocations will not affect any functionality.
+The database is automatically setup and migrated to what it should look like for the latest commit whenever you start the server.
 
 #### Verify
 
@@ -228,7 +230,7 @@ Your build directory should now look like this:
 * **locale/**
   * locale.xml
 * **res/**
-  * CDServer.sqlite
+  * cdclient.fdb
   * chatplus_en_us.txt
   * **macros/**
     * ...

build.sh

@@ -8,5 +8,3 @@ cmake ..
 # To build utilizing multiple cores, append `-j` and the amount of cores to utilize, for example `cmake --build . --config Release -j8'
 cmake --build . --config Release
 
-# Run migrations
-./MasterServer -m

dCommon/AMFFormat.cpp

@@ -58,16 +58,6 @@ void AMFArrayValue::RemoveValue(const std::string& key) {
 	}
 }
 
-// AMFArray Find Value
-AMFValue* AMFArrayValue::FindValue(const std::string& key) {
-	_AMFArrayMap_::iterator it = this->associative.find(key);
-	if (it != this->associative.end()) {
-		return it->second;
-	}
-
-	return nullptr;
-}
-
 // AMFArray Get Associative Iterator Begin
 _AMFArrayMap_::iterator AMFArrayValue::GetAssociativeIteratorValueBegin() {
 	return this->associative.begin();
@@ -93,11 +83,6 @@ uint32_t AMFArrayValue::GetDenseValueSize() {
 	return (uint32_t)this->dense.size();
 }
 
-// AMFArray Get value at index in Dense List
-AMFValue* AMFArrayValue::GetValueAt(uint32_t index) {
-	return this->dense.at(index);
-}
-
 // AMFArray Get Dense Iterator Begin
 _AMFArrayList_::iterator AMFArrayValue::GetDenseIteratorBegin() {
 	return this->dense.begin();

dCommon/AMFFormat.h

@@ -75,7 +75,9 @@ private:
 	/*!
 	  \return The AMF value type
 	 */
-	AMFValueType GetValueType() { return AMFUndefined; }
+	AMFValueType GetValueType() { return ValueType; }
+public:
+	static const AMFValueType ValueType = AMFUndefined;
 };
 
 //! The null value AMF type
@@ -85,7 +87,9 @@ private:
 	/*!
 	 \return The AMF value type
 	 */
-	AMFValueType GetValueType() { return AMFNull; }
+	AMFValueType GetValueType() { return ValueType; }
+public:
+	static const AMFValueType ValueType = AMFNull;
 };
 
 //! The false value AMF type
@@ -95,7 +99,9 @@ private:
 	/*!
 	 \return The AMF value type
 	 */
-	AMFValueType GetValueType() { return AMFFalse; }
+	AMFValueType GetValueType() { return ValueType; }
+public:
+	static const AMFValueType ValueType = AMFFalse;
 };
 
 //! The true value AMF type
@@ -105,7 +111,9 @@ private:
 	/*!
 	 \return The AMF value type
 	 */
-	AMFValueType GetValueType() { return AMFTrue; }
+	AMFValueType GetValueType() { return ValueType; }
+public:
+	static const AMFValueType ValueType = AMFTrue;
 };
 
 //! The integer value AMF type
@@ -117,9 +125,10 @@ private:
 	/*!
 	 \return The AMF value type
 	 */
-	AMFValueType GetValueType() { return AMFInteger; }
+	AMFValueType GetValueType() { return ValueType; }
 
 public:
+	static const AMFValueType ValueType = AMFInteger;
 	//! Sets the integer value
 	/*!
 	  \param value The value to set
@@ -142,9 +151,10 @@ private:
 	/*!
 	 \return The AMF value type
 	 */
-	AMFValueType GetValueType() { return AMFDouble; }
+	AMFValueType GetValueType() { return ValueType; }
 
 public:
+	static const AMFValueType ValueType = AMFDouble;
 	//! Sets the double value
 	/*!
 	  \param value The value to set to
@@ -167,9 +177,10 @@ private:
 	/*!
 	 \return The AMF value type
 	 */
-	AMFValueType GetValueType() { return AMFString; }
+	AMFValueType GetValueType() { return ValueType; }
 
 public:
+	static const AMFValueType ValueType = AMFString;
 	//! Sets the string value
 	/*!
 	  \param value The string value to set to
@@ -192,9 +203,10 @@ private:
 	/*!
 	 \return The AMF value type
 	 */
-	AMFValueType GetValueType() { return AMFXMLDoc; }
+	AMFValueType GetValueType() { return ValueType; }
 
 public:
+	static const AMFValueType ValueType = AMFXMLDoc;
 	//! Sets the XML Doc value
 	/*!
 	  \param value The value to set to
@@ -217,9 +229,10 @@ private:
 	/*!
 	 \return The AMF value type
 	 */
-	AMFValueType GetValueType() { return AMFDate; }
+	AMFValueType GetValueType() { return ValueType; }
 
 public:
+	static const AMFValueType ValueType = AMFDate;
 	//! Sets the date time
 	/*!
 	  \param value The value to set to
@@ -244,9 +257,11 @@ private:
 	/*!
 	 \return The AMF value type
 	 */
-	AMFValueType GetValueType() { return AMFArray; }
+	AMFValueType GetValueType() { return ValueType; }
 
 public:
+	static const AMFValueType ValueType = AMFArray;
+
 	~AMFArrayValue() override;
 	//! Inserts an item into the array map for a specific key
 	/*!
@@ -265,7 +280,15 @@ public:
 	/*!
 	  \return The AMF value if found, nullptr otherwise
 	 */
-	AMFValue* FindValue(const std::string& key);
+	template <typename T>
+	T* FindValue(const std::string& key) const {
+		_AMFArrayMap_::const_iterator it = this->associative.find(key);
+		if (it != this->associative.end() && T::ValueType == it->second->GetValueType()) {
+			return dynamic_cast<T*>(it->second);
+		}
+
+		return nullptr;
+	};
 
 	//! Returns where the associative iterator begins
 	/*!
@@ -298,7 +321,12 @@ public:
 	/*!
 	  \param index The index to get
 	 */
-	AMFValue* GetValueAt(uint32_t index);
+	template <typename T>
+	T* GetValueAt(uint32_t index) {
+		if (index >= this->dense.size()) return nullptr;
+		AMFValue* foundValue = this->dense.at(index);
+		return T::ValueType == foundValue->GetValueType() ? dynamic_cast<T*>(foundValue) : nullptr;
+	};
 
 	//! Returns where the dense iterator begins
 	/*!
@@ -334,10 +362,11 @@ private:
 	/*!
 	 \return The AMF value type
 	 */
-	AMFValueType GetValueType() { return AMFObject; }
+	AMFValueType GetValueType() { return ValueType; }
 	~AMFObjectValue() override;
 
 public:
+	static const AMFValueType ValueType = AMFObject;
 	//! Constructor
 	/*!
 	  \param traits The traits to set

dCommon/BrickByBrickFix.cpp

@@ -0,0 +1,180 @@
+#include "BrickByBrickFix.h"
+
+#include <memory>
+#include <iostream>
+#include <sstream>
+
+#include "tinyxml2.h"
+
+#include "Database.h"
+#include "Game.h"
+#include "ZCompression.h"
+#include "dLogger.h"
+
+//! Forward declarations
+
+std::unique_ptr<sql::ResultSet> GetModelsFromDatabase();
+void WriteSd0Magic(char* input, uint32_t chunkSize);
+bool CheckSd0Magic(sql::Blob* streamToCheck);
+
+/**
+ * @brief Truncates all models with broken data from the database.
+ *
+ * @return The number of models deleted
+ */
+uint32_t BrickByBrickFix::TruncateBrokenBrickByBrickXml() {
+	uint32_t modelsTruncated{};
+	auto modelsToTruncate = GetModelsFromDatabase();
+	bool previousCommitValue = Database::GetAutoCommit();
+	Database::SetAutoCommit(false);
+	while (modelsToTruncate->next()) {
+		std::unique_ptr<sql::PreparedStatement> ugcModelToDelete(Database::CreatePreppedStmt("DELETE FROM ugc WHERE ugc.id = ?;"));
+		std::unique_ptr<sql::PreparedStatement> pcModelToDelete(Database::CreatePreppedStmt("DELETE FROM properties_contents WHERE ugc_id = ?;"));
+		std::string completeUncompressedModel{};
+		uint32_t chunkCount{};
+		uint64_t modelId = modelsToTruncate->getInt(1);
+		std::unique_ptr<sql::Blob> modelAsSd0(modelsToTruncate->getBlob(2));
+		// Check that header is sd0 by checking for the sd0 magic.
+		if (CheckSd0Magic(modelAsSd0.get())) {
+			while (true) {
+				uint32_t chunkSize{};
+				modelAsSd0->read(reinterpret_cast<char*>(&chunkSize), sizeof(uint32_t)); // Extract chunk size from istream
+
+				// Check if good here since if at the end of an sd0 file, this will have eof flagged.
+				if (!modelAsSd0->good()) break;
+
+				std::unique_ptr<uint8_t[]> compressedChunk(new uint8_t[chunkSize]);
+				for (uint32_t i = 0; i < chunkSize; i++) {
+					compressedChunk[i] = modelAsSd0->get();
+				}
+
+				// Ignore the valgrind warning about uninitialized values.  These are discarded later when we know the actual uncompressed size.
+				std::unique_ptr<uint8_t[]> uncompressedChunk(new uint8_t[MAX_SD0_CHUNK_SIZE]);
+				int32_t err{};
+				int32_t actualUncompressedSize = ZCompression::Decompress(
+					compressedChunk.get(), chunkSize, uncompressedChunk.get(), MAX_SD0_CHUNK_SIZE, err);
+
+				if (actualUncompressedSize != -1) {
+					uint32_t previousSize = completeUncompressedModel.size();
+					completeUncompressedModel.append((char*)uncompressedChunk.get());
+					completeUncompressedModel.resize(previousSize + actualUncompressedSize);
+				} else {
+					Game::logger->Log("BrickByBrickFix", "Failed to inflate chunk %i for model %llu.  Error: %i", chunkCount, modelId, err);
+					break;
+				}
+				chunkCount++;
+			}
+			std::unique_ptr<tinyxml2::XMLDocument> document = std::make_unique<tinyxml2::XMLDocument>();
+			if (!document) {
+				Game::logger->Log("BrickByBrickFix", "Failed to initialize tinyxml document.  Aborting.");
+				return 0;
+			}
+
+			if (!(document->Parse(completeUncompressedModel.c_str(), completeUncompressedModel.size()) == tinyxml2::XML_SUCCESS)) {
+				if (completeUncompressedModel.find(
+					"</LXFML>",
+					completeUncompressedModel.length() >= 15 ? completeUncompressedModel.length() - 15 : 0) == std::string::npos
+					) {
+					Game::logger->Log("BrickByBrickFix",
+						"Brick-by-brick model %llu will be deleted!", modelId);
+					ugcModelToDelete->setInt64(1, modelsToTruncate->getInt64(1));
+					pcModelToDelete->setInt64(1, modelsToTruncate->getInt64(1));
+					ugcModelToDelete->execute();
+					pcModelToDelete->execute();
+					modelsTruncated++;
+				}
+			}
+		} else {
+			Game::logger->Log("BrickByBrickFix",
+				"Brick-by-brick model %llu will be deleted!", modelId);
+			ugcModelToDelete->setInt64(1, modelsToTruncate->getInt64(1));
+			pcModelToDelete->setInt64(1, modelsToTruncate->getInt64(1));
+			ugcModelToDelete->execute();
+			pcModelToDelete->execute();
+			modelsTruncated++;
+		}
+	}
+
+	Database::Commit();
+	Database::SetAutoCommit(previousCommitValue);
+	return modelsTruncated;
+}
+
+/**
+ * @brief Updates all current models in the database to have the Segmented Data 0 (SD0) format.
+ * Any models that do not start with zlib and best compression magic will not be updated.
+ *
+ * @return The number of models updated to SD0
+ */
+uint32_t BrickByBrickFix::UpdateBrickByBrickModelsToSd0() {
+	uint32_t updatedModels = 0;
+	auto modelsToUpdate = GetModelsFromDatabase();
+	auto previousAutoCommitState = Database::GetAutoCommit();
+	Database::SetAutoCommit(false);
+	std::unique_ptr<sql::PreparedStatement> insertionStatement(Database::CreatePreppedStmt("UPDATE ugc SET lxfml = ? WHERE id = ?;"));
+	while (modelsToUpdate->next()) {
+		int64_t modelId = modelsToUpdate->getInt64(1);
+		std::unique_ptr<sql::Blob> oldLxfml(modelsToUpdate->getBlob(2));
+		// Check if the stored blob starts with zlib magic (0x78 0xDA - best compression of zlib)
+		// If it does, convert it to sd0.
+		if (oldLxfml->get() == 0x78 && oldLxfml->get() == 0xDA) {
+
+			// Get and save size of zlib compressed chunk.
+			oldLxfml->seekg(0, std::ios::end);
+			uint32_t oldLxfmlSize = static_cast<uint32_t>(oldLxfml->tellg());
+			oldLxfml->seekg(0);
+
+			// Allocate 9 extra bytes.  5 for sd0 magic, 4 for the only zlib compressed size.
+			uint32_t oldLxfmlSizeWithHeader = oldLxfmlSize + 9;
+			std::unique_ptr<char[]> sd0ConvertedModel(new char[oldLxfmlSizeWithHeader]);
+
+			WriteSd0Magic(sd0ConvertedModel.get(), oldLxfmlSize);
+			for (uint32_t i = 9; i < oldLxfmlSizeWithHeader; i++) {
+				sd0ConvertedModel.get()[i] = oldLxfml->get();
+			}
+
+			std::string outputString(sd0ConvertedModel.get(), oldLxfmlSizeWithHeader);
+			std::istringstream outputStringStream(outputString);
+
+			insertionStatement->setBlob(1, static_cast<std::istream*>(&outputStringStream));
+			insertionStatement->setInt64(2, modelId);
+			try {
+				insertionStatement->executeUpdate();
+				Game::logger->Log("BrickByBrickFix", "Updated model %i to sd0", modelId);
+				updatedModels++;
+			} catch (sql::SQLException exception) {
+				Game::logger->Log(
+					"BrickByBrickFix",
+					"Failed to update model %i.  This model should be inspected manually to see why."
+					"The database error is %s", modelId, exception.what());
+			}
+		}
+	}
+	Database::Commit();
+	Database::SetAutoCommit(previousAutoCommitState);
+	return updatedModels;
+}
+
+std::unique_ptr<sql::ResultSet> GetModelsFromDatabase() {
+	std::unique_ptr<sql::PreparedStatement> modelsRawDataQuery(Database::CreatePreppedStmt("SELECT id, lxfml FROM ugc;"));
+	return std::unique_ptr<sql::ResultSet>(modelsRawDataQuery->executeQuery());
+}
+
+/**
+ * @brief Writes sd0 magic at the front of a char*
+ *
+ * @param input the char* to write at the front of
+ * @param chunkSize The size of the first chunk to write the size of
+ */
+void WriteSd0Magic(char* input, uint32_t chunkSize) {
+	input[0] = 's';
+	input[1] = 'd';
+	input[2] = '0';
+	input[3] = 0x01;
+	input[4] = 0xFF;
+	*reinterpret_cast<uint32_t*>(input + 5) = chunkSize; // Write the integer to the character array
+}
+
+bool CheckSd0Magic(sql::Blob* streamToCheck) {
+	return streamToCheck->get() == 's' && streamToCheck->get() == 'd' && streamToCheck->get() == '0' && streamToCheck->get() == 0x01 && streamToCheck->get() == 0xFF;
+}

dCommon/BrickByBrickFix.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include <cstdint>
+
+namespace BrickByBrickFix {
+	/**
+	 * @brief Deletes all broken BrickByBrick models that have invalid XML
+	 *
+	 * @return The number of BrickByBrick models that were truncated
+	 */
+	uint32_t TruncateBrokenBrickByBrickXml();
+
+	/**
+	 * @brief Updates all BrickByBrick models in the database to be
+	 * in the sd0 format as opposed to a zlib compressed format.
+	 *
+	 * @return The number of BrickByBrick models that were updated
+	 */
+	uint32_t UpdateBrickByBrickModelsToSd0();
+
+	/**
+	 * @brief Max size of an inflated sd0 zlib chunk
+	 * 
+	 */
+	constexpr uint32_t MAX_SD0_CHUNK_SIZE = 1024 * 256;
+};

dCommon/CMakeLists.txt

@@ -13,13 +13,15 @@ set(DCOMMON_SOURCES "AMFFormat.cpp"
 		"NiQuaternion.cpp"
 		"SHA512.cpp"
 		"Type.cpp"
-		"ZCompression.cpp")
+		"ZCompression.cpp"
+		"BrickByBrickFix.cpp"
+)
 
 include_directories(${PROJECT_SOURCE_DIR}/dCommon/)
 
 add_library(dCommon STATIC ${DCOMMON_SOURCES})
 
-target_link_libraries(dCommon bcrypt)
+target_link_libraries(dCommon bcrypt dDatabase tinyxml2)
 
 if (UNIX)
 	find_package(ZLIB REQUIRED)

dCommon/dConfig.cpp

@@ -12,6 +12,15 @@ dConfig::dConfig(const std::string& filepath) {
 			if (line[0] != '#') ProcessLine(line);
 		}
 	}
+
+	std::ifstream sharedConfig("sharedconfig.ini", std::ios::in);
+	if (!sharedConfig.good()) return;
+
+	while (std::getline(sharedConfig, line)) {
+		if (line.length() > 0) {
+			if (line[0] != '#') ProcessLine(line);
+		}
+	}
 }
 
 dConfig::~dConfig(void) {
@@ -40,6 +49,12 @@ void dConfig::ProcessLine(const std::string& line) {
 	if (!seglist[1].empty() && seglist[1][seglist[1].size() - 1] == '\r')
 		seglist[1].erase(seglist[1].size() - 1);
 
+	for (const auto& key : m_Keys) {
+		if (seglist[0] == key) {
+			return; // first loaded key is preferred due to loading shared config secondarily
+		}
+	}
+
 	m_Keys.push_back(seglist[0]);
 	m_Values.push_back(seglist[1]);
 }

dDatabase/CDClientDatabase.cpp

@@ -14,6 +14,11 @@ CppSQLite3Query CDClientDatabase::ExecuteQuery(const std::string& query) {
 	return conn->execQuery(query.c_str());
 }
 
+//! Updates the CDClient file with Data Manipulation Language (DML) commands.
+int CDClientDatabase::ExecuteDML(const std::string& query) {
+	return conn->execDML(query.c_str());
+}
+
 //! Makes prepared statements
 CppSQLite3Statement CDClientDatabase::CreatePreppedStmt(const std::string& query) {
 	return conn->compileStatement(query.c_str());

dDatabase/CDClientDatabase.h

@@ -40,6 +40,14 @@ namespace CDClientDatabase {
 	 */
 	CppSQLite3Query ExecuteQuery(const std::string& query);
 
+	//! Updates the CDClient file with Data Manipulation Language (DML) commands.
+	/*!
+	  \param query The DML command to run.  DML command can be multiple queries in one string but only
+		the last one will return its number of updated rows.
+	  \return The number of updated rows.
+	*/
+	int ExecuteDML(const std::string& query);
+
 	//! Queries the CDClient and parses arguments
 	/*!
 	  \param query The query with formatted arguments

dDatabase/Database.cpp

@@ -83,3 +83,15 @@ sql::PreparedStatement* Database::CreatePreppedStmt(const std::string& query) {
 void Database::Commit() {
 	Database::con->commit();
 }
+
+bool Database::GetAutoCommit() {
+	// TODO This should not just access a pointer.  A future PR should update this
+	// to check for null and throw an error if the connection is not valid.
+	return con->getAutoCommit();
+}
+
+void Database::SetAutoCommit(bool value) {
+	// TODO This should not just access a pointer.  A future PR should update this
+	// to check for null and throw an error if the connection is not valid.
+	Database::con->setAutoCommit(value);
+}

dDatabase/Database.h

@@ -23,6 +23,8 @@ public:
 	static sql::Statement* CreateStmt();
 	static sql::PreparedStatement* CreatePreppedStmt(const std::string& query);
 	static void Commit();
+	static bool GetAutoCommit();
+	static void SetAutoCommit(bool value);
 
 	static std::string GetDatabase() { return database; }
 	static sql::Properties GetProperties() { return props; }

dDatabase/MigrationRunner.cpp

@@ -1,65 +1,19 @@
 #include "MigrationRunner.h"
 
+#include "BrickByBrickFix.h"
+#include "CDClientDatabase.h"
+#include "Database.h"
+#include "Game.h"
 #include "GeneralUtils.h"
+#include "dLogger.h"
 
-#include <fstream>
+#include <istream>
-#include <algorithm>
-#include <thread>
 
-void MigrationRunner::RunMigrations() {
+Migration LoadMigration(std::string path) {
-	auto stmt = Database::CreatePreppedStmt("CREATE TABLE IF NOT EXISTS migration_history (name TEXT NOT NULL, date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP());");
-	stmt->executeQuery();
-	delete stmt;
-
-	sql::SQLString finalSQL = "";
-	Migration checkMigration{};
-
-	for (const auto& entry : GeneralUtils::GetFileNamesFromFolder("./migrations/")) {
-		auto migration = LoadMigration(entry);
-
-		if (migration.data.empty()) {
-			continue;
-		}
-
-		checkMigration = migration;
-
-		stmt = Database::CreatePreppedStmt("SELECT name FROM migration_history WHERE name = ?;");
-		stmt->setString(1, migration.name);
-		auto res = stmt->executeQuery();
-		bool doExit = res->next();
-		delete res;
-		delete stmt;
-		if (doExit) continue;
-
-		Game::logger->Log("MigrationRunner", "Running migration: %s", migration.name.c_str());
-
-		finalSQL.append(migration.data);
-		finalSQL.append('\n');
-
-		stmt = Database::CreatePreppedStmt("INSERT INTO migration_history (name) VALUES (?);");
-		stmt->setString(1, entry);
-		stmt->execute();
-		delete stmt;
-	}
-
-	if (!finalSQL.empty()) {
-		try {
-			auto simpleStatement = Database::CreateStmt();
-			simpleStatement->execute(finalSQL);
-			delete simpleStatement;
-		} catch (sql::SQLException e) {
-			Game::logger->Log("MigrationRunner", "Encountered error running migration: %s", e.what());
-		}
-	}
-}
-
-Migration MigrationRunner::LoadMigration(std::string path) {
 	Migration migration{};
 	std::ifstream file("./migrations/" + path);
 
 	if (file.is_open()) {
-		std::hash<std::string> hash;
-
 		std::string line;
 		std::string total = "";
 
@@ -75,3 +29,102 @@ Migration MigrationRunner::LoadMigration(std::string path) {
 
 	return migration;
 }
+
+void MigrationRunner::RunMigrations() {
+	auto* stmt = Database::CreatePreppedStmt("CREATE TABLE IF NOT EXISTS migration_history (name TEXT NOT NULL, date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP());");
+	stmt->execute();
+	delete stmt;
+
+	sql::SQLString finalSQL = "";
+	bool runSd0Migrations = false;
+	for (const auto& entry : GeneralUtils::GetFileNamesFromFolder("./migrations/dlu/")) {
+		auto migration = LoadMigration("dlu/" + entry);
+
+		if (migration.data.empty()) {
+			continue;
+		}
+
+		stmt = Database::CreatePreppedStmt("SELECT name FROM migration_history WHERE name = ?;");
+		stmt->setString(1, migration.name);
+		auto* res = stmt->executeQuery();
+		bool doExit = res->next();
+		delete res;
+		delete stmt;
+		if (doExit) continue;
+
+		Game::logger->Log("MigrationRunner", "Running migration: %s", migration.name.c_str());
+		if (migration.name == "5_brick_model_sd0.sql") {
+			runSd0Migrations = true;
+		} else {
+			finalSQL.append(migration.data);
+		}
+
+		stmt = Database::CreatePreppedStmt("INSERT INTO migration_history (name) VALUES (?);");
+		stmt->setString(1, migration.name);
+		stmt->execute();
+		delete stmt;
+	}
+
+	if (finalSQL.empty() && !runSd0Migrations) {
+		Game::logger->Log("MigrationRunner", "Server database is up to date.");
+		return;
+	}
+
+	if (!finalSQL.empty()) {
+		auto migration = GeneralUtils::SplitString(static_cast<std::string>(finalSQL), ';');
+		std::unique_ptr<sql::Statement> simpleStatement(Database::CreateStmt());
+		for (auto& query : migration) {
+			try {
+				if (query.empty()) continue;
+				simpleStatement->execute(query);
+			} catch (sql::SQLException& e) {
+				Game::logger->Log("MigrationRunner", "Encountered error running migration: %s", e.what());
+			}
+		}
+	}
+
+	// Do this last on the off chance none of the other migrations have been run yet.
+	if (runSd0Migrations) {
+		uint32_t numberOfUpdatedModels = BrickByBrickFix::UpdateBrickByBrickModelsToSd0();
+		Game::logger->Log("MasterServer", "%i models were updated from zlib to sd0.", numberOfUpdatedModels);
+		uint32_t numberOfTruncatedModels = BrickByBrickFix::TruncateBrokenBrickByBrickXml();
+		Game::logger->Log("MasterServer", "%i models were truncated from the database.", numberOfTruncatedModels);
+	}
+}
+
+void MigrationRunner::RunSQLiteMigrations() {
+	auto* stmt = Database::CreatePreppedStmt("CREATE TABLE IF NOT EXISTS migration_history (name TEXT NOT NULL, date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP());");
+	stmt->execute();
+	delete stmt;
+
+	for (const auto& entry : GeneralUtils::GetFileNamesFromFolder("./migrations/cdserver/")) {
+		auto migration = LoadMigration("cdserver/" + entry);
+
+		if (migration.data.empty()) continue;
+
+		stmt = Database::CreatePreppedStmt("SELECT name FROM migration_history WHERE name = ?;");
+		stmt->setString(1, migration.name);
+		auto* res = stmt->executeQuery();
+		bool doExit = res->next();
+		delete res;
+		delete stmt;
+		if (doExit) continue;
+
+		// Doing these 1 migration at a time since one takes a long time and some may think it is crashing.
+		// This will at the least guarentee that the full migration needs to be run in order to be counted as "migrated".
+		Game::logger->Log("MigrationRunner", "Executing migration: %s.  This may take a while.  Do not shut down server.", migration.name.c_str());
+		for (const auto& dml : GeneralUtils::SplitString(migration.data, ';')) {
+			if (dml.empty()) continue;
+			try {
+				CDClientDatabase::ExecuteDML(dml.c_str());
+			} catch (CppSQLite3Exception& e) {
+				Game::logger->Log("MigrationRunner", "Encountered error running DML command: (%i) : %s", e.errorCode(), e.errorMessage());
+			}
+		}
+		stmt = Database::CreatePreppedStmt("INSERT INTO migration_history (name) VALUES (?);");
+		stmt->setString(1, migration.name);
+		stmt->execute();
+		delete stmt;
+	}
+	Game::logger->Log("MigrationRunner", "CDServer database is up to date.");
+}

dDatabase/MigrationRunner.h

@@ -1,19 +1,13 @@
 #pragma once
 
-#include "Database.h"
+#include <string>
-
-#include "dCommonVars.h"
-#include "Game.h"
-#include "dCommonVars.h"
-#include "dLogger.h"
 
 struct Migration {
 	std::string data;
 	std::string name;
 };
 
-class MigrationRunner {
+namespace MigrationRunner {
-public:
+	void RunMigrations();
-	static void RunMigrations();
+	void RunSQLiteMigrations();
-	static Migration LoadMigration(std::string path);
 };

dGame/dGameMessages/GameMessages.cpp

@@ -2413,7 +2413,6 @@ void GameMessages::HandleBBBSaveRequest(RakNet::BitStream* inStream, Entity* ent
 		 /__\/// _ \ \ /\ / / _` | '__/ _ \/  /
 		/ \/  \  __/\ V  V / (_| | | |  __/\_/
 		\_____/\___| \_/\_/ \__,_|_|  \___\/
-
 						<>=======()
 							   (/\___   /|\\          ()==========<>_
 									 \_/ | \\        //|\   ______/ \)
@@ -2430,34 +2429,25 @@ void GameMessages::HandleBBBSaveRequest(RakNet::BitStream* inStream, Entity* ent
 												______/ /
 												'------'
 	*/
-
-	//First, we have Wincent's clean methods of reading in the data received from the client.
 	LWOOBJID localId;
-	uint32_t timeTaken;
 
 	inStream->Read(localId);
 
-	uint32_t ld0Size;
+	uint32_t sd0Size;
-	inStream->Read(ld0Size);
+	inStream->Read(sd0Size);
-	for (auto i = 0; i < 5; ++i) {
+	std::shared_ptr<char[]> sd0Data(new char[sd0Size]);
-		uint8_t c;
-		inStream->Read(c);
-	}
 
-	uint32_t lxfmlSize;
+	if (sd0Data == nullptr) {
-	inStream->Read(lxfmlSize);
-	uint8_t* inData = static_cast<uint8_t*>(std::malloc(lxfmlSize));
-
-	if (inData == nullptr) {
 		return;
 	}
 
-	for (uint32_t i = 0; i < lxfmlSize; ++i) {
+	for (uint32_t i = 0; i < sd0Size; ++i) {
 		uint8_t c;
 		inStream->Read(c);
-		inData[i] = c;
+		sd0Data[i] = c;
 	}
 
+	uint32_t timeTaken;
 	inStream->Read(timeTaken);
 
 	/*
@@ -2469,6 +2459,8 @@ void GameMessages::HandleBBBSaveRequest(RakNet::BitStream* inStream, Entity* ent
 
 		Note, in the live client it'll still display the bricks going out as they're being used, but on relog/world change,
 		they reappear as we didn't take them.
+
+		TODO Apparently the bricks are supposed to be taken via MoveInventoryBatch?
 	*/
 
 	////Decompress the SD0 from the client so we can process the lxfml properly
@@ -2513,9 +2505,7 @@ void GameMessages::HandleBBBSaveRequest(RakNet::BitStream* inStream, Entity* ent
 
 				auto result = query.execQuery();
 
-				if (result.eof() || result.fieldIsNull(0)) {
+				if (result.eof() || result.fieldIsNull(0)) return;
-					return;
-				}
 
 				int templateId = result.getIntField(0);
 
@@ -2533,6 +2523,7 @@ void GameMessages::HandleBBBSaveRequest(RakNet::BitStream* inStream, Entity* ent
 					propertyId = propertyEntry->getUInt64(1);
 				}
 
+				delete propertyEntry;
 				delete propertyLookup;
 
 				//Insert into ugc:
@@ -2543,7 +2534,7 @@ void GameMessages::HandleBBBSaveRequest(RakNet::BitStream* inStream, Entity* ent
 				ugcs->setInt(4, 0);
 
 				//whacky stream biz
-				std::string s((char*)inData, lxfmlSize);
+				std::string s(sd0Data.get(), sd0Size);
 				std::istringstream iss(s);
 				std::istream& stream = iss;
 
@@ -2562,10 +2553,10 @@ void GameMessages::HandleBBBSaveRequest(RakNet::BitStream* inStream, Entity* ent
 				stmt->setDouble(5, 0.0f); // x
 				stmt->setDouble(6, 0.0f); // y
 				stmt->setDouble(7, 0.0f); // z
-				stmt->setDouble(8, 0.0f);
+				stmt->setDouble(8, 0.0f); // rx
-				stmt->setDouble(9, 0.0f);
+				stmt->setDouble(9, 0.0f); // ry
-				stmt->setDouble(10, 0.0f);
+				stmt->setDouble(10, 0.0f); // rz
-				stmt->setDouble(11, 0.0f);
+				stmt->setDouble(11, 0.0f); // rw
 				stmt->execute();
 				delete stmt;
 
@@ -2591,38 +2582,22 @@ void GameMessages::HandleBBBSaveRequest(RakNet::BitStream* inStream, Entity* ent
 
 				//Tell the client their model is saved: (this causes us to actually pop out of our current state):
 				CBITSTREAM;
-				PacketUtils::WriteHeader(bitStream, CLIENT, MSG_CLIENT_BLUEPRINT_SAVE_RESPONSE);
+				PacketUtils::WriteHeader(bitStream, CLIENT, CLIENT::MSG_CLIENT_BLUEPRINT_SAVE_RESPONSE);
 				bitStream.Write(localId);
 				bitStream.Write<unsigned int>(0);
 				bitStream.Write<unsigned int>(1);
 				bitStream.Write(blueprintID);
 
-				bitStream.Write(lxfmlSize + 9);
+				bitStream.Write<uint32_t>(sd0Size);
 
-				//Write a fake sd0 header:
+				for (size_t i = 0; i < sd0Size; ++i) {
-				bitStream.Write<unsigned char>(0x73); //s
+					bitStream.Write(sd0Data[i]);
-				bitStream.Write<unsigned char>(0x64); //d
+				}
-				bitStream.Write<unsigned char>(0x30); //0
-				bitStream.Write<unsigned char>(0x01); //1
-				bitStream.Write<unsigned char>(0xFF); //end magic
-
-				bitStream.Write(lxfmlSize);
-
-				for (size_t i = 0; i < lxfmlSize; ++i)
-					bitStream.Write(inData[i]);
 
 				SEND_PACKET;
-				PacketUtils::SavePacket("MSG_CLIENT_BLUEPRINT_SAVE_RESPONSE.bin", (char*)bitStream.GetData(), bitStream.GetNumberOfBytesUsed());
+				// PacketUtils::SavePacket("MSG_CLIENT_BLUEPRINT_SAVE_RESPONSE.bin", (char*)bitStream.GetData(), bitStream.GetNumberOfBytesUsed());
 
 				//Now we have to construct this object:
-				/*
-				* This needs to be sent as config data, but I don't know how to right now.
-					'blueprintid': (9, 1152921508346399522),
-					'componentWhitelist': (1, 1),
-					'modelType': (1, 2),
-					'propertyObjectID': (7, True),
-					'userModelID': (9, 1152921510759098799)
-				*/
 
 				EntityInfo info;
 				info.lot = 14;
@@ -2653,6 +2628,7 @@ void GameMessages::HandleBBBSaveRequest(RakNet::BitStream* inStream, Entity* ent
 					//there was an issue with builds not appearing since it was placed above ConstructEntity.
 					PropertyManagementComponent::Instance()->AddModel(newEntity->GetObjectID(), newIDL);
 				}
+
 				});
 			});
 		});

dMasterServer/MasterServer.cpp

@@ -85,7 +85,6 @@ int main(int argc, char** argv) {
 	Game::logger->SetLogToConsole(bool(std::stoi(config.GetValue("log_to_console"))));
 	Game::logger->SetLogDebugStatements(config.GetValue("log_debug_statements") == "1");
 
-	if (argc > 1 && (strcmp(argv[1], "-m") == 0 || strcmp(argv[1], "--migrations") == 0)) {
 	//Connect to the MySQL Database
 	std::string mysql_host = config.GetValue("mysql_host");
 	std::string mysql_database = config.GetValue("mysql_database");
@@ -101,20 +100,40 @@ int main(int argc, char** argv) {
 	}
 
 	MigrationRunner::RunMigrations();
-		Game::logger->Log("MigrationRunner", "Finished running migrations");
-
-		return EXIT_SUCCESS;
-	} else {
 
 	//Check CDClient exists
 	const std::string cdclient_path = "./res/CDServer.sqlite";
 	std::ifstream cdclient_fd(cdclient_path);
 	if (!cdclient_fd.good()) {
-			Game::logger->Log("WorldServer", "%s could not be opened", cdclient_path.c_str());
+		Game::logger->Log("WorldServer", "%s could not be opened.  Looking for cdclient.fdb to convert to sqlite.", cdclient_path.c_str());
+		cdclient_fd.close();
+
+		const std::string cdclientFdbPath = "./res/cdclient.fdb";
+		cdclient_fd.open(cdclientFdbPath);
+		if (!cdclient_fd.good()) {
+			Game::logger->Log(
+				"WorldServer", "%s could not be opened."
+				"Please move a cdclient.fdb or an already converted database to build/res.", cdclientFdbPath.c_str());
 			return EXIT_FAILURE;
 		}
+		Game::logger->Log("WorldServer", "Found %s.  Clearing cdserver migration_history then copying and converting to sqlite.", cdclientFdbPath.c_str());
+		auto stmt = Database::CreatePreppedStmt(R"#(DELETE FROM migration_history WHERE name LIKE "%cdserver%";)#");
+		stmt->executeUpdate();
+		delete stmt;
 		cdclient_fd.close();
 
+		std::string res = "python3 ../thirdparty/docker-utils/utils/fdb_to_sqlite.py " + cdclientFdbPath;
+		int r = system(res.c_str());
+		if (r != 0) {
+			Game::logger->Log("MasterServer", "Failed to convert fdb to sqlite");
+			return EXIT_FAILURE;
+		}
+		if (std::rename("./cdclient.sqlite", "./res/CDServer.sqlite") != 0) {
+			Game::logger->Log("MasterServer", "failed to move cdclient file.");
+			return EXIT_FAILURE;
+		}
+	}
+
 	//Connect to CDClient
 	try {
 		CDClientDatabase::Connect(cdclient_path);
@@ -125,6 +144,9 @@ int main(int argc, char** argv) {
 		return EXIT_FAILURE;
 	}
 
+	// Run migrations should any need to be run.
+	MigrationRunner::RunSQLiteMigrations();
+
 	//Get CDClient initial information
 	try {
 		CDClientManager::Instance()->Initialize();
@@ -136,21 +158,6 @@ int main(int argc, char** argv) {
 		return EXIT_FAILURE;
 	}
 
-		//Connect to the MySQL Database
-		std::string mysql_host = config.GetValue("mysql_host");
-		std::string mysql_database = config.GetValue("mysql_database");
-		std::string mysql_username = config.GetValue("mysql_username");
-		std::string mysql_password = config.GetValue("mysql_password");
-
-		try {
-			Database::Connect(mysql_host, mysql_database, mysql_username, mysql_password);
-		} catch (sql::SQLException& ex) {
-			Game::logger->Log("MasterServer", "Got an error while connecting to the database: %s", ex.what());
-			return EXIT_FAILURE;
-		}
-	}
-
-
 	//If the first command line argument is -a or --account then make the user
 	//input a username and password, with the password being hidden.
 	if (argc > 1 &&
@@ -825,9 +832,9 @@ void ShutdownSequence() {
 int FinalizeShutdown() {
 	//Delete our objects here:
 	Database::Destroy("MasterServer");
-	delete Game::im;
+	if (Game::im) delete Game::im;
-	delete Game::server;
+	if (Game::server) delete Game::server;
-	delete Game::logger;
+	if (Game::logger) delete Game::logger;
 
 	exit(EXIT_SUCCESS);
 	return EXIT_SUCCESS;

dWorldServer/WorldServer.cpp

@@ -1069,15 +1069,6 @@ void HandlePacket(Packet* packet) {
 								bitStream.Write<unsigned int>(1);
 								bitStream.Write(blueprintID);
 
-								bitStream.Write<uint32_t>(lxfmlSize + 9);
-
-								//Write a fake sd0 header:
-								bitStream.Write<unsigned char>(0x73); //s
-								bitStream.Write<unsigned char>(0x64); //d
-								bitStream.Write<unsigned char>(0x30); //0
-								bitStream.Write<unsigned char>(0x01); //1
-								bitStream.Write<unsigned char>(0xFF); //end magic
-
 								bitStream.Write<uint32_t>(lxfmlSize);
 
 								for (size_t i = 0; i < lxfmlSize; ++i)

dZoneManager/Level.cpp

@@ -266,7 +266,7 @@ void Level::ReadSceneObjectDataChunk(std::ifstream& file, Header& header) {
 							spawnInfo.respawnTime = std::stof(data->GetValueAsString());
 						} else if (data->GetValueType() == eLDFType::LDF_TYPE_U32) // Ints are in ms?
 						{
-							spawnInfo.respawnTime = std::stoi(data->GetValueAsString()) / 1000;
+							spawnInfo.respawnTime = std::stoul(data->GetValueAsString()) / 1000;
 						}
 					}
 					if (data->GetKey() == u"spawnsGroupOnSmash") {

dZoneManager/Zone.cpp

@@ -403,6 +403,8 @@ void Zone::LoadPath(std::ifstream& file) {
 		BinaryIO::BinaryRead(file, path.property.price);
 		BinaryIO::BinaryRead(file, path.property.rentalTime);
 		BinaryIO::BinaryRead(file, path.property.associatedZone);
+
+		if (path.pathVersion >= 5) {
 			uint8_t count1;
 			BinaryIO::BinaryRead(file, count1);
 			for (uint8_t i = 0; i < count1; ++i) {
@@ -417,16 +419,26 @@ void Zone::LoadPath(std::ifstream& file) {
 				BinaryIO::BinaryRead(file, character);
 				path.property.displayDesc.push_back(character);
 			}
+		}
+
+		if (path.pathVersion >= 6) {
 			int32_t unknown1;
 			BinaryIO::BinaryRead(file, unknown1);
+		}
+
+		if (path.pathVersion >= 7) {
 			BinaryIO::BinaryRead(file, path.property.cloneLimit);
 			BinaryIO::BinaryRead(file, path.property.repMultiplier);
 			BinaryIO::BinaryRead(file, path.property.rentalTimeUnit);
+		}
+
+		if (path.pathVersion >= 8) {
 			BinaryIO::BinaryRead(file, path.property.achievementRequired);
 			BinaryIO::BinaryRead(file, path.property.playerZoneCoords.x);
 			BinaryIO::BinaryRead(file, path.property.playerZoneCoords.y);
 			BinaryIO::BinaryRead(file, path.property.playerZoneCoords.z);
 			BinaryIO::BinaryRead(file, path.property.maxBuildHeight);
+		}
 	} else if (path.pathType == PathType::Camera) {
 		uint8_t count;
 		BinaryIO::BinaryRead(file, count);

migrations/cdserver/0_nt_footrace.sql

@@ -1,6 +1,2 @@
-BEGIN TRANSACTION;
-
 UPDATE ComponentsRegistry SET component_id = 1901 WHERE id = 12916 AND component_type = 39;
 INSERT INTO ActivityRewards (objectTemplate, ActivityRewardIndex, activityRating, LootMatrixIndex, CurrencyIndex, ChallengeRating, description) VALUES (1901, 166, -1, 598, 1, 4, 'NT Foot Race');
-
-COMMIT;

migrations/dlu/5_brick_model_sd0.sql

@@ -0,0 +1 @@
+# This file is here as a mock.  The real migration is located in BrickByBrickFix.cpp

resources/authconfig.ini

@@ -1,27 +1,6 @@
-# MySQL connection info:
-mysql_host=
-mysql_database=
-mysql_username=
-mysql_password=
-
-# The public facing IP address. Can be 'localhost' for locally hosted servers
-external_ip=localhost
-
 # Port number. The client has the authserver port hardcoded to 1001
 port=1001
 
-# Where to put crashlogs
-dump_folder=
-
-# How many clients can be connected to the server at once
-max_clients=999
-
-# 0 or 1, should log to console
-log_to_console=1
-
-# 0 or 1, should log debug (developer only) statements to console for debugging, not needed for normal operation
-log_debug_statements=0
-
 # 0 or 1, should ignore playkeys
 # If 1 everyone with an account will be able to login, regardless of if they have a key or not
 dont_use_keys=0

resources/chatconfig.ini

@@ -1,26 +1,2 @@
-# MySQL connection info:
-mysql_host=
-mysql_database=
-mysql_username=
-mysql_password=
-
-# The public facing IP address. Can be 'localhost' for locally hosted servers
-external_ip=localhost
-
 # Port number
 port=2005
-
-# Where to put crashlogs
-dump_folder=
-
-# How many clients can be connected to the server at once
-max_clients=999
-
-# 0 or 1, should log to console
-log_to_console=1
-
-# 0 or 1, should log debug (developer only) statements to console for debugging, not needed for normal operation
-log_debug_statements=0
-
-# 0 or 1, should not compile chat hash map to file
-dont_generate_dcf=0

resources/masterconfig.ini

@@ -1,12 +1,3 @@
-# MySQL connection info:
-mysql_host=
-mysql_database=
-mysql_username=
-mysql_password=
-
-# The public facing IP address. Can be 'localhost' for locally hosted servers
-external_ip=localhost
-
 # The internal ip of the master server
 master_ip=localhost
 
@@ -26,17 +17,5 @@ use_sudo_chat=0
 # Use sudo when launching world servers
 use_sudo_world=0
 
-# Where to put crashlogs
-dump_folder=
-
-# How many clients can be connected to the server at once
-max_clients=999
-
-# 0 or 1, should log to console
-log_to_console=1
-
-# 0 or 1, should log debug (developer only) statements to console for debugging, not needed for normal operation
-log_debug_statements=0
-
 # 0 or 1, should autostart auth, chat, and char servers
 prestart_servers=1

resources/sharedconfig.ini

@@ -0,0 +1,23 @@
+# MySQL connection info:
+mysql_host=
+mysql_database=
+mysql_username=
+mysql_password=
+
+# 0 or 1, should log to console
+log_to_console=1
+
+# 0 or 1, should log debug (developer only) statements to console for debugging, not needed for normal operation
+log_debug_statements=0
+
+# The public facing IP address. Can be 'localhost' for locally hosted servers
+external_ip=localhost
+
+# 0 or 1, should not compile chat hash map to file
+dont_generate_dcf=0
+
+# How many clients can be connected to the server at once
+max_clients=999
+
+# Where to put crashlogs
+dump_folder=

resources/worldconfig.ini

@@ -1,9 +1,3 @@
-# MySQL connection info:
-mysql_host=
-mysql_database=
-mysql_username=
-mysql_password=
-
 # URL to the code repository for the hosted server
 # If you fork this repository and/or make changes to the code, reflect that here to comply with AGPLv3
 source=https://github.com/DarkflameUniverse/DarkflameServer
@@ -11,21 +5,6 @@ source=https://github.com/DarkflameUniverse/DarkflameServer
 # Port to the chat server, same as in chatconfig.ini
 chat_server_port=2005
 
-# Where to put crashlogs
-dump_folder=
-
-# How many clients can be connected to the server at once
-max_clients=999
-
-# 0 or 1, should log to console
-log_to_console=1
-
-# 0 or 1, should log debug (developer only) statements to console for debugging, not needed for normal operation
-log_debug_statements=0
-
-# 0 or 1, should not compile chat hash map to file
-dont_generate_dcf=0
-
 # 0 or 1, should disable chat
 disable_chat=0
 

tests/AMFDeserializeTests.cpp

@@ -146,8 +146,8 @@ int ReadAMFArrayFromBitStream() {
 		ASSERT_EQ(res->GetValueType(), AMFValueType::AMFArray);
 		ASSERT_EQ(static_cast<AMFArrayValue*>(res.get())->GetAssociativeMap().size(), 1);
 		ASSERT_EQ(static_cast<AMFArrayValue*>(res.get())->GetDenseArray().size(), 1);
-		ASSERT_EQ(static_cast<AMFStringValue*>(static_cast<AMFArrayValue*>(res.get())->FindValue("BehaviorID"))->GetStringValue(), "10447");
+		ASSERT_EQ(static_cast<AMFArrayValue*>(res.get())->FindValue<AMFStringValue>("BehaviorID")->GetStringValue(), "10447");
-		ASSERT_EQ(static_cast<AMFStringValue*>(static_cast<AMFArrayValue*>(res.get())->GetDenseArray()[0])->GetStringValue(), "10447");
+		ASSERT_EQ(static_cast<AMFArrayValue*>(res.get())->GetValueAt<AMFStringValue>(0)->GetStringValue(), "10447");
 	}
 	// Test a dense array
 	return 0;
@@ -219,102 +219,103 @@ int TestLiveCapture() {
 	auto result = static_cast<AMFArrayValue*>(resultFromFn.get());
 	// Test the outermost array
 
-	ASSERT_EQ(dynamic_cast<AMFStringValue*>(result->FindValue("BehaviorID"))->GetStringValue(), "10447");
+	ASSERT_EQ(result->FindValue<AMFStringValue>("BehaviorID")->GetStringValue(), "10447");
-	ASSERT_EQ(dynamic_cast<AMFStringValue*>(result->FindValue("objectID"))->GetStringValue(), "288300744895913279")
+	ASSERT_EQ(result->FindValue<AMFStringValue>("objectID")->GetStringValue(), "288300744895913279");
 
 	// Test the execution state array
-		auto executionState = dynamic_cast<AMFArrayValue*>(result->FindValue("executionState"));
+	auto executionState = result->FindValue<AMFArrayValue>("executionState");
+
 	ASSERT_NE(executionState, nullptr);
 
-	auto strips = dynamic_cast<AMFArrayValue*>(executionState->FindValue("strips"))->GetDenseArray();
+	auto strips = executionState->FindValue<AMFArrayValue>("strips")->GetDenseArray();
 
 	ASSERT_EQ(strips.size(), 1);
 
 	auto stripsPosition0 = dynamic_cast<AMFArrayValue*>(strips[0]);
 
-	auto actionIndex = dynamic_cast<AMFDoubleValue*>(stripsPosition0->FindValue("actionIndex"));
+	auto actionIndex = stripsPosition0->FindValue<AMFDoubleValue>("actionIndex");
 
 	ASSERT_EQ(actionIndex->GetDoubleValue(), 0.0f);
 
-	auto stripIDExecution = dynamic_cast<AMFDoubleValue*>(stripsPosition0->FindValue("id"));
+	auto stripIDExecution = stripsPosition0->FindValue<AMFDoubleValue>("id");
 
 	ASSERT_EQ(stripIDExecution->GetDoubleValue(), 0.0f);
 
-	auto stateIDExecution = dynamic_cast<AMFDoubleValue*>(executionState->FindValue("stateID"));
+	auto stateIDExecution = executionState->FindValue<AMFDoubleValue>("stateID");
 
 	ASSERT_EQ(stateIDExecution->GetDoubleValue(), 0.0f);
 
-	auto states = dynamic_cast<AMFArrayValue*>(result->FindValue("states"))->GetDenseArray();
+	auto states = result->FindValue<AMFArrayValue>("states")->GetDenseArray();
 
 	ASSERT_EQ(states.size(), 1);
 
 	auto firstState = dynamic_cast<AMFArrayValue*>(states[0]);
 
-	auto stateID = dynamic_cast<AMFDoubleValue*>(firstState->FindValue("id"));
+	auto stateID = firstState->FindValue<AMFDoubleValue>("id");
 
 	ASSERT_EQ(stateID->GetDoubleValue(), 0.0f);
 
-	auto stripsInState = dynamic_cast<AMFArrayValue*>(firstState->FindValue("strips"))->GetDenseArray();
+	auto stripsInState = firstState->FindValue<AMFArrayValue>("strips")->GetDenseArray();
 
 	ASSERT_EQ(stripsInState.size(), 1);
 
 	auto firstStrip = dynamic_cast<AMFArrayValue*>(stripsInState[0]);
 
-	auto actionsInFirstStrip = dynamic_cast<AMFArrayValue*>(firstStrip->FindValue("actions"))->GetDenseArray();
+	auto actionsInFirstStrip = firstStrip->FindValue<AMFArrayValue>("actions")->GetDenseArray();
 
 	ASSERT_EQ(actionsInFirstStrip.size(), 3);
 
-	auto actionID = dynamic_cast<AMFDoubleValue*>(firstStrip->FindValue("id"));
+	auto actionID = firstStrip->FindValue<AMFDoubleValue>("id");
 
 	ASSERT_EQ(actionID->GetDoubleValue(), 0.0f)
 
-		auto uiArray = dynamic_cast<AMFArrayValue*>(firstStrip->FindValue("ui"));
+		auto uiArray = firstStrip->FindValue<AMFArrayValue>("ui");
 
-	auto xPos = dynamic_cast<AMFDoubleValue*>(uiArray->FindValue("x"));
+	auto xPos = uiArray->FindValue<AMFDoubleValue>("x");
-	auto yPos = dynamic_cast<AMFDoubleValue*>(uiArray->FindValue("y"));
+	auto yPos = uiArray->FindValue<AMFDoubleValue>("y");
 
 	ASSERT_EQ(xPos->GetDoubleValue(), 103.0f);
 	ASSERT_EQ(yPos->GetDoubleValue(), 82.0f);
 
-	auto stripID = dynamic_cast<AMFDoubleValue*>(firstStrip->FindValue("id"));
+	auto stripID = firstStrip->FindValue<AMFDoubleValue>("id");
 
 	ASSERT_EQ(stripID->GetDoubleValue(), 0.0f)
 
 		auto firstAction = dynamic_cast<AMFArrayValue*>(actionsInFirstStrip[0]);
 
-	auto firstType = dynamic_cast<AMFStringValue*>(firstAction->FindValue("Type"));
+	auto firstType = firstAction->FindValue<AMFStringValue>("Type");
 
 	ASSERT_EQ(firstType->GetStringValue(), "OnInteract");
 
-	auto firstCallback = dynamic_cast<AMFStringValue*>(firstAction->FindValue("__callbackID__"));
+	auto firstCallback = firstAction->FindValue<AMFStringValue>("__callbackID__");
 
 	ASSERT_EQ(firstCallback->GetStringValue(), "");
 
 	auto secondAction = dynamic_cast<AMFArrayValue*>(actionsInFirstStrip[1]);
 
-	auto secondType = dynamic_cast<AMFStringValue*>(secondAction->FindValue("Type"));
+	auto secondType = secondAction->FindValue<AMFStringValue>("Type");
 
 	ASSERT_EQ(secondType->GetStringValue(), "FlyUp");
 
-	auto secondCallback = dynamic_cast<AMFStringValue*>(secondAction->FindValue("__callbackID__"));
+	auto secondCallback = secondAction->FindValue<AMFStringValue>("__callbackID__");
 
 	ASSERT_EQ(secondCallback->GetStringValue(), "");
 
-	auto secondDistance = dynamic_cast<AMFDoubleValue*>(secondAction->FindValue("Distance"));
+	auto secondDistance = secondAction->FindValue<AMFDoubleValue>("Distance");
 
 	ASSERT_EQ(secondDistance->GetDoubleValue(), 25.0f);
 
 	auto thirdAction = dynamic_cast<AMFArrayValue*>(actionsInFirstStrip[2]);
 
-	auto thirdType = dynamic_cast<AMFStringValue*>(thirdAction->FindValue("Type"));
+	auto thirdType = thirdAction->FindValue<AMFStringValue>("Type");
 
 	ASSERT_EQ(thirdType->GetStringValue(), "FlyDown");
 
-	auto thirdCallback = dynamic_cast<AMFStringValue*>(thirdAction->FindValue("__callbackID__"));
+	auto thirdCallback = thirdAction->FindValue<AMFStringValue>("__callbackID__");
 
 	ASSERT_EQ(thirdCallback->GetStringValue(), "");
 
-	auto thirdDistance = dynamic_cast<AMFDoubleValue*>(thirdAction->FindValue("Distance"));
+	auto thirdDistance = thirdAction->FindValue<AMFDoubleValue>("Distance");
 
 	ASSERT_EQ(thirdDistance->GetDoubleValue(), 25.0f);
 
@@ -327,6 +328,42 @@ int TestNullStream() {
 	return 0;
 }
 
+int TestBadConversion() {
+	std::ifstream testFileStream;
+	testFileStream.open("AMFBitStreamTest.bin", std::ios::binary);
+
+	// Read a test BitStream from a file
+	RakNet::BitStream testBitStream;
+	char byte = 0;
+	while (testFileStream.get(byte)) {
+		testBitStream.Write<char>(byte);
+	}
+
+	testFileStream.close();
+
+	auto resultFromFn = ReadFromBitStream(&testBitStream);
+	auto result = static_cast<AMFArrayValue*>(resultFromFn.get());
+
+	// Actually a string value.
+	ASSERT_EQ(result->FindValue<AMFDoubleValue>("BehaviorID"), nullptr);
+
+	// Does not exist in the associative portion
+	ASSERT_EQ(result->FindValue<AMFNullValue>("DOES_NOT_EXIST"), nullptr);
+
+	result->PushBackValue(new AMFTrueValue());
+
+	// Exists and is correct type
+	ASSERT_NE(result->GetValueAt<AMFTrueValue>(0), nullptr);
+
+	// Value exists but is wrong typing
+	ASSERT_EQ(result->GetValueAt<AMFFalseValue>(0), nullptr);
+
+	// Value is out of bounds
+	ASSERT_EQ(result->GetValueAt<AMFTrueValue>(1), nullptr);
+
+	return 0;
+}
+
 int AMFDeserializeTests(int argc, char** const argv) {
 	std::cout << "Checking that using a null bitstream doesnt cause exception" << std::endl;
 	if (TestNullStream()) return 1;
@@ -343,6 +380,8 @@ int AMFDeserializeTests(int argc, char** const argv) {
 	if (TestLiveCapture() != 0) return 1;
 	std::cout << "Passed live capture, checking unimplemented amf values" << std::endl;
 	if (TestUnimplementedAMFValues() != 0) return 1;
+	std::cout << "Passed unimplemented values, checking poor casting" << std::endl;
+	if (TestBadConversion() != 0) return 1;
 	std::cout << "Passed all tests." << std::endl;
 	return 0;
 }