Merge remote-tracking branch 'upstream/main' into addSlashCommand

dCommon/CMakeLists.txt

@@ -16,6 +16,7 @@ set(DCOMMON_SOURCES "AMFFormat.cpp"
 		"ZCompression.cpp"
 		"BrickByBrickFix.cpp"
 		"BinaryPathFinder.cpp"
+		"FdbToSqlite.cpp"
 )
 
 add_subdirectory(dClient)

dCommon/FdbToSqlite.cpp

@@ -0,0 +1,248 @@
+#include "FdbToSqlite.h"
+
+#include <map>
+#include <fstream>
+#include <cassert>
+#include <iomanip>
+
+#include "BinaryIO.h"
+#include "CDClientDatabase.h"
+#include "GeneralUtils.h"
+#include "Game.h"
+#include "dLogger.h"
+
+#include "eSqliteDataType.h"
+
+std::map<eSqliteDataType, std::string> FdbToSqlite::Convert::sqliteType = {
+			{ eSqliteDataType::NONE, "none"},
+			{ eSqliteDataType::INT32, "int32"},
+			{ eSqliteDataType::REAL, "real"},
+			{ eSqliteDataType::TEXT_4, "text_4"},
+			{ eSqliteDataType::INT_BOOL, "int_bool"},
+			{ eSqliteDataType::INT64, "int64"},
+			{ eSqliteDataType::TEXT_8, "text_8"}
+};
+
+FdbToSqlite::Convert::Convert(std::string basePath) {
+	this->basePath = basePath;
+}
+
+bool FdbToSqlite::Convert::ConvertDatabase() {
+	fdb.open(basePath + "/cdclient.fdb", std::ios::binary);
+
+	try {
+		CDClientDatabase::Connect(basePath + "/CDServer.sqlite");
+
+		CDClientDatabase::ExecuteQuery("BEGIN TRANSACTION;");
+
+		int32_t numberOfTables = ReadInt32();
+		ReadTables(numberOfTables);
+
+		CDClientDatabase::ExecuteQuery("COMMIT;");
+	} catch (CppSQLite3Exception& e) {
+		Game::logger->Log("FdbToSqlite", "Encountered error %s converting FDB to SQLite", e.errorMessage());
+		return false;
+	}
+
+	fdb.close();
+	return true;
+}
+
+int32_t FdbToSqlite::Convert::ReadInt32() {
+	uint32_t numberOfTables{};
+	BinaryIO::BinaryRead(fdb, numberOfTables);
+	return numberOfTables;
+}
+
+int64_t FdbToSqlite::Convert::ReadInt64() {
+	int32_t prevPosition = SeekPointer();
+
+	int64_t value{};
+	BinaryIO::BinaryRead(fdb, value);
+
+	fdb.seekg(prevPosition);
+	return value;
+}
+
+std::string FdbToSqlite::Convert::ReadString() {
+	int32_t prevPosition = SeekPointer();
+
+	auto readString = BinaryIO::ReadString(fdb);
+
+	fdb.seekg(prevPosition);
+	return readString;
+}
+
+int32_t FdbToSqlite::Convert::SeekPointer() {
+	int32_t position{};
+	BinaryIO::BinaryRead(fdb, position);
+	int32_t prevPosition = fdb.tellg();
+	fdb.seekg(position);
+	return prevPosition;
+}
+
+std::string FdbToSqlite::Convert::ReadColumnHeader() {
+	int32_t prevPosition = SeekPointer();
+
+	int32_t numberOfColumns = ReadInt32();
+	std::string tableName = ReadString();
+
+	auto columns = ReadColumns(numberOfColumns);
+	std::string newTable = "CREATE TABLE IF NOT EXISTS '" + tableName + "' (" + columns + ");";
+	CDClientDatabase::ExecuteDML(newTable);
+
+	fdb.seekg(prevPosition);
+
+	return tableName;
+}
+
+void FdbToSqlite::Convert::ReadTables(int32_t& numberOfTables) {
+	int32_t prevPosition = SeekPointer();
+
+	for (int32_t i = 0; i < numberOfTables; i++) {
+		auto columnHeader = ReadColumnHeader();
+		ReadRowHeader(columnHeader);
+	}
+
+	fdb.seekg(prevPosition);
+}
+
+std::string FdbToSqlite::Convert::ReadColumns(int32_t& numberOfColumns) {
+	std::stringstream columnsToCreate;
+	int32_t prevPosition = SeekPointer();
+
+	std::string name{};
+	eSqliteDataType dataType{};
+	for (int32_t i = 0; i < numberOfColumns; i++) {
+		if (i != 0) columnsToCreate << ", ";
+		dataType = static_cast<eSqliteDataType>(ReadInt32());
+		name = ReadString();
+		columnsToCreate << "'" << name << "' " << FdbToSqlite::Convert::sqliteType[dataType];
+	}
+
+	fdb.seekg(prevPosition);
+	return columnsToCreate.str();
+}
+
+void FdbToSqlite::Convert::ReadRowHeader(std::string& tableName) {
+	int32_t prevPosition = SeekPointer();
+
+	int32_t numberOfAllocatedRows = ReadInt32();
+	if (numberOfAllocatedRows != 0) assert((numberOfAllocatedRows & (numberOfAllocatedRows - 1)) == 0);  // assert power of 2 allocation size
+	ReadRows(numberOfAllocatedRows, tableName);
+
+	fdb.seekg(prevPosition);
+}
+
+void FdbToSqlite::Convert::ReadRows(int32_t& numberOfAllocatedRows, std::string& tableName) {
+	int32_t prevPosition = SeekPointer();
+
+	int32_t rowid = 0;
+	for (int32_t row = 0; row < numberOfAllocatedRows; row++) {
+		int32_t rowPointer = ReadInt32();
+		if (rowPointer == -1) rowid++;
+		else ReadRow(rowid, rowPointer, tableName);
+	}
+
+	fdb.seekg(prevPosition);
+}
+
+void FdbToSqlite::Convert::ReadRow(int32_t& rowid, int32_t& position, std::string& tableName) {
+	int32_t prevPosition = fdb.tellg();
+	fdb.seekg(position);
+
+	while (true) {
+		ReadRowInfo(tableName);
+		int32_t linked = ReadInt32();
+
+		rowid += 1;
+
+		if (linked == -1) break;
+
+		fdb.seekg(linked);
+	}
+
+	fdb.seekg(prevPosition);
+}
+
+void FdbToSqlite::Convert::ReadRowInfo(std::string& tableName) {
+	int32_t prevPosition = SeekPointer();
+
+	int32_t numberOfColumns = ReadInt32();
+	ReadRowValues(numberOfColumns, tableName);
+
+	fdb.seekg(prevPosition);
+}
+
+void FdbToSqlite::Convert::ReadRowValues(int32_t& numberOfColumns, std::string& tableName) {
+	int32_t prevPosition = SeekPointer();
+
+	int32_t emptyValue{};
+	int32_t intValue{};
+	float_t floatValue{};
+	std::string stringValue{};
+	int32_t boolValue{};
+	int64_t int64Value{};
+	bool insertedFirstEntry = false;
+	std::stringstream insertedRow;
+	insertedRow << "INSERT INTO " << tableName << " values (";
+
+	for (int32_t i = 0; i < numberOfColumns; i++) {
+		if (i != 0) insertedRow << ", "; // Only append comma and space after first entry in row.
+		switch (static_cast<eSqliteDataType>(ReadInt32())) {
+		case eSqliteDataType::NONE:
+			BinaryIO::BinaryRead(fdb, emptyValue);
+			assert(emptyValue == 0);
+			insertedRow << "\"\"";
+			break;
+
+		case eSqliteDataType::INT32:
+			intValue = ReadInt32();
+			insertedRow << intValue;
+			break;
+
+		case eSqliteDataType::REAL:
+			BinaryIO::BinaryRead(fdb, floatValue);
+			insertedRow << std::fixed << std::setprecision(34) << floatValue; // maximum precision of floating point number
+			break;
+
+		case eSqliteDataType::TEXT_4:
+		case eSqliteDataType::TEXT_8: {
+			stringValue = ReadString();
+			size_t position = 0;
+
+			// Need to escape quote with a double of ".
+			while (position < stringValue.size()) {
+				if (stringValue.at(position) == '\"') {
+					stringValue.insert(position, "\"");
+					position++;
+				}
+				position++;
+			}
+			insertedRow << "\"" << stringValue << "\"";
+			break;
+		}
+
+		case eSqliteDataType::INT_BOOL:
+			BinaryIO::BinaryRead(fdb, boolValue);
+			insertedRow << static_cast<bool>(boolValue);
+			break;
+
+		case eSqliteDataType::INT64:
+			int64Value = ReadInt64();
+			insertedRow << std::to_string(int64Value);
+			break;
+
+		default:
+			throw std::invalid_argument("Unsupported SQLite type encountered.");
+			break;
+
+		}
+	}
+
+	insertedRow << ");";
+
+	auto copiedString = insertedRow.str();
+	CDClientDatabase::ExecuteDML(copiedString);
+	fdb.seekg(prevPosition);
+}

dCommon/FdbToSqlite.h

@@ -0,0 +1,49 @@
+#ifndef __FDBTOSQLITE__H__
+#define __FDBTOSQLITE__H__
+
+#pragma once
+
+#include <cstdint>
+#include <iosfwd>
+#include <map>
+
+enum class eSqliteDataType : int32_t;
+
+namespace FdbToSqlite {
+	class Convert {
+	public:
+		Convert(std::string inputFile);
+
+		bool ConvertDatabase();
+
+		int32_t ReadInt32();
+
+		int64_t ReadInt64();
+
+		std::string ReadString();
+
+		int32_t SeekPointer();
+
+		std::string ReadColumnHeader();
+
+		void ReadTables(int32_t& numberOfTables);
+
+		std::string ReadColumns(int32_t& numberOfColumns);
+
+		void ReadRowHeader(std::string& tableName);
+
+		void ReadRows(int32_t& numberOfAllocatedRows, std::string& tableName);
+
+		void ReadRow(int32_t& rowid, int32_t& position, std::string& tableName);
+
+		void ReadRowInfo(std::string& tableName);
+
+		void ReadRowValues(int32_t& numberOfColumns, std::string& tableName);
+	private:
+		static std::map<eSqliteDataType, std::string> sqliteType;
+		std::string basePath{};
+		std::ifstream fdb{};
+	}; // class FdbToSqlite
+}; //! namespace FdbToSqlite
+
+#endif  //!__FDBTOSQLITE__H__

dCommon/GeneralUtils.cpp

@@ -4,6 +4,8 @@
 #include <cstdint>
 #include <cassert>
 #include <algorithm>
+#include <filesystem>
+#include <map>
 
 template <typename T>
 inline size_t MinSize(size_t size, const std::basic_string_view<T>& string) {
@@ -290,51 +292,30 @@ std::u16string GeneralUtils::ReadWString(RakNet::BitStream* inStream) {
 	return string;
 }
 
-#ifdef _WIN32
-#define WIN32_LEAN_AND_MEAN
-#include <Windows.h>
-
-std::vector<std::string> GeneralUtils::GetFileNamesFromFolder(const std::string& folder) {
-	std::vector<std::string> names;
-	std::string search_path = folder + "/*.*";
-	WIN32_FIND_DATA fd;
-	HANDLE hFind = ::FindFirstFile(search_path.c_str(), &fd);
-	if (hFind != INVALID_HANDLE_VALUE) {
-		do {
-			if (!(fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) {
-				names.push_back(fd.cFileName);
-			}
-		} while (::FindNextFile(hFind, &fd));
-		::FindClose(hFind);
-	}
-	return names;
-}
-#else
-#include <stdio.h>
-#include <dirent.h>
-#include <sys/types.h>
-#include <iostream>
-#include <vector>
-#include <cstring>
-
-std::vector<std::string> GeneralUtils::GetFileNamesFromFolder(const std::string& folder) {
-	std::vector<std::string> names;
-	struct dirent* entry;
-	DIR* dir = opendir(folder.c_str());
-	if (dir == NULL) {
-		return names;
+std::vector<std::string> GeneralUtils::GetSqlFileNamesFromFolder(const std::string& folder) {
+	// Because we dont know how large the initial number before the first _ is we need to make it a map like so.
+    std::map<uint32_t, std::string> filenames{};
+	for (auto& t : std::filesystem::directory_iterator(folder)) {
+        auto filename = t.path().filename().string();
+        auto index = std::stoi(GeneralUtils::SplitString(filename, '_').at(0));
+        filenames.insert(std::make_pair(index, filename));
 	}
 
-	while ((entry = readdir(dir)) != NULL) {
-		std::string value(entry->d_name, strlen(entry->d_name));
-		if (value == "." || value == "..") {
-			continue;
+	// Now sort the map by the oldest migration.
+	std::vector<std::string> sortedFiles{};
+    auto fileIterator = filenames.begin();
+    std::map<uint32_t, std::string>::iterator oldest = filenames.begin();
+    while (!filenames.empty()) {
+		if (fileIterator == filenames.end()) {
+            sortedFiles.push_back(oldest->second);
+            filenames.erase(oldest);
+            fileIterator = filenames.begin();
+            oldest = filenames.begin();
+            continue;
 		}
-		names.push_back(value);
+        if (oldest->first > fileIterator->first) oldest = fileIterator;
+        fileIterator++;
 	}
 
-	closedir(dir);
-
-	return names;
+	return sortedFiles;
 }
-#endif

dCommon/GeneralUtils.h

@@ -12,6 +12,7 @@
 #include <BitStream.h>
 
 #include "Game.h"
+#include "dLogger.h"
 
 enum eInventoryType : uint32_t;
 
@@ -140,7 +141,7 @@ namespace GeneralUtils {
 
 	std::vector<std::string> SplitString(const std::string& str, char delimiter);
 
-	std::vector<std::string> GetFileNamesFromFolder(const std::string& folder);
+	std::vector<std::string> GetSqlFileNamesFromFolder(const std::string& folder);
 
 	template <typename T>
 	T Parse(const char* value);

dCommon/dEnums/eSqliteDataType.h

@@ -0,0 +1,16 @@
+#ifndef __ESQLITEDATATYPE__H__
+#define __ESQLITEDATATYPE__H__
+
+#include <cstdint>
+
+enum class eSqliteDataType : int32_t {
+	NONE = 0,
+	INT32,
+	REAL = 3,
+	TEXT_4,
+	INT_BOOL,
+	INT64,
+	TEXT_8 = 8
+};
+
+#endif  //!__ESQLITEDATATYPE__H__