feat(cdclient): make a new CDServer.sqlite off-thread and swap tables in place

CDClientSnapshot converts a copy of the client's fdb into its own
CDServer-<hash>.sqlite with the cdserver migrations applied, on its own
connection. CDClientDatabase::Reconnect opens the new file before letting go
of the old one; CDClientManager::Reload empties every table (old entries kept
alive so references from spawned objects stay valid), retires the old fdb view
and loads again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dDatabase/CDClientDatabase/CDClientDatabase.cpp

@@ -67,6 +67,23 @@ void CDClientDatabase::Connect(const std::string& filename) {
 	sqlite3_trace_v2(conn->handle(), SQLITE_TRACE_STMT | SQLITE_TRACE_PROFILE, Trace, nullptr);
 }
 
+void CDClientDatabase::Reconnect(const std::string& filename) {
+	auto* next = new CppSQLite3DB();
+	try {
+		next->open(filename.c_str());
+	} catch (...) {
+		delete next;
+		throw;
+	}
+	sqlite3_trace_v2(next->handle(), SQLITE_TRACE_STMT | SQLITE_TRACE_PROFILE, Trace, nullptr);
+	g_Running.clear();
+	auto* old = std::exchange(conn, next);
+	isConnected = true;
+	// close_v2 waits for statements still open on the old file (a query a caller holds) before it really closes. The
+	// wrapper is left behind on purpose: its destructor would close the handle a second time
+	sqlite3_close_v2(old->handle());
+}
+
 //! Queries the CDClient
 CppSQLite3Query CDClientDatabase::ExecuteQuery(const std::string& query) {
 	return conn->execQuery(query.c_str());

dDatabase/CDClientDatabase/CDClientDatabase.h

@@ -26,6 +26,12 @@ namespace CDClientDatabase {
 	 */
 	void Connect(const std::string& filename);
 
+	/**
+	 * Switches to another CDServer.sqlite (a CDClient reload, see FdbSnapshot.h). The new file is opened before the
+	 * old connection is let go; throws CppSQLite3Exception and keeps the old one if it can't be opened.
+	 */
+	void Reconnect(const std::string& filename);
+
 	//! Queries the CDClient
 	/*!
 	  \param query The query

dDatabase/CDClientDatabase/CDClientManager.cpp

@@ -1,4 +1,9 @@
 #include "CDClientManager.h"
+
+#include <functional>
+#include <memory>
+#include <vector>
+
 #include "CDFdb.h"
 #include "Logger.h"
 #include "CDActivityRewardsTable.h"
@@ -66,7 +71,25 @@
 #define SPECIALIZE_TABLE_STORAGE(table) \
 	template<> typename table::StorageType& CDClientManager::GetEntriesMutable<table>() { return table##Entries; };
 
-#define DEFINE_TABLE_STORAGE(table) namespace { table::StorageType table##Entries; }; SPECIALIZE_TABLE_STORAGE(table)
+namespace {
+	// Empties each table's storage for Reload, keeping the old entries alive in g_Retired
+	std::vector<std::function<void()>>& Resetters() {
+		static std::vector<std::function<void()>> resetters;
+		return resetters;
+	}
+	std::vector<std::shared_ptr<void>> g_Retired;
+
+	template<typename Storage>
+	bool RegisterReset(Storage& storage) {
+		Resetters().push_back([&storage]() {
+			g_Retired.push_back(std::make_shared<Storage>(std::move(storage)));
+			storage = Storage{};
+		});
+		return true;
+	}
+}
+
+#define DEFINE_TABLE_STORAGE(table) namespace { table::StorageType table##Entries; const bool table##Reset = RegisterReset(table##Entries); }; SPECIALIZE_TABLE_STORAGE(table)
 
 DEFINE_TABLE_STORAGE(CDActivityRewardsTable);
 DEFINE_TABLE_STORAGE(CDActivitiesTable);
@@ -178,6 +201,21 @@ void CDClientManager::LoadValuesFromDatabase(const std::filesystem::path& fdbPat
 	CDZoneTableTable::LoadValuesFromDatabase();
 }
 
+void CDClientManager::ResetTables() {
+	CDFdb::Retire();
+	for (const auto& reset : Resetters()) reset();
+	CDZoneTableTable::Reset();
+}
+
+void CDClientManager::Reload(const std::filesystem::path& fdbPath) {
+	ResetTables();
+	LoadValuesFromDatabase(fdbPath);
+}
+
+uint32_t CDClientManager::GetTableCount() {
+	return static_cast<uint32_t>(Resetters().size()) + 1;
+}
+
 void CDClientManager::LoadValuesFromDefaults() {
 	LOG("Loading default CDClient tables!");
 

dDatabase/CDClientDatabase/CDClientManager.h

@@ -17,6 +17,20 @@ namespace CDClientManager {
 	void LoadValuesFromDatabase(const std::filesystem::path& fdbPath = {});
 	void LoadValuesFromDefaults();
 
+	/**
+	 * Loads every table again after a CDClient reload (the caller has already reconnected CDClientDatabase to the new
+	 * CDServer.sqlite). Each table's old entries are emptied out but kept alive, and so is the old fdb's view, so a
+	 * reference an entity took from a table before the reload stays valid: what is spawned keeps what it loaded, and
+	 * what is made afterwards reads the new data. Main thread only.
+	 */
+	void Reload(const std::filesystem::path& fdbPath);
+
+	// The first half of Reload: lets go of the fdb and empties every table, keeping the old entries alive
+	void ResetTables();
+
+	// How many tables ResetTables empties
+	uint32_t GetTableCount();
+
 	/**
 	 * Fetch a table from CDClient
 	 * 

dDatabase/CDClientDatabase/CDClientTables/CDZoneTableTable.cpp

@@ -1,8 +1,17 @@
 #include "CDZoneTableTable.h"
 
+#include <memory>
+#include <vector>
+
 namespace CDZoneTableTable {
 	Table entries;
 
+	void Reset() {
+		static std::vector<std::unique_ptr<Table>> retired;
+		retired.push_back(std::make_unique<Table>(std::move(entries)));
+		entries = {};
+	}
+
 	void LoadValuesFromDatabase() {
 		// Get the data from the database
 		auto tableData = CDClientDatabase::ExecuteQuery("SELECT * FROM ZoneTable");

dDatabase/CDClientDatabase/CDClientTables/CDZoneTableTable.h

@@ -37,6 +37,9 @@ namespace CDZoneTableTable {
 	using Table = std::map<uint32_t, CDZoneTable>;
 	void LoadValuesFromDatabase();
 
+	// Empties the table for a CDClient reload. The old entries are kept alive, so a pointer from Query stays valid
+	void Reset();
+
 	// Queries the table with a zoneID to find.
 	const CDZoneTable* Query(uint32_t zoneID);
 };

dDatabase/CDClientDatabase/CDFdb.cpp

@@ -7,6 +7,7 @@
 
 namespace {
 	std::unique_ptr<FdbReader> g_Fdb;
+	std::unique_ptr<FdbReader> g_Retired;
 
 	// Column name -> index for each table handed out by GetTable, so RowFields looks up names in O(1)
 	std::unordered_map<const FdbReader::Table*, std::unordered_map<std::string, int32_t>> g_ColumnIndices;
@@ -127,6 +128,11 @@ void CDFdb::Close() {
 	g_Fdb.reset();
 }
 
+void CDFdb::Retire() {
+	g_ColumnIndices.clear();
+	if (g_Fdb) g_Retired = std::move(g_Fdb);
+}
+
 const FdbReader* CDFdb::Get() {
 	return g_Fdb.get();
 }

dDatabase/CDClientDatabase/CDFdb.h

@@ -30,6 +30,12 @@ namespace CDFdb {
 	bool Open(const std::filesystem::path& path, bool allowMapping = true);
 	void Close();
 
+	/**
+	 * Closes the fdb for a CDClient reload, but keeps its view open until the next Retire, so a row or table pointer
+	 * handed out before the reload still reads from memory that is there. Only the latest retired fdb is kept.
+	 */
+	void Retire();
+
 	// The open fdb, or nullptr
 	const FdbReader* Get();
 

dDatabase/CDClientSnapshot.cpp

@@ -0,0 +1,116 @@
+#include "CDClientSnapshot.h"
+
+#include <fstream>
+#include <random>
+#include <sstream>
+
+#include "CppSQLite3.h"
+#include "FdbToSqlite.h"
+#include "GeneralUtils.h"
+#include "MigrationRunner.h"
+
+namespace {
+	std::string ReadFile(const std::filesystem::path& path) {
+		std::ifstream file(path, std::ios::binary);
+		std::stringstream text;
+		text << file.rdbuf();
+		return text.str();
+	}
+
+	// The same cdserver migrations RunSQLiteMigrations applies, into a fresh conversion: every one of them, since the
+	// file never had any. Recorded in its migration_history like RunSQLiteMigrations does.
+	void ApplyMigrations(CppSQLite3DB& db, const std::filesystem::path& folder, std::vector<std::string>& applied) {
+		db.execDML("CREATE TABLE IF NOT EXISTS migration_history (name TEXT NOT NULL, date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP);");
+		// RunSQLiteMigrations marks this one done on files it converts
+		db.execDML("INSERT INTO migration_history (name) VALUES ('7_migration_for_migrations.sql');");
+		std::error_code code;
+		if (!std::filesystem::is_directory(folder, code)) return;
+
+		for (const auto& name : GeneralUtils::GetSqlFileNamesFromFolder(folder.string())) {
+			const auto sql = ReadFile(folder / name);
+			if (sql.empty()) continue;
+			db.execDML("BEGIN TRANSACTION;");
+			for (const auto& statement : MigrationRunner::SplitStatements(sql, false)) {
+				if (statement.empty()) continue;
+				// A statement that fails is skipped, as RunSQLiteMigrations does
+				try {
+					db.execDML(statement.c_str());
+				} catch (const CppSQLite3Exception&) {}
+			}
+			auto record = db.compileStatement("INSERT INTO migration_history (name) VALUES (?);");
+			record.bind(1, name.c_str());
+			record.execDML();
+			record.finalize();
+			db.execDML("COMMIT;");
+			applied.push_back(name);
+		}
+	}
+}
+
+bool CDClientSnapshot::MakeSqlite(const std::filesystem::path& fdb, const std::filesystem::path& out, const std::filesystem::path& migrationsDir,
+	std::vector<std::string>& migrations, std::string& error) {
+	std::random_device random;
+	const auto temp = out.parent_path() / (out.filename().string() + "." + FdbSnapshot::HashText((static_cast<uint64_t>(random()) << 32) ^ random()) + ".tmp");
+	std::error_code code;
+
+	std::ifstream input(fdb, std::ios::binary);
+	if (!input) {
+		error = "could not open " + fdb.string();
+		return false;
+	}
+
+	try {
+		CppSQLite3DB db;
+		db.open(temp.string().c_str());
+		FdbToSqlite::Convert convert(db);
+		if (!convert.ConvertDatabase(input)) {
+			error = "could not convert " + fdb.string() + ": " + convert.GetError();
+			db.close();
+			std::filesystem::remove(temp, code);
+			return false;
+		}
+		ApplyMigrations(db, migrationsDir, migrations);
+		db.close();
+	} catch (const CppSQLite3Exception& e) {
+		error = std::string("SQLite error making ") + out.string() + ": " + e.errorMessage();
+		std::filesystem::remove(temp, code);
+		return false;
+	}
+
+	std::filesystem::rename(temp, out, code);
+	if (code) {
+		error = "could not rename to " + out.string() + ": " + code.message();
+		std::filesystem::remove(temp, code);
+		return false;
+	}
+	return true;
+}
+
+CDClientSnapshot::Result CDClientSnapshot::Build(const std::filesystem::path& clientFdb, const std::filesystem::path& resServer,
+	const std::filesystem::path& migrationsDir, std::optional<uint64_t> currentHash, const std::filesystem::path& previousFdb) {
+	Result result;
+	const auto hash = FdbSnapshot::MakeCopy(clientFdb, resServer, result.error);
+	if (!hash) return result;
+	result.hash = *hash;
+	result.current.fdb = FdbSnapshot::FdbName(*hash);
+	result.current.sqlite = FdbSnapshot::SqliteName(*hash);
+
+	if (currentHash == hash) {
+		result.ok = true;
+		result.unchanged = true;
+		return result;
+	}
+
+	std::error_code code;
+	const auto sqlite = resServer / result.current.sqlite;
+	// Made before for these bytes (the fdb went back to an older version); the file is complete, as it is renamed in
+	if (!std::filesystem::exists(sqlite, code)) {
+		if (!MakeSqlite(resServer / result.current.fdb, sqlite, migrationsDir, result.migrations, result.error)) return result;
+	}
+
+	if (!previousFdb.empty()) {
+		result.changes = FdbSnapshot::DescribeChanges(FdbSnapshot::Summarize(previousFdb), FdbSnapshot::Summarize(resServer / result.current.fdb));
+	}
+	result.ok = true;
+	return result;
+}

dDatabase/CDClientSnapshot.h

@@ -0,0 +1,48 @@
+#ifndef CDCLIENTSNAPSHOT_H
+#define CDCLIENTSNAPSHOT_H
+
+#include <cstdint>
+#include <filesystem>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "FdbSnapshot.h"
+
+/**
+ * Makes a new copy of the client's cdclient.fdb and the CDServer.sqlite that goes with it (see FdbSnapshot.h).
+ *
+ * Everything here uses its own files and SQLite connection and never logs, so master runs it on a worker thread and
+ * logs the result on the main thread.
+ */
+namespace CDClientSnapshot {
+	struct Result {
+		bool ok = false;
+		// The copy has the same bytes as the current one; nothing to do
+		bool unchanged = false;
+		std::string error;
+		uint64_t hash = 0;
+		FdbSnapshot::Current current;
+		// What the new fdb changed, one line per table (FdbSnapshot::DescribeChanges)
+		std::vector<std::string> changes;
+		// The cdserver migrations applied to the new CDServer.sqlite
+		std::vector<std::string> migrations;
+	};
+
+	/**
+	 * Converts fdb into a new SQLite file at out and applies every cdserver migration in migrationsDir to it. It is
+	 * written under a temporary name and renamed to out when complete.
+	 */
+	bool MakeSqlite(const std::filesystem::path& fdb, const std::filesystem::path& out, const std::filesystem::path& migrationsDir,
+		std::vector<std::string>& migrations, std::string& error);
+
+	/**
+	 * Copies clientFdb into resServer and, if it differs from currentHash, makes its CDServer-<hash>.sqlite.
+	 *
+	 * @param previousFdb The copy in use now, compared with the new one for Result::changes (may be empty)
+	 */
+	Result Build(const std::filesystem::path& clientFdb, const std::filesystem::path& resServer, const std::filesystem::path& migrationsDir,
+		std::optional<uint64_t> currentHash, const std::filesystem::path& previousFdb);
+};
+
+#endif // CDCLIENTSNAPSHOT_H

dDatabase/CMakeLists.txt

@@ -1,7 +1,7 @@
 add_subdirectory(CDClientDatabase)
 add_subdirectory(GameDatabase)
 
-add_library(dDatabase STATIC "MigrationRunner.cpp" "ModelNormalizeMigration.cpp" "ModularBuildIdMigration.cpp" "ConfigSync.cpp")
+add_library(dDatabase STATIC "MigrationRunner.cpp" "ModelNormalizeMigration.cpp" "ModularBuildIdMigration.cpp" "ConfigSync.cpp" "CDClientSnapshot.cpp")
 
 add_custom_target(conncpp_dylib
 	${CMAKE_COMMAND} -E copy $<TARGET_FILE:MariaDB::ConnCpp> ${PROJECT_BINARY_DIR})

tests/dGameTests/CDClientReloadTests.cpp

@@ -0,0 +1,124 @@
+#include <gtest/gtest.h>
+
+#include <filesystem>
+#include <fstream>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "CDClientDatabase.h"
+#include "CDClientManager.h"
+#include "CDClientSnapshot.h"
+#include "CDComponentsRegistryTable.h"
+#include "CDFdb.h"
+#include "CDItemComponentTable.h"
+#include "CDObjectsTable.h"
+#include "Game.h"
+#include "Logger.h"
+#include "eReplicaComponentType.h"
+#include "../FdbTestWriter.h"
+
+using FdbTestWriter::Value;
+
+namespace {
+	int32_t Component(uint32_t id, int32_t type) {
+		return CDComponentsRegistryTable::Instance().GetByIDAndType(id, static_cast<eReplicaComponentType>(type), -99);
+	}
+
+	std::vector<uint8_t> Registry(int32_t lot1Component) {
+		FdbTestWriter::Table registry;
+		registry.name = "ComponentsRegistry";
+		registry.columns = { { "id", eSqliteDataType::INT32 }, { "component_type", eSqliteDataType::INT32 }, { "component_id", eSqliteDataType::INT32 } };
+		registry.bucketCount = 4;
+		registry.rows = {
+			{ Value::Int(1), Value::Int(1), Value::Int(lot1Component) },
+			{ Value::Int(5), Value::Int(1), Value::Int(50) },
+		};
+		return FdbTestWriter::Write({ registry });
+	}
+
+	class CDClientReloadTest : public ::testing::Test {
+	protected:
+		void SetUp() override {
+			m_Logger = std::make_unique<Logger>("./testing.log", false, false);
+			Game::logger = m_Logger.get();
+			m_Dir = std::filesystem::temp_directory_path() / ("dlu_cdreload_" + std::string(::testing::UnitTest::GetInstance()->current_test_info()->name()));
+			std::filesystem::remove_all(m_Dir);
+			std::filesystem::create_directories(m_Dir / "res");
+			std::filesystem::create_directories(m_Dir / "resServer");
+			std::filesystem::create_directories(m_Dir / "migrations");
+			// A cdserver migration that changes one row, like the real ones do
+			std::ofstream(m_Dir / "migrations" / "1_change_lot_5.sql") << "UPDATE ComponentsRegistry SET component_id = 55 WHERE id = 5; -- a; comment\n";
+		}
+
+		void TearDown() override {
+			CDFdb::Close();
+			CDComponentsRegistryTable::Instance().LoadFromFdb();
+			CDItemComponentTable::Instance().LoadFromFdb();
+			CDObjectsTable::Instance().LoadFromFdb();
+			CDClientManager::ResetTables();
+			CDFdb::Retire();
+			Game::logger = nullptr;
+			std::error_code error;
+			std::filesystem::remove_all(m_Dir, error);
+		}
+
+		// What master does on a change, then what every server does with the result
+		CDClientSnapshot::Result BuildAndSwap(std::optional<uint64_t> currentHash, const std::filesystem::path& previous) {
+			auto result = CDClientSnapshot::Build(m_Dir / "res" / "cdclient.fdb", m_Dir / "resServer", m_Dir / "migrations", currentHash, previous);
+			if (!result.ok || result.unchanged) return result;
+			CDClientDatabase::Reconnect((m_Dir / "resServer" / result.current.sqlite).string());
+			CDClientManager::ResetTables();
+			EXPECT_TRUE(CDFdb::Open(m_Dir / "resServer" / result.current.fdb));
+			EXPECT_TRUE(CDComponentsRegistryTable::Instance().LoadFromFdb());
+			return result;
+		}
+
+		std::unique_ptr<Logger> m_Logger;
+		std::filesystem::path m_Dir;
+	};
+}
+
+TEST_F(CDClientReloadTest, ResetTablesEmptiesEveryTableButKeepsOldEntriesAlive) {
+	EXPECT_GE(CDClientManager::GetTableCount(), 40u);
+
+	auto& items = CDClientManager::GetEntriesMutable<CDItemComponentTable>();
+	items[7].baseValue = 1234;
+	const auto& held = items[7];
+
+	CDClientManager::ResetTables();
+	EXPECT_TRUE(CDClientManager::GetEntriesMutable<CDItemComponentTable>().empty());
+	// An entity that took this entry before the reload still reads it
+	EXPECT_EQ(held.baseValue, 1234u);
+}
+
+TEST_F(CDClientReloadTest, SwapsToANewFdbAndSqlite) {
+	FdbTestWriter::WriteFile(m_Dir / "res" / "cdclient.fdb", Registry(10));
+	const auto first = BuildAndSwap(std::nullopt, {});
+	ASSERT_TRUE(first.ok) << first.error;
+	EXPECT_EQ(first.migrations, std::vector<std::string>{ "1_change_lot_5.sql" });
+	EXPECT_EQ(Component(1, 1), 10);
+	// From the migration, which overrides the fdb row
+	EXPECT_EQ(Component(5, 1), 55);
+
+	// Same bytes: nothing to do
+	const auto same = CDClientSnapshot::Build(m_Dir / "res" / "cdclient.fdb", m_Dir / "resServer", m_Dir / "migrations", first.hash, {});
+	EXPECT_TRUE(same.ok);
+	EXPECT_TRUE(same.unchanged);
+
+	// The client's fdb is edited while the server maps its copy
+	FdbTestWriter::WriteFile(m_Dir / "res" / "cdclient.fdb", Registry(11));
+	const auto second = BuildAndSwap(first.hash, m_Dir / "resServer" / first.current.fdb);
+	ASSERT_TRUE(second.ok) << second.error;
+	EXPECT_NE(second.hash, first.hash);
+	EXPECT_EQ(second.changes, std::vector<std::string>{ "ComponentsRegistry: 2 -> 2 rows (values changed)" });
+	EXPECT_EQ(Component(1, 1), 11);
+	EXPECT_EQ(Component(5, 1), 55);
+
+	// Both copies are there until the old one is removed
+	EXPECT_TRUE(std::filesystem::exists(m_Dir / "resServer" / first.current.fdb));
+	EXPECT_TRUE(std::filesystem::exists(m_Dir / "resServer" / second.current.sqlite));
+	FdbSnapshot::RemoveOld(m_Dir / "resServer", { second.hash });
+	EXPECT_TRUE(std::filesystem::exists(m_Dir / "resServer" / second.current.fdb));
+	EXPECT_EQ(Component(1, 1), 11);
+}

tests/dGameTests/CMakeLists.txt

@@ -28,6 +28,7 @@ set(DGAMETEST_SOURCES
 	"SpiderQueenTests.cpp"
 	"TacArcTests.cpp"
 	"CDFdbTests.cpp"
+	"CDClientReloadTests.cpp"
 	"CDFdbBenchmark.cpp"
 )