feat(cdclient): make a new CDServer.sqlite off-thread and swap tables in place

CDClientSnapshot converts a copy of the client's fdb into its own
CDServer-<hash>.sqlite with the cdserver migrations applied, on its own
connection. CDClientDatabase::Reconnect opens the new file before letting go
of the old one; CDClientManager::Reload empties every table (old entries kept
alive so references from spawned objects stay valid), retires the old fdb view
and loads again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dDatabase/CDClientDatabase/CDClientDatabase.cpp

@@ -67,6 +67,23 @@ void CDClientDatabase::Connect(const std::string& filename) {
 	sqlite3_trace_v2(conn->handle(), SQLITE_TRACE_STMT | SQLITE_TRACE_PROFILE, Trace, nullptr);
 }
 
+void CDClientDatabase::Reconnect(const std::string& filename) {
+	auto* next = new CppSQLite3DB();
+	try {
+		next->open(filename.c_str());
+	} catch (...) {
+		delete next;
+		throw;
+	}
+	sqlite3_trace_v2(next->handle(), SQLITE_TRACE_STMT | SQLITE_TRACE_PROFILE, Trace, nullptr);
+	g_Running.clear();
+	auto* old = std::exchange(conn, next);
+	isConnected = true;
+	// close_v2 waits for statements still open on the old file (a query a caller holds) before it really closes. The
+	// wrapper is left behind on purpose: its destructor would close the handle a second time
+	sqlite3_close_v2(old->handle());
+}
+
 //! Queries the CDClient
 CppSQLite3Query CDClientDatabase::ExecuteQuery(const std::string& query) {
 	return conn->execQuery(query.c_str());

dDatabase/CDClientDatabase/CDClientDatabase.h

@@ -26,6 +26,12 @@ namespace CDClientDatabase {
 	 */
 	void Connect(const std::string& filename);
 
+	/**
+	 * Switches to another CDServer.sqlite (a CDClient reload, see FdbSnapshot.h). The new file is opened before the
+	 * old connection is let go; throws CppSQLite3Exception and keeps the old one if it can't be opened.
+	 */
+	void Reconnect(const std::string& filename);
+
 	//! Queries the CDClient
 	/*!
 	  \param query The query

dDatabase/CDClientDatabase/CDClientManager.cpp

@@ -1,4 +1,9 @@
 #include "CDClientManager.h"
+
+#include <functional>
+#include <memory>
+#include <vector>
+
 #include "CDFdb.h"
 #include "Logger.h"
 #include "CDActivityRewardsTable.h"
@@ -66,7 +71,25 @@
 #define SPECIALIZE_TABLE_STORAGE(table) \
 	template<> typename table::StorageType& CDClientManager::GetEntriesMutable<table>() { return table##Entries; };
 
-#define DEFINE_TABLE_STORAGE(table) namespace { table::StorageType table##Entries; }; SPECIALIZE_TABLE_STORAGE(table)
+namespace {
+	// Empties each table's storage for Reload, keeping the old entries alive in g_Retired
+	std::vector<std::function<void()>>& Resetters() {
+		static std::vector<std::function<void()>> resetters;
+		return resetters;
+	}
+	std::vector<std::shared_ptr<void>> g_Retired;
+
+	template<typename Storage>
+	bool RegisterReset(Storage& storage) {
+		Resetters().push_back([&storage]() {
+			g_Retired.push_back(std::make_shared<Storage>(std::move(storage)));
+			storage = Storage{};
+		});
+		return true;
+	}
+}
+
+#define DEFINE_TABLE_STORAGE(table) namespace { table::StorageType table##Entries; const bool table##Reset = RegisterReset(table##Entries); }; SPECIALIZE_TABLE_STORAGE(table)
 
 DEFINE_TABLE_STORAGE(CDActivityRewardsTable);
 DEFINE_TABLE_STORAGE(CDActivitiesTable);
@@ -178,6 +201,21 @@ void CDClientManager::LoadValuesFromDatabase(const std::filesystem::path& fdbPat
 	CDZoneTableTable::LoadValuesFromDatabase();
 }
 
+void CDClientManager::ResetTables() {
+	CDFdb::Retire();
+	for (const auto& reset : Resetters()) reset();
+	CDZoneTableTable::Reset();
+}
+
+void CDClientManager::Reload(const std::filesystem::path& fdbPath) {
+	ResetTables();
+	LoadValuesFromDatabase(fdbPath);
+}
+
+uint32_t CDClientManager::GetTableCount() {
+	return static_cast<uint32_t>(Resetters().size()) + 1;
+}
+
 void CDClientManager::LoadValuesFromDefaults() {
 	LOG("Loading default CDClient tables!");
 

dDatabase/CDClientDatabase/CDClientManager.h

@@ -17,6 +17,20 @@ namespace CDClientManager {
 	void LoadValuesFromDatabase(const std::filesystem::path& fdbPath = {});
 	void LoadValuesFromDefaults();
 
+	/**
+	 * Loads every table again after a CDClient reload (the caller has already reconnected CDClientDatabase to the new
+	 * CDServer.sqlite). Each table's old entries are emptied out but kept alive, and so is the old fdb's view, so a
+	 * reference an entity took from a table before the reload stays valid: what is spawned keeps what it loaded, and
+	 * what is made afterwards reads the new data. Main thread only.
+	 */
+	void Reload(const std::filesystem::path& fdbPath);
+
+	// The first half of Reload: lets go of the fdb and empties every table, keeping the old entries alive
+	void ResetTables();
+
+	// How many tables ResetTables empties
+	uint32_t GetTableCount();
+
 	/**
 	 * Fetch a table from CDClient
 	 * 

dDatabase/CDClientDatabase/CDClientTables/CDZoneTableTable.cpp

@@ -1,8 +1,17 @@
 #include "CDZoneTableTable.h"
 
+#include <memory>
+#include <vector>
+
 namespace CDZoneTableTable {
 	Table entries;
 
+	void Reset() {
+		static std::vector<std::unique_ptr<Table>> retired;
+		retired.push_back(std::make_unique<Table>(std::move(entries)));
+		entries = {};
+	}
+
 	void LoadValuesFromDatabase() {
 		// Get the data from the database
 		auto tableData = CDClientDatabase::ExecuteQuery("SELECT * FROM ZoneTable");

dDatabase/CDClientDatabase/CDClientTables/CDZoneTableTable.h

@@ -37,6 +37,9 @@ namespace CDZoneTableTable {
 	using Table = std::map<uint32_t, CDZoneTable>;
 	void LoadValuesFromDatabase();
 
+	// Empties the table for a CDClient reload. The old entries are kept alive, so a pointer from Query stays valid
+	void Reset();
+
 	// Queries the table with a zoneID to find.
 	const CDZoneTable* Query(uint32_t zoneID);
 };

dDatabase/CDClientDatabase/CDFdb.cpp

@@ -7,6 +7,7 @@
 
 namespace {
 	std::unique_ptr<FdbReader> g_Fdb;
+	std::unique_ptr<FdbReader> g_Retired;
 
 	// Column name -> index for each table handed out by GetTable, so RowFields looks up names in O(1)
 	std::unordered_map<const FdbReader::Table*, std::unordered_map<std::string, int32_t>> g_ColumnIndices;
@@ -127,6 +128,11 @@ void CDFdb::Close() {
 	g_Fdb.reset();
 }
 
+void CDFdb::Retire() {
+	g_ColumnIndices.clear();
+	if (g_Fdb) g_Retired = std::move(g_Fdb);
+}
+
 const FdbReader* CDFdb::Get() {
 	return g_Fdb.get();
 }

dDatabase/CDClientDatabase/CDFdb.h

@@ -30,6 +30,12 @@ namespace CDFdb {
 	bool Open(const std::filesystem::path& path, bool allowMapping = true);
 	void Close();
 
+	/**
+	 * Closes the fdb for a CDClient reload, but keeps its view open until the next Retire, so a row or table pointer
+	 * handed out before the reload still reads from memory that is there. Only the latest retired fdb is kept.
+	 */
+	void Retire();
+
 	// The open fdb, or nullptr
 	const FdbReader* Get();
 

dDatabase/CDClientSnapshot.cpp

@@ -0,0 +1,116 @@
+#include "CDClientSnapshot.h"
+
+#include <fstream>
+#include <random>
+#include <sstream>
+
+#include "CppSQLite3.h"
+#include "FdbToSqlite.h"
+#include "GeneralUtils.h"
+#include "MigrationRunner.h"
+
+namespace {
+	std::string ReadFile(const std::filesystem::path& path) {
+		std::ifstream file(path, std::ios::binary);
+		std::stringstream text;
+		text << file.rdbuf();
+		return text.str();
+	}
+
+	// The same cdserver migrations RunSQLiteMigrations applies, into a fresh conversion: every one of them, since the
+	// file never had any. Recorded in its migration_history like RunSQLiteMigrations does.
+	void ApplyMigrations(CppSQLite3DB& db, const std::filesystem::path& folder, std::vector<std::string>& applied) {
+		db.execDML("CREATE TABLE IF NOT EXISTS migration_history (name TEXT NOT NULL, date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP);");
+		// RunSQLiteMigrations marks this one done on files it converts
+		db.execDML("INSERT INTO migration_history (name) VALUES ('7_migration_for_migrations.sql');");
+		std::error_code code;
+		if (!std::filesystem::is_directory(folder, code)) return;
+
+		for (const auto& name : GeneralUtils::GetSqlFileNamesFromFolder(folder.string())) {
+			const auto sql = ReadFile(folder / name);
+			if (sql.empty()) continue;
+			db.execDML("BEGIN TRANSACTION;");
+			for (const auto& statement : MigrationRunner::SplitStatements(sql, false)) {
+				if (statement.empty()) continue;
+				// A statement that fails is skipped, as RunSQLiteMigrations does
+				try {
+					db.execDML(statement.c_str());
+				} catch (const CppSQLite3Exception&) {}
+			}
+			auto record = db.compileStatement("INSERT INTO migration_history (name) VALUES (?);");
+			record.bind(1, name.c_str());
+			record.execDML();
+			record.finalize();
+			db.execDML("COMMIT;");
+			applied.push_back(name);
+		}
+	}
+}
+
+bool CDClientSnapshot::MakeSqlite(const std::filesystem::path& fdb, const std::filesystem::path& out, const std::filesystem::path& migrationsDir,
+	std::vector<std::string>& migrations, std::string& error) {
+	std::random_device random;
+	const auto temp = out.parent_path() / (out.filename().string() + "." + FdbSnapshot::HashText((static_cast<uint64_t>(random()) << 32) ^ random()) + ".tmp");
+	std::error_code code;
+
+	std::ifstream input(fdb, std::ios::binary);
+	if (!input) {
+		error = "could not open " + fdb.string();
+		return false;
+	}
+
+	try {
+		CppSQLite3DB db;
+		db.open(temp.string().c_str());
+		FdbToSqlite::Convert convert(db);
+		if (!convert.ConvertDatabase(input)) {
+			error = "could not convert " + fdb.string() + ": " + convert.GetError();
+			db.close();
+			std::filesystem::remove(temp, code);
+			return false;
+		}
+		ApplyMigrations(db, migrationsDir, migrations);
+		db.close();
+	} catch (const CppSQLite3Exception& e) {
+		error = std::string("SQLite error making ") + out.string() + ": " + e.errorMessage();
+		std::filesystem::remove(temp, code);
+		return false;
+	}
+
+	std::filesystem::rename(temp, out, code);
+	if (code) {
+		error = "could not rename to " + out.string() + ": " + code.message();
+		std::filesystem::remove(temp, code);
+		return false;
+	}
+	return true;
+}
+
+CDClientSnapshot::Result CDClientSnapshot::Build(const std::filesystem::path& clientFdb, const std::filesystem::path& resServer,
+	const std::filesystem::path& migrationsDir, std::optional<uint64_t> currentHash, const std::filesystem::path& previousFdb) {
+	Result result;
+	const auto hash = FdbSnapshot::MakeCopy(clientFdb, resServer, result.error);
+	if (!hash) return result;
+	result.hash = *hash;
+	result.current.fdb = FdbSnapshot::FdbName(*hash);
+	result.current.sqlite = FdbSnapshot::SqliteName(*hash);
+
+	if (currentHash == hash) {
+		result.ok = true;
+		result.unchanged = true;
+		return result;
+	}
+
+	std::error_code code;
+	const auto sqlite = resServer / result.current.sqlite;
+	// Made before for these bytes (the fdb went back to an older version); the file is complete, as it is renamed in
+	if (!std::filesystem::exists(sqlite, code)) {
+		if (!MakeSqlite(resServer / result.current.fdb, sqlite, migrationsDir, result.migrations, result.error)) return result;
+	}
+
+	if (!previousFdb.empty()) {
+		result.changes = FdbSnapshot::DescribeChanges(FdbSnapshot::Summarize(previousFdb), FdbSnapshot::Summarize(resServer / result.current.fdb));
+	}
+	result.ok = true;
+	return result;
+}

dDatabase/CDClientSnapshot.h

@@ -0,0 +1,48 @@
+#ifndef CDCLIENTSNAPSHOT_H
+#define CDCLIENTSNAPSHOT_H
+
+#include <cstdint>
+#include <filesystem>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "FdbSnapshot.h"
+
+/**
+ * Makes a new copy of the client's cdclient.fdb and the CDServer.sqlite that goes with it (see FdbSnapshot.h).
+ *
+ * Everything here uses its own files and SQLite connection and never logs, so master runs it on a worker thread and
+ * logs the result on the main thread.
+ */
+namespace CDClientSnapshot {
+	struct Result {
+		bool ok = false;
+		// The copy has the same bytes as the current one; nothing to do
+		bool unchanged = false;
+		std::string error;
+		uint64_t hash = 0;
+		FdbSnapshot::Current current;
+		// What the new fdb changed, one line per table (FdbSnapshot::DescribeChanges)
+		std::vector<std::string> changes;
+		// The cdserver migrations applied to the new CDServer.sqlite
+		std::vector<std::string> migrations;
+	};
+
+	/**
+	 * Converts fdb into a new SQLite file at out and applies every cdserver migration in migrationsDir to it. It is
+	 * written under a temporary name and renamed to out when complete.
+	 */
+	bool MakeSqlite(const std::filesystem::path& fdb, const std::filesystem::path& out, const std::filesystem::path& migrationsDir,
+		std::vector<std::string>& migrations, std::string& error);
+
+	/**
+	 * Copies clientFdb into resServer and, if it differs from currentHash, makes its CDServer-<hash>.sqlite.
+	 *
+	 * @param previousFdb The copy in use now, compared with the new one for Result::changes (may be empty)
+	 */
+	Result Build(const std::filesystem::path& clientFdb, const std::filesystem::path& resServer, const std::filesystem::path& migrationsDir,
+		std::optional<uint64_t> currentHash, const std::filesystem::path& previousFdb);
+};
+
+#endif // CDCLIENTSNAPSHOT_H

dDatabase/CMakeLists.txt

@@ -1,7 +1,7 @@
 add_subdirectory(CDClientDatabase)
 add_subdirectory(GameDatabase)
 
-add_library(dDatabase STATIC "MigrationRunner.cpp" "ModelNormalizeMigration.cpp" "ModularBuildIdMigration.cpp" "ConfigSync.cpp")
+add_library(dDatabase STATIC "MigrationRunner.cpp" "ModelNormalizeMigration.cpp" "ModularBuildIdMigration.cpp" "ConfigSync.cpp" "CDClientSnapshot.cpp")
 
 add_custom_target(conncpp_dylib
 	${CMAKE_COMMAND} -E copy $<TARGET_FILE:MariaDB::ConnCpp> ${PROJECT_BINARY_DIR})