feat(master): watch the client's cdclient.fdb and reload it on every server

Master copies the fdb at startup, polls its size and mtime every
cdclient_watch_seconds, builds the new copy and CDServer.sqlite on a
worker, switches itself, writes the pointer file, tells every world and
keeps the last two copies.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dDashboardServer/routes/SettingsCatalog.cpp

@@ -140,6 +140,7 @@ namespace {
 		c.AddSection("Live updates", "Moving every server and world instance onto a new build without a restart (the dashboard's Live update, /liveupdate or SIGUSR2 to master). Read when one starts.");
 		c.Add(Unit(Int(MASTER, "live_update_warn_seconds", "Warning before moving players", "The game's Mythran maintenance warning is shown this long before players are moved. The dashboard can pick another for one update.", "10", 0, 300), "seconds"));
 		c.Add(Int(MASTER, "live_update_parallel_worlds", "Worlds at once", "World instances replaced at the same time.", "4", 1, 64));
+		c.Add(Int(MASTER, "cdclient_watch_seconds", "CDClient watch interval", "Seconds between checks of the client's cdclient.fdb for changes; a change reloads it on every server. 0 turns the check off (/reloadcdclient still works).", "5", 0, 3600));
 		c.Add(Unit(Int(MASTER, "live_update_player_wait", "Wait for busy players", "Players who are dead or building are moved once they are done, or after this long.", "30", 0, 600), "seconds"));
 		c.Add(Unit(Int(MASTER, "live_update_property_build_wait", "Wait for property builders", "A property is saved for its new instance once nobody builds there, or after this long (they leave build mode then).", "60", 0, 600), "seconds"));
 		c.Add(Unit(Int(MASTER, "live_update_char_select_wait", "Wait at character select", "Players picking a character get this long to go in by themselves; then they are moved to the new character select.", "60", 0, 3600), "seconds"));

dMasterServer/CDClientReloader.cpp

@@ -0,0 +1,198 @@
+#include "CDClientReloader.h"
+
+#include <chrono>
+#include <future>
+#include <optional>
+#include <set>
+
+#include "CDClientDatabase.h"
+#include "CDClientManager.h"
+#include "CDClientSnapshot.h"
+#include "FdbSnapshot.h"
+#include "Game.h"
+#include "GeneralUtils.h"
+#include "Logger.h"
+#include "dConfig.h"
+#include "master/CDClientReload.h"
+
+namespace {
+	std::filesystem::path g_ClientFdb;
+	std::filesystem::path g_ResServer;
+	std::filesystem::path g_MigrationsDir;
+	bool g_Enabled = false;
+
+	std::optional<uint64_t> g_CurrentHash;
+	std::optional<uint64_t> g_PreviousHash;
+	std::string g_CurrentFdb;
+
+	FdbSnapshot::Watcher g_Watcher;
+	std::chrono::steady_clock::time_point g_NextPoll;
+
+	std::future<CDClientSnapshot::Result> g_Job;
+	FdbSnapshot::Stamp g_JobStamp;
+	std::string g_JobWho;
+	std::string g_Pending; // a request while a job runs; checked after it
+
+	std::function<void(const CDClientReload&)> g_Broadcast;
+
+	std::chrono::seconds PollInterval() {
+		const auto seconds = GeneralUtils::TryParse<uint32_t>(Game::config->GetValue("cdclient_watch_seconds")).value_or(5);
+		return std::chrono::seconds(seconds);
+	}
+
+	std::set<uint64_t> Keep() {
+		std::set<uint64_t> keep;
+		if (g_CurrentHash) keep.insert(*g_CurrentHash);
+		if (g_PreviousHash) keep.insert(*g_PreviousHash);
+		return keep;
+	}
+
+	void RemoveOld() {
+		for (const auto& name : FdbSnapshot::RemoveOld(g_ResServer, Keep())) LOG("CDClient reload: removed the old %s", name.c_str());
+	}
+
+	void Start(const std::string& who) {
+		g_JobStamp = FdbSnapshot::StampOf(g_ClientFdb);
+		g_JobWho = who;
+		const auto previous = g_CurrentFdb.empty() ? std::filesystem::path{} : g_ResServer / g_CurrentFdb;
+		LOG("CDClient reload: checking %s (%s)", g_ClientFdb.string().c_str(), who.c_str());
+		// The worker gets copies of everything and its own files and SQLite connection; it never logs
+		g_Job = std::async(std::launch::async, CDClientSnapshot::Build, g_ClientFdb, g_ResServer, g_MigrationsDir, g_CurrentHash, previous);
+	}
+
+	void Finish(CDClientSnapshot::Result result) {
+		// Taken either way, so a broken file isn't retried every poll; the next change to it is
+		g_Watcher.Accept(g_JobStamp);
+		if (!result.ok) {
+			LOG("CDClient reload failed: %s", result.error.c_str());
+			return;
+		}
+		if (result.unchanged) {
+			LOG("CDClient reload: %s has not changed", g_ClientFdb.filename().string().c_str());
+			return;
+		}
+
+		const auto sqlite = g_ResServer / result.current.sqlite;
+		const auto fdb = g_ResServer / result.current.fdb;
+		try {
+			CDClientDatabase::Reconnect(sqlite.string());
+			CDClientManager::Reload(fdb);
+		} catch (const std::exception& e) {
+			LOG("CDClient reload: master could not switch to %s: %s", sqlite.string().c_str(), e.what());
+			return;
+		}
+
+		LOG("CDClient reload: switched to %s and %s (%s)", result.current.fdb.c_str(), result.current.sqlite.c_str(), g_JobWho.c_str());
+		for (const auto& migration : result.migrations) LOG("CDClient reload: applied cdserver migration %s", migration.c_str());
+		if (result.changes.empty()) LOG("CDClient reload: no table changed");
+		for (const auto& change : result.changes) LOG("CDClient reload: %s", change.c_str());
+
+		if (!FdbSnapshot::WriteCurrent(g_ResServer, result.current)) LOG("CDClient reload: could not write %s", FdbSnapshot::CURRENT_FILE);
+		g_PreviousHash = g_CurrentHash;
+		g_CurrentHash = result.hash;
+		g_CurrentFdb = result.current.fdb;
+
+		CDClientReload reload;
+		reload.fdb = result.current.fdb;
+		reload.sqlite = result.current.sqlite;
+		if (g_Broadcast) g_Broadcast(reload);
+
+		// Copies older than the last two: a server still on one of those switches before the next reload, and a file
+		// still mapped (Windows won't remove it) is tried again next time
+		RemoveOld();
+	}
+}
+
+CDClientReloader::Files CDClientReloader::Init(const std::filesystem::path& clientFdb, const std::filesystem::path& resServer, const std::filesystem::path& migrationsDir) {
+	g_ClientFdb = clientFdb;
+	g_ResServer = resServer;
+	g_MigrationsDir = migrationsDir;
+	Files files{ resServer / FdbSnapshot::DEFAULT_SQLITE, {} };
+
+	std::error_code code;
+	if (!std::filesystem::is_regular_file(clientFdb, code)) {
+		LOG("No loose %s: reading CDClient from CDServer.sqlite only, and not watching for changes", clientFdb.string().c_str());
+		return files;
+	}
+
+	const auto stamp = FdbSnapshot::StampOf(clientFdb);
+	std::string error;
+	const auto hash = FdbSnapshot::MakeCopy(clientFdb, resServer, error);
+	if (!hash) {
+		LOG("Could not copy %s (%s): reading CDClient from CDServer.sqlite only", clientFdb.string().c_str(), error.c_str());
+		return files;
+	}
+
+	FdbSnapshot::Current current{ FdbSnapshot::FdbName(*hash), FdbSnapshot::DEFAULT_SQLITE };
+	const auto pointer = FdbSnapshot::ReadCurrent(resServer);
+	if (pointer && pointer->fdb == current.fdb && std::filesystem::is_regular_file(resServer / pointer->sqlite, code)) {
+		current = *pointer;
+	} else if (pointer) {
+		// The client's fdb changed while master was down: make its CDServer.sqlite now
+		const auto sqlite = FdbSnapshot::SqliteName(*hash);
+		if (!std::filesystem::is_regular_file(resServer / sqlite, code)) {
+			LOG("%s changed since the last run; making %s. This may take a while", clientFdb.string().c_str(), sqlite.c_str());
+			Game::logger->Flush();
+			std::vector<std::string> migrations;
+			if (!CDClientSnapshot::MakeSqlite(resServer / current.fdb, resServer / sqlite, migrationsDir, migrations, error)) {
+				LOG("Could not make %s (%s); using CDServer.sqlite", sqlite.c_str(), error.c_str());
+			}
+		}
+		if (std::filesystem::is_regular_file(resServer / sqlite, code)) current.sqlite = sqlite;
+		g_PreviousHash = FdbSnapshot::ParseName(pointer->fdb);
+	}
+	// With no pointer file yet (the first run with copies), CDServer.sqlite is the one made from this fdb
+
+	if (!FdbSnapshot::WriteCurrent(resServer, current)) LOG("Could not write %s", (resServer / FdbSnapshot::CURRENT_FILE).string().c_str());
+	g_CurrentHash = *hash;
+	g_CurrentFdb = current.fdb;
+	g_Enabled = true;
+	g_Watcher.Accept(stamp);
+	RemoveOld();
+
+	files.sqlite = resServer / current.sqlite;
+	files.fdb = resServer / current.fdb;
+	LOG("CDClient: using %s and %s (copied from %s)", current.fdb.c_str(), current.sqlite.c_str(), clientFdb.string().c_str());
+	return files;
+}
+
+void CDClientReloader::SetBroadcast(std::function<void(const CDClientReload&)> broadcast) {
+	g_Broadcast = std::move(broadcast);
+}
+
+void CDClientReloader::Request(const std::string& who) {
+	if (!g_Enabled) {
+		LOG("CDClient reload asked for by %s, but there is no loose cdclient.fdb to reload from", who.c_str());
+		return;
+	}
+	if (g_Job.valid()) {
+		g_Pending = who;
+		return;
+	}
+	Start(who);
+}
+
+void CDClientReloader::Update() {
+	if (!g_Enabled) return;
+
+	if (g_Job.valid()) {
+		if (g_Job.wait_for(std::chrono::seconds(0)) != std::future_status::ready) return;
+		Finish(g_Job.get());
+		if (!g_Pending.empty()) {
+			Start(g_Pending);
+			g_Pending.clear();
+		}
+		return;
+	}
+
+	const auto now = std::chrono::steady_clock::now();
+	if (now < g_NextPoll) return;
+	const auto interval = PollInterval();
+	g_NextPoll = now + (interval.count() > 0 ? interval : std::chrono::seconds(60));
+	if (interval.count() == 0) return;
+	if (g_Watcher.Poll(FdbSnapshot::StampOf(g_ClientFdb))) Start("the file changed");
+}
+
+void CDClientReloader::Shutdown() {
+	if (g_Job.valid()) g_Job.wait();
+}

dMasterServer/CDClientReloader.h

@@ -0,0 +1,47 @@
+#ifndef CDCLIENTRELOADER_H
+#define CDCLIENTRELOADER_H
+
+#include <filesystem>
+#include <functional>
+#include <string>
+
+#include "dCommonVars.h"
+
+struct CDClientReload;
+
+/**
+ * Master's side of the CDClient reload (docs/CDClientFdb.md).
+ *
+ * Master keeps a content-addressed copy of the client's cdclient.fdb in resServer and the CDServer.sqlite made from
+ * it, and every server opens those instead of the client's file. It watches the client's file (size and mtime every
+ * cdclient_watch_seconds) and, when it changes or a GM or the dashboard asks, makes a new copy and CDServer.sqlite on a
+ * worker thread, switches itself over and tells every server (CDCLIENT_RELOAD). The last two copies are kept.
+ */
+namespace CDClientReloader {
+	/**
+	 * At startup, once resServer/CDServer.sqlite exists and before CDClientDatabase connects: makes sure the current
+	 * copy of the client's fdb is there (and its CDServer.sqlite, if the fdb changed while master was down) and writes
+	 * the pointer file. Main thread.
+	 *
+	 * @return the CDServer.sqlite to connect to and the fdb copy to map (empty when the client has no loose cdclient.fdb)
+	 */
+	struct Files {
+		std::filesystem::path sqlite;
+		std::filesystem::path fdb;
+	};
+	Files Init(const std::filesystem::path& clientFdb, const std::filesystem::path& resServer, const std::filesystem::path& migrationsDir);
+
+	// Tells every server to switch (CDCLIENT_RELOAD)
+	void SetBroadcast(std::function<void(const CDClientReload&)> broadcast);
+
+	// Checks the client's fdb now; who is for the log
+	void Request(const std::string& who);
+
+	// Every frame: polls the file and finishes a reload whose worker is done. Main thread.
+	void Update();
+
+	// Waits for a running worker (at shutdown)
+	void Shutdown();
+};
+
+#endif // CDCLIENTRELOADER_H

dMasterServer/CMakeLists.txt

@@ -1,4 +1,5 @@
 set(DMASTERSERVER_SOURCES
+	"CDClientReloader.cpp"
 	"InstanceManager.cpp"
 	"MigrationCoordinator.cpp"
 	"LiveUpdateCoordinator.cpp"

dMasterServer/MasterServer.cpp

@@ -59,6 +59,8 @@
 #include "master/ServerTraffic.h"
 #include "master/Profiling.h"
 #include "master/UgcModelsMade.h"
+#include "master/CDClientReload.h"
+#include "CDClientReloader.h"
 #include "BuildInfo.h"
 
 #ifdef DARKFLAME_PLATFORM_UNIX
@@ -335,9 +337,14 @@ int main(int argc, char** argv) {
 		}
 	}
 
+	// Servers read a copy of the client's fdb and the CDServer.sqlite made from it, never the client's own file, so it
+	// can be replaced while they run (docs/CDClientFdb.md)
+	const auto cdclientFiles = CDClientReloader::Init(Game::assetManager->GetResPath() / "cdclient.fdb", resServerPath,
+		BinaryPathFinder::GetBinaryDir() / "migrations" / "cdserver");
+
 	//Connect to CDClient
 	try {
-		CDClientDatabase::Connect((BinaryPathFinder::GetBinaryDir() / "resServer" / "CDServer.sqlite").string());
+		CDClientDatabase::Connect(cdclientFiles.sqlite.string());
 	} catch (CppSQLite3Exception& e) {
 		LOG("Unable to connect to CDServer SQLite Database");
 		LOG("Error: %s", e.errorMessage());
@@ -501,7 +508,7 @@ int main(int argc, char** argv) {
 
 	//Get CDClient initial information
 	try {
-		CDClientManager::LoadValuesFromDatabase(Game::assetManager->GetResPath() / "cdclient.fdb");
+		CDClientManager::LoadValuesFromDatabase(cdclientFiles.fdb);
 	} catch (CppSQLite3Exception& e) {
 		LOG("Failed to initialize CDServer SQLite Database");
 		LOG("May be caused by corrupted file: %s", (Game::assetManager->GetResPath() / "CDServer.sqlite").string().c_str());
@@ -591,6 +598,7 @@ int main(int argc, char** argv) {
 			}
 		}
 		LiveUpdateCoordinator::Update();
+		CDClientReloader::Update();
 		CheckPlayerActionTimeouts();
 
 		// Spare instances for busy zones (zone_limits), checked every few seconds
@@ -1090,6 +1098,13 @@ namespace {
 		}
 	}
 
+	// A GM's /reloadcdclient or the dashboard: check the client's fdb now
+	void OnCDClientReload(const CDClientReload& request, const SystemAddress& sysAddr) {
+		if (!request.IsRequest()) return;
+		if (sysAddr == dashboardServerMasterPeerSysAddr) CDClientReloader::Request("the dashboard");
+		else CDClientReloader::Request("a GM, character " + std::to_string(request.requesterId));
+	}
+
 	void OnConfigReload(const ConfigReload& reload, const SystemAddress& sysAddr) {
 		if (sysAddr != dashboardServerMasterPeerSysAddr) {
 			LOG("Ignoring config reload from a server that is not the dashboard");
@@ -1193,6 +1208,7 @@ namespace {
 			// Only world servers report game writes; pass them on unchanged
 			handlers.On<DataChanged>(Master::DATA_CHANGED, ForwardWorldToDashboard<DataChanged>);
 			handlers.On<MessageCaptureControl>(Master::MESSAGE_CAPTURE_CONTROL, OnMessageCaptureControl);
+			handlers.On<CDClientReload>(Master::CDCLIENT_RELOAD, OnCDClientReload);
 			handlers.On<MessageCaptureData>(Master::MESSAGE_CAPTURE_DATA, OnMessageCaptureData);
 			handlers.On<RequestServerList>(Master::REQUEST_SERVER_LIST, OnRequestServerList);
 			handlers.On<ServerTraffic>(Master::SERVER_TRAFFIC, OnServerTraffic);
@@ -1288,6 +1304,7 @@ void HandlePacket(Packet* packet) {
 
 int ShutdownSequence(int32_t signal) {
 	if (!Game::logger) return -1;
+	CDClientReloader::Shutdown();
 	LOG("Recieved Signal %d", signal);
 	if (shutdownSequenceStarted) {
 		LOG("Duplicate Shutdown Sequence");
@@ -1478,4 +1495,11 @@ void InitializeLiveUpdates() {
 	};
 	LiveUpdateCoordinator::Initialize(std::move(hooks));
 	MigrationCoordinator::SetObserver(LiveUpdateCoordinator::OnMigrationStatus);
+
+	// The worlds; the UGC and dashboard servers read the current files when they start (docs/CDClientFdb.md)
+	CDClientReloader::SetBroadcast([](const CDClientReload& reload) {
+		for (const auto& instance : Game::im->GetInstances()) {
+			if (instance && instance->GetIsReady() && !instance->GetIsShuttingDown()) MasterPackets::SendTo(instance->GetSysAddr(), reload);
+		}
+	});
 }

resources/masterconfig.ini

@@ -29,6 +29,9 @@ enable_ugc_server=0
 live_update_warn_seconds=10
 # World instances replaced at the same time
 live_update_parallel_worlds=4
+
+# Seconds between checks of the client's cdclient.fdb for changes (a change is reloaded on every server); 0 = off
+cdclient_watch_seconds=5
 # Seconds dead or building players may take before they are moved anyway
 live_update_player_wait=30
 # Seconds builders on a property may take before it is saved for its new instance anyway