From e19f6c5fdc314fcaea055e4cf7571bede4734824 Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Tue, 29 Sep 2026 23:19:55 -0500 Subject: [PATCH] feat(master): watch the client's cdclient.fdb and reload it on every server Master copies the fdb at startup, polls its size and mtime every cdclient_watch_seconds, builds the new copy and CDServer.sqlite on a worker, switches itself, writes the pointer file, tells every world and keeps the last two copies. Co-Authored-By: Claude Opus 5.5 --- dDashboardServer/routes/SettingsCatalog.cpp | 1 + dMasterServer/CDClientReloader.cpp | 198 ++++++++++++++++++++ dMasterServer/CDClientReloader.h | 47 +++++ dMasterServer/CMakeLists.txt | 1 + dMasterServer/MasterServer.cpp | 28 ++- resources/masterconfig.ini | 3 + 6 files changed, 276 insertions(+), 2 deletions(-) create mode 100644 dMasterServer/CDClientReloader.cpp create mode 100644 dMasterServer/CDClientReloader.h diff --git a/dDashboardServer/routes/SettingsCatalog.cpp b/dDashboardServer/routes/SettingsCatalog.cpp index dc5833c6f..9d12d3dd6 100644 --- a/dDashboardServer/routes/SettingsCatalog.cpp +++ b/dDashboardServer/routes/SettingsCatalog.cpp @@ -140,6 +140,7 @@ namespace { c.AddSection("Live updates", "Moving every server and world instance onto a new build without a restart (the dashboard's Live update, /liveupdate or SIGUSR2 to master). Read when one starts."); c.Add(Unit(Int(MASTER, "live_update_warn_seconds", "Warning before moving players", "The game's Mythran maintenance warning is shown this long before players are moved. The dashboard can pick another for one update.", "10", 0, 300), "seconds")); c.Add(Int(MASTER, "live_update_parallel_worlds", "Worlds at once", "World instances replaced at the same time.", "4", 1, 64)); + c.Add(Int(MASTER, "cdclient_watch_seconds", "CDClient watch interval", "Seconds between checks of the client's cdclient.fdb for changes; a change reloads it on every server. 0 turns the check off (/reloadcdclient still works).", "5", 0, 3600)); c.Add(Unit(Int(MASTER, "live_update_player_wait", "Wait for busy players", "Players who are dead or building are moved once they are done, or after this long.", "30", 0, 600), "seconds")); c.Add(Unit(Int(MASTER, "live_update_property_build_wait", "Wait for property builders", "A property is saved for its new instance once nobody builds there, or after this long (they leave build mode then).", "60", 0, 600), "seconds")); c.Add(Unit(Int(MASTER, "live_update_char_select_wait", "Wait at character select", "Players picking a character get this long to go in by themselves; then they are moved to the new character select.", "60", 0, 3600), "seconds")); diff --git a/dMasterServer/CDClientReloader.cpp b/dMasterServer/CDClientReloader.cpp new file mode 100644 index 000000000..524ed0808 --- /dev/null +++ b/dMasterServer/CDClientReloader.cpp @@ -0,0 +1,198 @@ +#include "CDClientReloader.h" + +#include +#include +#include +#include + +#include "CDClientDatabase.h" +#include "CDClientManager.h" +#include "CDClientSnapshot.h" +#include "FdbSnapshot.h" +#include "Game.h" +#include "GeneralUtils.h" +#include "Logger.h" +#include "dConfig.h" +#include "master/CDClientReload.h" + +namespace { + std::filesystem::path g_ClientFdb; + std::filesystem::path g_ResServer; + std::filesystem::path g_MigrationsDir; + bool g_Enabled = false; + + std::optional g_CurrentHash; + std::optional g_PreviousHash; + std::string g_CurrentFdb; + + FdbSnapshot::Watcher g_Watcher; + std::chrono::steady_clock::time_point g_NextPoll; + + std::future g_Job; + FdbSnapshot::Stamp g_JobStamp; + std::string g_JobWho; + std::string g_Pending; // a request while a job runs; checked after it + + std::function g_Broadcast; + + std::chrono::seconds PollInterval() { + const auto seconds = GeneralUtils::TryParse(Game::config->GetValue("cdclient_watch_seconds")).value_or(5); + return std::chrono::seconds(seconds); + } + + std::set Keep() { + std::set keep; + if (g_CurrentHash) keep.insert(*g_CurrentHash); + if (g_PreviousHash) keep.insert(*g_PreviousHash); + return keep; + } + + void RemoveOld() { + for (const auto& name : FdbSnapshot::RemoveOld(g_ResServer, Keep())) LOG("CDClient reload: removed the old %s", name.c_str()); + } + + void Start(const std::string& who) { + g_JobStamp = FdbSnapshot::StampOf(g_ClientFdb); + g_JobWho = who; + const auto previous = g_CurrentFdb.empty() ? std::filesystem::path{} : g_ResServer / g_CurrentFdb; + LOG("CDClient reload: checking %s (%s)", g_ClientFdb.string().c_str(), who.c_str()); + // The worker gets copies of everything and its own files and SQLite connection; it never logs + g_Job = std::async(std::launch::async, CDClientSnapshot::Build, g_ClientFdb, g_ResServer, g_MigrationsDir, g_CurrentHash, previous); + } + + void Finish(CDClientSnapshot::Result result) { + // Taken either way, so a broken file isn't retried every poll; the next change to it is + g_Watcher.Accept(g_JobStamp); + if (!result.ok) { + LOG("CDClient reload failed: %s", result.error.c_str()); + return; + } + if (result.unchanged) { + LOG("CDClient reload: %s has not changed", g_ClientFdb.filename().string().c_str()); + return; + } + + const auto sqlite = g_ResServer / result.current.sqlite; + const auto fdb = g_ResServer / result.current.fdb; + try { + CDClientDatabase::Reconnect(sqlite.string()); + CDClientManager::Reload(fdb); + } catch (const std::exception& e) { + LOG("CDClient reload: master could not switch to %s: %s", sqlite.string().c_str(), e.what()); + return; + } + + LOG("CDClient reload: switched to %s and %s (%s)", result.current.fdb.c_str(), result.current.sqlite.c_str(), g_JobWho.c_str()); + for (const auto& migration : result.migrations) LOG("CDClient reload: applied cdserver migration %s", migration.c_str()); + if (result.changes.empty()) LOG("CDClient reload: no table changed"); + for (const auto& change : result.changes) LOG("CDClient reload: %s", change.c_str()); + + if (!FdbSnapshot::WriteCurrent(g_ResServer, result.current)) LOG("CDClient reload: could not write %s", FdbSnapshot::CURRENT_FILE); + g_PreviousHash = g_CurrentHash; + g_CurrentHash = result.hash; + g_CurrentFdb = result.current.fdb; + + CDClientReload reload; + reload.fdb = result.current.fdb; + reload.sqlite = result.current.sqlite; + if (g_Broadcast) g_Broadcast(reload); + + // Copies older than the last two: a server still on one of those switches before the next reload, and a file + // still mapped (Windows won't remove it) is tried again next time + RemoveOld(); + } +} + +CDClientReloader::Files CDClientReloader::Init(const std::filesystem::path& clientFdb, const std::filesystem::path& resServer, const std::filesystem::path& migrationsDir) { + g_ClientFdb = clientFdb; + g_ResServer = resServer; + g_MigrationsDir = migrationsDir; + Files files{ resServer / FdbSnapshot::DEFAULT_SQLITE, {} }; + + std::error_code code; + if (!std::filesystem::is_regular_file(clientFdb, code)) { + LOG("No loose %s: reading CDClient from CDServer.sqlite only, and not watching for changes", clientFdb.string().c_str()); + return files; + } + + const auto stamp = FdbSnapshot::StampOf(clientFdb); + std::string error; + const auto hash = FdbSnapshot::MakeCopy(clientFdb, resServer, error); + if (!hash) { + LOG("Could not copy %s (%s): reading CDClient from CDServer.sqlite only", clientFdb.string().c_str(), error.c_str()); + return files; + } + + FdbSnapshot::Current current{ FdbSnapshot::FdbName(*hash), FdbSnapshot::DEFAULT_SQLITE }; + const auto pointer = FdbSnapshot::ReadCurrent(resServer); + if (pointer && pointer->fdb == current.fdb && std::filesystem::is_regular_file(resServer / pointer->sqlite, code)) { + current = *pointer; + } else if (pointer) { + // The client's fdb changed while master was down: make its CDServer.sqlite now + const auto sqlite = FdbSnapshot::SqliteName(*hash); + if (!std::filesystem::is_regular_file(resServer / sqlite, code)) { + LOG("%s changed since the last run; making %s. This may take a while", clientFdb.string().c_str(), sqlite.c_str()); + Game::logger->Flush(); + std::vector migrations; + if (!CDClientSnapshot::MakeSqlite(resServer / current.fdb, resServer / sqlite, migrationsDir, migrations, error)) { + LOG("Could not make %s (%s); using CDServer.sqlite", sqlite.c_str(), error.c_str()); + } + } + if (std::filesystem::is_regular_file(resServer / sqlite, code)) current.sqlite = sqlite; + g_PreviousHash = FdbSnapshot::ParseName(pointer->fdb); + } + // With no pointer file yet (the first run with copies), CDServer.sqlite is the one made from this fdb + + if (!FdbSnapshot::WriteCurrent(resServer, current)) LOG("Could not write %s", (resServer / FdbSnapshot::CURRENT_FILE).string().c_str()); + g_CurrentHash = *hash; + g_CurrentFdb = current.fdb; + g_Enabled = true; + g_Watcher.Accept(stamp); + RemoveOld(); + + files.sqlite = resServer / current.sqlite; + files.fdb = resServer / current.fdb; + LOG("CDClient: using %s and %s (copied from %s)", current.fdb.c_str(), current.sqlite.c_str(), clientFdb.string().c_str()); + return files; +} + +void CDClientReloader::SetBroadcast(std::function broadcast) { + g_Broadcast = std::move(broadcast); +} + +void CDClientReloader::Request(const std::string& who) { + if (!g_Enabled) { + LOG("CDClient reload asked for by %s, but there is no loose cdclient.fdb to reload from", who.c_str()); + return; + } + if (g_Job.valid()) { + g_Pending = who; + return; + } + Start(who); +} + +void CDClientReloader::Update() { + if (!g_Enabled) return; + + if (g_Job.valid()) { + if (g_Job.wait_for(std::chrono::seconds(0)) != std::future_status::ready) return; + Finish(g_Job.get()); + if (!g_Pending.empty()) { + Start(g_Pending); + g_Pending.clear(); + } + return; + } + + const auto now = std::chrono::steady_clock::now(); + if (now < g_NextPoll) return; + const auto interval = PollInterval(); + g_NextPoll = now + (interval.count() > 0 ? interval : std::chrono::seconds(60)); + if (interval.count() == 0) return; + if (g_Watcher.Poll(FdbSnapshot::StampOf(g_ClientFdb))) Start("the file changed"); +} + +void CDClientReloader::Shutdown() { + if (g_Job.valid()) g_Job.wait(); +} diff --git a/dMasterServer/CDClientReloader.h b/dMasterServer/CDClientReloader.h new file mode 100644 index 000000000..d760a6fa7 --- /dev/null +++ b/dMasterServer/CDClientReloader.h @@ -0,0 +1,47 @@ +#ifndef CDCLIENTRELOADER_H +#define CDCLIENTRELOADER_H + +#include +#include +#include + +#include "dCommonVars.h" + +struct CDClientReload; + +/** + * Master's side of the CDClient reload (docs/CDClientFdb.md). + * + * Master keeps a content-addressed copy of the client's cdclient.fdb in resServer and the CDServer.sqlite made from + * it, and every server opens those instead of the client's file. It watches the client's file (size and mtime every + * cdclient_watch_seconds) and, when it changes or a GM or the dashboard asks, makes a new copy and CDServer.sqlite on a + * worker thread, switches itself over and tells every server (CDCLIENT_RELOAD). The last two copies are kept. + */ +namespace CDClientReloader { + /** + * At startup, once resServer/CDServer.sqlite exists and before CDClientDatabase connects: makes sure the current + * copy of the client's fdb is there (and its CDServer.sqlite, if the fdb changed while master was down) and writes + * the pointer file. Main thread. + * + * @return the CDServer.sqlite to connect to and the fdb copy to map (empty when the client has no loose cdclient.fdb) + */ + struct Files { + std::filesystem::path sqlite; + std::filesystem::path fdb; + }; + Files Init(const std::filesystem::path& clientFdb, const std::filesystem::path& resServer, const std::filesystem::path& migrationsDir); + + // Tells every server to switch (CDCLIENT_RELOAD) + void SetBroadcast(std::function broadcast); + + // Checks the client's fdb now; who is for the log + void Request(const std::string& who); + + // Every frame: polls the file and finishes a reload whose worker is done. Main thread. + void Update(); + + // Waits for a running worker (at shutdown) + void Shutdown(); +}; + +#endif // CDCLIENTRELOADER_H diff --git a/dMasterServer/CMakeLists.txt b/dMasterServer/CMakeLists.txt index fcad65779..adc6c28be 100644 --- a/dMasterServer/CMakeLists.txt +++ b/dMasterServer/CMakeLists.txt @@ -1,4 +1,5 @@ set(DMASTERSERVER_SOURCES + "CDClientReloader.cpp" "InstanceManager.cpp" "MigrationCoordinator.cpp" "LiveUpdateCoordinator.cpp" diff --git a/dMasterServer/MasterServer.cpp b/dMasterServer/MasterServer.cpp index 59ddf193c..92961396b 100644 --- a/dMasterServer/MasterServer.cpp +++ b/dMasterServer/MasterServer.cpp @@ -59,6 +59,8 @@ #include "master/ServerTraffic.h" #include "master/Profiling.h" #include "master/UgcModelsMade.h" +#include "master/CDClientReload.h" +#include "CDClientReloader.h" #include "BuildInfo.h" #ifdef DARKFLAME_PLATFORM_UNIX @@ -335,9 +337,14 @@ int main(int argc, char** argv) { } } + // Servers read a copy of the client's fdb and the CDServer.sqlite made from it, never the client's own file, so it + // can be replaced while they run (docs/CDClientFdb.md) + const auto cdclientFiles = CDClientReloader::Init(Game::assetManager->GetResPath() / "cdclient.fdb", resServerPath, + BinaryPathFinder::GetBinaryDir() / "migrations" / "cdserver"); + //Connect to CDClient try { - CDClientDatabase::Connect((BinaryPathFinder::GetBinaryDir() / "resServer" / "CDServer.sqlite").string()); + CDClientDatabase::Connect(cdclientFiles.sqlite.string()); } catch (CppSQLite3Exception& e) { LOG("Unable to connect to CDServer SQLite Database"); LOG("Error: %s", e.errorMessage()); @@ -501,7 +508,7 @@ int main(int argc, char** argv) { //Get CDClient initial information try { - CDClientManager::LoadValuesFromDatabase(Game::assetManager->GetResPath() / "cdclient.fdb"); + CDClientManager::LoadValuesFromDatabase(cdclientFiles.fdb); } catch (CppSQLite3Exception& e) { LOG("Failed to initialize CDServer SQLite Database"); LOG("May be caused by corrupted file: %s", (Game::assetManager->GetResPath() / "CDServer.sqlite").string().c_str()); @@ -591,6 +598,7 @@ int main(int argc, char** argv) { } } LiveUpdateCoordinator::Update(); + CDClientReloader::Update(); CheckPlayerActionTimeouts(); // Spare instances for busy zones (zone_limits), checked every few seconds @@ -1090,6 +1098,13 @@ namespace { } } + // A GM's /reloadcdclient or the dashboard: check the client's fdb now + void OnCDClientReload(const CDClientReload& request, const SystemAddress& sysAddr) { + if (!request.IsRequest()) return; + if (sysAddr == dashboardServerMasterPeerSysAddr) CDClientReloader::Request("the dashboard"); + else CDClientReloader::Request("a GM, character " + std::to_string(request.requesterId)); + } + void OnConfigReload(const ConfigReload& reload, const SystemAddress& sysAddr) { if (sysAddr != dashboardServerMasterPeerSysAddr) { LOG("Ignoring config reload from a server that is not the dashboard"); @@ -1193,6 +1208,7 @@ namespace { // Only world servers report game writes; pass them on unchanged handlers.On(Master::DATA_CHANGED, ForwardWorldToDashboard); handlers.On(Master::MESSAGE_CAPTURE_CONTROL, OnMessageCaptureControl); + handlers.On(Master::CDCLIENT_RELOAD, OnCDClientReload); handlers.On(Master::MESSAGE_CAPTURE_DATA, OnMessageCaptureData); handlers.On(Master::REQUEST_SERVER_LIST, OnRequestServerList); handlers.On(Master::SERVER_TRAFFIC, OnServerTraffic); @@ -1288,6 +1304,7 @@ void HandlePacket(Packet* packet) { int ShutdownSequence(int32_t signal) { if (!Game::logger) return -1; + CDClientReloader::Shutdown(); LOG("Recieved Signal %d", signal); if (shutdownSequenceStarted) { LOG("Duplicate Shutdown Sequence"); @@ -1478,4 +1495,11 @@ void InitializeLiveUpdates() { }; LiveUpdateCoordinator::Initialize(std::move(hooks)); MigrationCoordinator::SetObserver(LiveUpdateCoordinator::OnMigrationStatus); + + // The worlds; the UGC and dashboard servers read the current files when they start (docs/CDClientFdb.md) + CDClientReloader::SetBroadcast([](const CDClientReload& reload) { + for (const auto& instance : Game::im->GetInstances()) { + if (instance && instance->GetIsReady() && !instance->GetIsShuttingDown()) MasterPackets::SendTo(instance->GetSysAddr(), reload); + } + }); } diff --git a/resources/masterconfig.ini b/resources/masterconfig.ini index 6db5e89b8..6fc6d901e 100644 --- a/resources/masterconfig.ini +++ b/resources/masterconfig.ini @@ -29,6 +29,9 @@ enable_ugc_server=0 live_update_warn_seconds=10 # World instances replaced at the same time live_update_parallel_worlds=4 + +# Seconds between checks of the client's cdclient.fdb for changes (a change is reloaded on every server); 0 = off +cdclient_watch_seconds=5 # Seconds dead or building players may take before they are moved anyway live_update_player_wait=30 # Seconds builders on a property may take before it is saved for its new instance anyway