From e714ff2f0a2e0d4434f5b9e6f384b6949c7b63cb Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Tue, 10 Nov 2026 18:25:53 -0600 Subject: [PATCH] feat(master): the world file watch follows the override folder, and can hold a zone For a file under maps/ master watches the file a world would read now: the override when one is there, else the client's. An override that appears, changes or goes away makes the instances that read another version stale, also for files from the packs. The dashboard can hold a zone's automatic reloads (WORLD_RELOAD_HOLD, a new message after the existing ones) until a reload of the zone is asked for. The status sent to the dashboard counts what reloads did per zone. Co-Authored-By: Claude Fable 5.2 --- dCommon/dEnums/MessageType/Master.h | 4 + dMasterServer/MasterServer.cpp | 10 ++ dMasterServer/WorldFileWatch.h | 8 +- dMasterServer/WorldReloader.cpp | 130 +++++++++++++++++++-- dMasterServer/WorldReloader.h | 6 +- dNet/master/WorldFiles.h | 36 ++++++ tests/dCommonTests/MessageIdPinTests.cpp | 3 +- tests/dCommonTests/WorldFileWatchTests.cpp | 37 ++++++ 8 files changed, 217 insertions(+), 17 deletions(-) diff --git a/dCommon/dEnums/MessageType/Master.h b/dCommon/dEnums/MessageType/Master.h index 953215a5c..5ccfc6e06 100644 --- a/dCommon/dEnums/MessageType/Master.h +++ b/dCommon/dEnums/MessageType/Master.h @@ -105,5 +105,9 @@ namespace MessageType { // Dashboard -> master: what the update check found about this build (see UpdateStatus.h), for master's log UPDATE_STATUS, + + // Dashboard -> master: stop (or go back to) replacing a zone's instances by itself when its files change, while + // a set of edited zone files waits for its reload (see WorldFiles.h, docs/ZonePublish.md) + WORLD_RELOAD_HOLD, }; } diff --git a/dMasterServer/MasterServer.cpp b/dMasterServer/MasterServer.cpp index 9d8ede971..2ca965e67 100644 --- a/dMasterServer/MasterServer.cpp +++ b/dMasterServer/MasterServer.cpp @@ -1187,6 +1187,15 @@ namespace { WorldReloader::HandleRequest(request, fromDashboard ? "the dashboard, " + by : by); } + // The dashboard, while a set of edited zone files waits for its reload (docs/ZonePublish.md) + void OnWorldReloadHold(const WorldReloadHold& hold, const SystemAddress& sysAddr) { + if (sysAddr != dashboardServerMasterPeerSysAddr || sysAddr == UNASSIGNED_SYSTEM_ADDRESS) { + LOG("Ignoring a world reload hold from a server that is not the dashboard"); + return; + } + WorldReloader::SetHold(hold.zoneId, hold.hold); + } + void OnConfigReload(const ConfigReload& reload, const SystemAddress& sysAddr) { if (sysAddr != dashboardServerMasterPeerSysAddr) { LOG("Ignoring config reload from a server that is not the dashboard"); @@ -1301,6 +1310,7 @@ namespace { handlers.On(Master::MESSAGE_CAPTURE_CONTROL, OnMessageCaptureControl); handlers.On(Master::CDCLIENT_RELOAD, OnCDClientReload); handlers.On(Master::WORLD_FILES, OnWorldFiles); + handlers.On(Master::WORLD_RELOAD_HOLD, OnWorldReloadHold); handlers.On(Master::WORLD_RELOAD, OnWorldReload); handlers.On(Master::MESSAGE_CAPTURE_DATA, OnMessageCaptureData); handlers.On(Master::REQUEST_SERVER_LIST, OnRequestServerList); diff --git a/dMasterServer/WorldFileWatch.h b/dMasterServer/WorldFileWatch.h index 5cd5adf39..915be1291 100644 --- a/dMasterServer/WorldFileWatch.h +++ b/dMasterServer/WorldFileWatch.h @@ -21,7 +21,8 @@ * * Master reads (hashes) each watched file itself: once when a world first reports it, and again whenever its size or * mtime changed and then held still for one poll (FdbSnapshot::Watcher). An instance is stale when a file it loaded - * has another hash on disk now. Files a world read from the client's packs are listed but not watched. + * has another hash on disk now. Files a world read from the client's packs are listed but not watched, unless the + * caller watches a folder whose files replace them (Poll's watchPacked). */ namespace WorldFileWatch { using Stamp = FdbSnapshot::Stamp; @@ -85,10 +86,11 @@ namespace WorldFileWatch { * read yet, and files whose size or mtime changed and held still since the last poll. Those are marked as * being hashed until Hashed is called for them. */ - std::vector Poll(const std::function& stampOf) { + std::vector Poll(const std::function& stampOf, const std::function& watchPacked = nullptr) { std::vector due; for (auto& [path, file] : m_Files) { - if (file.packed || file.hashing) continue; + // A file from the client's packs is only looked at when the caller can say what replaces it (an override folder) + if (file.hashing || (file.packed && !(watchPacked && watchPacked(path)))) continue; const auto stamp = stampOf(path); file.missing = !stamp.exists; const bool first = !file.hash && stamp.exists; diff --git a/dMasterServer/WorldReloader.cpp b/dMasterServer/WorldReloader.cpp index a94f35b3f..9ae6d39d5 100644 --- a/dMasterServer/WorldReloader.cpp +++ b/dMasterServer/WorldReloader.cpp @@ -6,6 +6,7 @@ #include #include +#include "AssetManager.h" #include "Game.h" #include "GeneralUtils.h" #include "InstanceManager.h" @@ -30,6 +31,59 @@ namespace { uint64_t size{}; }; + // What the worker reads for a job: the file that is read for the path now, or a hash known without reading + struct HashTarget { + HashJob job; + std::string readPath; + std::optional knownHash; + }; + + /** + * A file under maps/ is read from the override folder when it is there (AssetManager::SetOverrideDir), so that is + * the file watched for it, whichever of the two a world reported. name: the file's name under res/ ("maps/..."). + */ + std::optional MapsName(const std::string& path) { + if (!Game::assetManager || Game::assetManager->GetOverrideDir().empty()) return std::nullopt; + std::error_code code; + const std::filesystem::path file(path); + for (const auto& base : { Game::assetManager->GetOverrideDir(), Game::assetManager->GetResPath() }) { + if (!file.is_absolute()) break; + const auto absolute = std::filesystem::absolute(base, code).lexically_normal(); + const auto relative = file.lexically_normal().lexically_relative(absolute).generic_string(); + if (!relative.empty() && !relative.starts_with("..")) return AssetManager::OverrideName(relative); + } + // A name in the client's packs + return file.is_absolute() ? std::nullopt : AssetManager::OverrideName(path); + } + + // The loose file read for a name under maps/ now: the override, else the client's; nullopt when it is in the packs + std::optional LooseFile(const std::string& name) { + if (const auto overridden = Game::assetManager->GetOverridePath(name)) return overridden->string(); + std::error_code code; + const auto client = Game::assetManager->GetResPath() / name; + if (std::filesystem::is_regular_file(client, code)) return client.string(); + return std::nullopt; + } + + FdbSnapshot::Stamp StampNow(const std::string& path) { + const auto name = MapsName(path); + if (!name) return FdbSnapshot::StampOf(path); + if (const auto loose = LooseFile(*name)) return FdbSnapshot::StampOf(*loose); + // Read from the packs: there, and never changing, until an override shows up + return FdbSnapshot::Stamp{ 0, 0, true }; + } + + // What reloads did per zone, for the dashboard (WorldFilesStatus::Zone) + struct ZoneCounters { + uint32_t requests{}; + uint32_t acted{}; + uint32_t playersMoved{}; + uint32_t failed{}; + }; + std::map g_ZoneCounters; + + std::set g_Held; // zones whose automatic reloads wait (SetHold) + Tracker g_Tracker; std::future> g_Job; Clock::time_point g_NextPoll{}; @@ -65,23 +119,50 @@ namespace { } // The worker gets copies of the paths; it only reads the files and never logs or touches shared state - std::vector HashFiles(std::vector jobs) { + std::vector HashFiles(std::vector targets) { std::vector results; - results.reserve(jobs.size()); - for (auto& job : jobs) { + results.reserve(targets.size()); + for (auto& target : targets) { HashResult result; - std::error_code code; - const auto size = std::filesystem::file_size(job.path, code); - if (!code) { - result.size = static_cast(size); - result.hash = FdbSnapshot::HashFile(job.path); + if (target.knownHash) { + result.hash = target.knownHash; + } else { + std::error_code code; + const auto size = std::filesystem::file_size(target.readPath, code); + if (!code) { + result.size = static_cast(size); + result.hash = FdbSnapshot::HashFile(target.readPath); + } } - result.job = std::move(job); + result.job = std::move(target.job); results.push_back(std::move(result)); } return results; } + // Main thread: which file each job reads (the settings and the asset manager stay off the worker) + std::vector Targets(std::vector jobs) { + std::vector targets; + targets.reserve(jobs.size()); + for (auto& job : jobs) { + HashTarget target; + target.readPath = job.path; + if (const auto name = MapsName(job.path)) { + if (const auto loose = LooseFile(*name)) { + target.readPath = *loose; + } else { + // Back to the client's packs: what a world read from them, or (a world only ever read an override + // of it) a hash no world read + const auto file = g_Tracker.Files().find(job.path); + target.knownHash = file != g_Tracker.Files().end() && file->second.packed ? file->second.reportedHash : 0; + } + } + target.job = std::move(job); + targets.push_back(std::move(target)); + } + return targets; + } + std::vector Views() { std::vector views; for (const auto& instance : Game::im->GetInstances()) { @@ -160,6 +241,7 @@ namespace { if (refusal != InstanceMigration::eRefusal::NONE) { g_Migrations.erase(request.requestId); LOG("World reload (%s): zone %u instance %u could not be replaced: %s", by.c_str(), view.zoneId, view.instanceId, InstanceMigration::Describe(refusal)); + g_ZoneCounters[view.zoneId].failed++; counts.second++; continue; } @@ -170,6 +252,7 @@ namespace { } } for (const auto& [zone, counts] : perZone) { + g_ZoneCounters[zone].acted += counts.first; std::string message = "Reload (" + by + "): " + std::to_string(counts.first) + " instance(s) replaced, stopped or left to empty"; if (counts.second) message += ", " + std::to_string(counts.second) + " left alone"; SetZoneMessage(zone, message); @@ -181,6 +264,7 @@ namespace { void ReloadStale() { std::set wanted; for (const auto& key : g_Tracker.Stale()) { + if (g_Held.contains(key.zone)) continue; const auto tried = g_Tried.find(key); if (tried != g_Tried.end() && tried->second == g_Tracker.Signature(key)) continue; wanted.insert(key); @@ -239,6 +323,13 @@ namespace { zone.instances.push_back(row); } if (const auto message = g_ZoneMessages.find(zoneId); message != g_ZoneMessages.end()) zone.message = message->second; + if (const auto counters = g_ZoneCounters.find(zoneId); counters != g_ZoneCounters.end()) { + zone.requests = counters->second.requests; + zone.acted = counters->second.acted; + zone.playersMoved = counters->second.playersMoved; + zone.failed = counters->second.failed; + } + for (const auto& [id, key] : g_Migrations) zone.pending += key.zone == zoneId ? 1 : 0; status.zones.push_back(std::move(zone)); } return status; @@ -282,6 +373,9 @@ void WorldReloader::HandleRequest(const WorldReloadRequest& request, const std:: return; } const auto zone = request.zoneId; + g_ZoneCounters[zone].requests++; + g_Dirty = true; + if (g_Held.erase(zone)) LOG("World reload of zone %u: its automatic reloads are no longer held", zone); const auto choices = Choose(Views(), [zone](const InstanceMigration::InstanceView& view) { return view.zoneId == zone; }, KeepZones()); if (choices.empty()) { LOG("World reload of zone %u asked for by %s: no instance of it runs", zone, who.c_str()); @@ -296,6 +390,13 @@ void WorldReloader::HandleRequest(const WorldReloadRequest& request, const std:: Apply(choices, std::min(request.warnSeconds, InstanceMigrationRequest::MAX_WARN_SECONDS), request.requesterId, who); } +void WorldReloader::SetHold(uint32_t zoneId, bool hold) { + if (zoneId == 0) return; + if (hold ? g_Held.insert(zoneId).second : g_Held.erase(zoneId) > 0) { + LOG("World reload: zone %u's instances are %s when its files change", zoneId, hold ? "left alone" : "replaced again"); + } +} + void WorldReloader::OnMigrationStatus(const MigrationStatus& status) { const auto it = g_Migrations.find(status.migrationId); if (it == g_Migrations.end()) return; @@ -306,6 +407,8 @@ void WorldReloader::OnMigrationStatus(const MigrationStatus& status) { const auto key = it->second; g_Migrations.erase(it); std::string message = "Instance " + std::to_string(key.instance); + if (status.state == InstanceMigration::eState::DONE) g_ZoneCounters[key.zone].playersMoved += status.moved; + else g_ZoneCounters[key.zone].failed++; if (status.state == InstanceMigration::eState::DONE) { message += " replaced by " + std::to_string(status.targetInstance) + " (" + std::to_string(status.moved) + " player(s) moved)"; } else { @@ -319,7 +422,10 @@ void WorldReloader::OnInstanceGone(const Instance& instance) { const InstanceKey key{ instance.GetMapID(), instance.GetInstanceID() }; g_Tracker.Forget(key.zone, key.instance); g_Tried.erase(key); - if (g_Tracker.Zones().count(key.zone) == 0) g_ZoneMessages.erase(key.zone); + if (g_Tracker.Zones().count(key.zone) == 0) { + g_ZoneMessages.erase(key.zone); + g_ZoneCounters.erase(key.zone); + } g_Dirty = true; } @@ -347,8 +453,8 @@ void WorldReloader::Update() { if (seconds > 0) { // Worlds that reported after the last change are caught here too ReloadStale(); - auto due = g_Tracker.Poll([](const std::string& path) { return FdbSnapshot::StampOf(path); }); - if (!due.empty()) g_Job = std::async(std::launch::async, HashFiles, std::move(due)); + auto due = g_Tracker.Poll(StampNow, [](const std::string& path) { return MapsName(path).has_value(); }); + if (!due.empty()) g_Job = std::async(std::launch::async, HashFiles, Targets(std::move(due))); } // Player counts change without telling us; the status is compared before it is sent g_Dirty = true; diff --git a/dMasterServer/WorldReloader.h b/dMasterServer/WorldReloader.h index 9c59201c8..297d09f0d 100644 --- a/dMasterServer/WorldReloader.h +++ b/dMasterServer/WorldReloader.h @@ -14,7 +14,8 @@ struct WorldReloadRequest; /** * Master's side of world hot reload (docs/WorldHotReload.md). Worlds report the zone files they loaded (WORLD_FILES); - * master watches them (size and mtime every world_watch_seconds, then a hash on a worker thread) and, when one changes + * master watches them (size and mtime every world_watch_seconds, then a hash on a worker thread; for a file under + * maps/ that is the file in the override folder, zone_override_dir, when one is there) and, when one changes * or a GM or the dashboard asks (WORLD_RELOAD), replaces the affected instances with new ones on the files on disk, * moving their players with the instance migration (MigrationCoordinator). Main thread only. */ @@ -28,6 +29,9 @@ namespace WorldReloader { // A GM's /reloadworld or the dashboard; who is for the log void HandleRequest(const WorldReloadRequest& request, const std::string& who); + // The dashboard holds (or stops holding) a zone's automatic reloads; a request for the zone ends a hold + void SetHold(uint32_t zoneId, bool hold); + // Every migration's progress (MigrationCoordinator's observer) void OnMigrationStatus(const MigrationStatus& status); diff --git a/dNet/master/WorldFiles.h b/dNet/master/WorldFiles.h index 9c8abd629..e7db562d0 100644 --- a/dNet/master/WorldFiles.h +++ b/dNet/master/WorldFiles.h @@ -102,6 +102,30 @@ struct WorldReloadRequest : public LUBitStream { } }; +/** + * WORLD_RELOAD_HOLD (dashboard -> master): while hold is set, master does not replace zoneId's instances by itself + * when their files change (a set of edited files is in place and its reload waits for someone to say so). A + * WORLD_RELOAD of the zone still reloads it, and ends the hold. + */ +struct WorldReloadHold : public LUBitStream { + WorldReloadHold() : LUBitStream(ServiceType::MASTER, MessageType::Master::WORLD_RELOAD_HOLD) {} + + uint32_t zoneId{}; + bool hold{}; + + void Serialize(RakNet::BitStream& stream) const override { + stream.Write(zoneId); + stream.Write(hold ? 1 : 0); + } + + bool Deserialize(RakNet::BitStream& stream) override { + uint8_t value{}; + if (!stream.Read(zoneId) || !stream.Read(value)) return false; + hold = value != 0; + return true; + } +}; + /** * WORLD_FILES_STATUS (master -> dashboard): every zone that runs, the files its instances loaded as they are on disk * now, and which instances loaded an older version. Sent when something changes and when the dashboard connects. @@ -137,6 +161,12 @@ struct WorldFilesStatus : public LUBitStream { std::vector files; std::vector instances; std::string message; // the last reload of the zone + // Since master started (or the zone last had no instance), so a reader can tell what one request did + uint32_t requests{}; // reloads asked for (WORLD_RELOAD) + uint32_t acted{}; // instances replaced, stopped or left to empty + uint32_t playersMoved{}; + uint32_t failed{}; // instances that could not be replaced (message says why) + uint16_t pending{}; // instances being replaced right now bool operator==(const Zone& other) const = default; }; @@ -172,6 +202,11 @@ struct WorldFilesStatus : public LUBitStream { stream.Write((instance.stale ? 1 : 0) | (instance.reloading ? 2 : 0) | (instance.outdated ? 4 : 0)); } InstanceMigration::WriteText(stream, zone.message, WorldFiles::MAX_MESSAGE); + stream.Write(zone.requests); + stream.Write(zone.acted); + stream.Write(zone.playersMoved); + stream.Write(zone.failed); + stream.Write(zone.pending); } } @@ -205,6 +240,7 @@ struct WorldFilesStatus : public LUBitStream { instance.outdated = flags & 4; } if (!InstanceMigration::ReadText(stream, zone.message, WorldFiles::MAX_MESSAGE)) return false; + if (!stream.Read(zone.requests) || !stream.Read(zone.acted) || !stream.Read(zone.playersMoved) || !stream.Read(zone.failed) || !stream.Read(zone.pending)) return false; } return true; } diff --git a/tests/dCommonTests/MessageIdPinTests.cpp b/tests/dCommonTests/MessageIdPinTests.cpp index a52218b31..b68eec2a7 100644 --- a/tests/dCommonTests/MessageIdPinTests.cpp +++ b/tests/dCommonTests/MessageIdPinTests.cpp @@ -1766,7 +1766,7 @@ static_assert(static_cast(MessageType::Game::CAN_ITEMS_BE_REFORGED) == static_assert(static_cast(MessageType::Game::NOTIFY_CLIENT_RAIL_START_FAILED) == 1771); static_assert(static_cast(MessageType::Game::GET_IS_ON_RAIL) == 1772); -// MessageType::Master: 47 enumerators +// MessageType::Master: 48 enumerators static_assert(static_cast(MessageType::Master::REQUEST_ZONE_TRANSFER) == 1); static_assert(static_cast(MessageType::Master::REQUEST_ZONE_TRANSFER_RESPONSE) == 2); static_assert(static_cast(MessageType::Master::SERVER_INFO) == 3); @@ -1817,6 +1817,7 @@ static_assert(static_cast(MessageType::Master::WORLD_FILES) == 47); static_assert(static_cast(MessageType::Master::WORLD_RELOAD) == 48); static_assert(static_cast(MessageType::Master::WORLD_FILES_STATUS) == 49); static_assert(static_cast(MessageType::Master::UPDATE_STATUS) == 50); +static_assert(static_cast(MessageType::Master::WORLD_RELOAD_HOLD) == 51); // MessageType::Server: 3 enumerators static_assert(static_cast(MessageType::Server::VERSION_CONFIRM) == 0); diff --git a/tests/dCommonTests/WorldFileWatchTests.cpp b/tests/dCommonTests/WorldFileWatchTests.cpp index 01a303900..40b4e5b80 100644 --- a/tests/dCommonTests/WorldFileWatchTests.cpp +++ b/tests/dCommonTests/WorldFileWatchTests.cpp @@ -250,3 +250,40 @@ TEST(WorldFileWatchTests, ChooseKeepsAnInstanceOfKeptZones) { EXPECT_EQ(choices[1].action, eAction::STOP); } } + +// A file a world read from the client's packs is watched once the caller says a folder can replace it: an override +// showing up for it makes the instances that read the pack's version stale +TEST(WorldFileWatchTests, PackedFilesAreWatchedWhenAnOverrideCanReplaceThem) { + Tracker tracker; + tracker.Report(1100, 1, 0, { File("maps/a.lvl", 7, Kind::SCENE, true) }); + const auto inPack = At(0, 0); + Stamp now = inPack; + const auto poll = [&] { + return tracker.Poll([&](const std::string&) { return now; }, [](const std::string& path) { return path.starts_with("maps/"); }); + }; + // Not watched without the second argument + EXPECT_TRUE(tracker.Poll([&](const std::string&) { return now; }).empty()); + + // First read: what the pack holds + auto due = poll(); + ASSERT_EQ(due.size(), 1u); + EXPECT_FALSE(tracker.Hashed(due[0], 7, 100)); + EXPECT_TRUE(tracker.Stale().empty()); + EXPECT_TRUE(poll().empty()); + + // An override appears and holds still for a poll + now = At(50, 9); + EXPECT_TRUE(poll().empty()); + due = poll(); + ASSERT_EQ(due.size(), 1u); + EXPECT_TRUE(tracker.Hashed(due[0], 8, 50)); + EXPECT_EQ(tracker.Stale().size(), 1u); + + // It is taken away again: the pack's version is what is read, and the instance matches it again + now = inPack; + EXPECT_TRUE(poll().empty()); + due = poll(); + ASSERT_EQ(due.size(), 1u); + EXPECT_TRUE(tracker.Hashed(due[0], 7, 100)); + EXPECT_TRUE(tracker.Stale().empty()); +}