diff --git a/dCommon/dEnums/MessageType/Master.h b/dCommon/dEnums/MessageType/Master.h index 953215a5c..5ccfc6e06 100644 --- a/dCommon/dEnums/MessageType/Master.h +++ b/dCommon/dEnums/MessageType/Master.h @@ -105,5 +105,9 @@ namespace MessageType { // Dashboard -> master: what the update check found about this build (see UpdateStatus.h), for master's log UPDATE_STATUS, + + // Dashboard -> master: stop (or go back to) replacing a zone's instances by itself when its files change, while + // a set of edited zone files waits for its reload (see WorldFiles.h, docs/ZonePublish.md) + WORLD_RELOAD_HOLD, }; } diff --git a/dMasterServer/MasterServer.cpp b/dMasterServer/MasterServer.cpp index 9d8ede971..2ca965e67 100644 --- a/dMasterServer/MasterServer.cpp +++ b/dMasterServer/MasterServer.cpp @@ -1187,6 +1187,15 @@ namespace { WorldReloader::HandleRequest(request, fromDashboard ? "the dashboard, " + by : by); } + // The dashboard, while a set of edited zone files waits for its reload (docs/ZonePublish.md) + void OnWorldReloadHold(const WorldReloadHold& hold, const SystemAddress& sysAddr) { + if (sysAddr != dashboardServerMasterPeerSysAddr || sysAddr == UNASSIGNED_SYSTEM_ADDRESS) { + LOG("Ignoring a world reload hold from a server that is not the dashboard"); + return; + } + WorldReloader::SetHold(hold.zoneId, hold.hold); + } + void OnConfigReload(const ConfigReload& reload, const SystemAddress& sysAddr) { if (sysAddr != dashboardServerMasterPeerSysAddr) { LOG("Ignoring config reload from a server that is not the dashboard"); @@ -1301,6 +1310,7 @@ namespace { handlers.On(Master::MESSAGE_CAPTURE_CONTROL, OnMessageCaptureControl); handlers.On(Master::CDCLIENT_RELOAD, OnCDClientReload); handlers.On(Master::WORLD_FILES, OnWorldFiles); + handlers.On(Master::WORLD_RELOAD_HOLD, OnWorldReloadHold); handlers.On(Master::WORLD_RELOAD, OnWorldReload); handlers.On(Master::MESSAGE_CAPTURE_DATA, OnMessageCaptureData); handlers.On(Master::REQUEST_SERVER_LIST, OnRequestServerList); diff --git a/dMasterServer/WorldFileWatch.h b/dMasterServer/WorldFileWatch.h index 5cd5adf39..915be1291 100644 --- a/dMasterServer/WorldFileWatch.h +++ b/dMasterServer/WorldFileWatch.h @@ -21,7 +21,8 @@ * * Master reads (hashes) each watched file itself: once when a world first reports it, and again whenever its size or * mtime changed and then held still for one poll (FdbSnapshot::Watcher). An instance is stale when a file it loaded - * has another hash on disk now. Files a world read from the client's packs are listed but not watched. + * has another hash on disk now. Files a world read from the client's packs are listed but not watched, unless the + * caller watches a folder whose files replace them (Poll's watchPacked). */ namespace WorldFileWatch { using Stamp = FdbSnapshot::Stamp; @@ -85,10 +86,11 @@ namespace WorldFileWatch { * read yet, and files whose size or mtime changed and held still since the last poll. Those are marked as * being hashed until Hashed is called for them. */ - std::vector Poll(const std::function& stampOf) { + std::vector Poll(const std::function& stampOf, const std::function& watchPacked = nullptr) { std::vector due; for (auto& [path, file] : m_Files) { - if (file.packed || file.hashing) continue; + // A file from the client's packs is only looked at when the caller can say what replaces it (an override folder) + if (file.hashing || (file.packed && !(watchPacked && watchPacked(path)))) continue; const auto stamp = stampOf(path); file.missing = !stamp.exists; const bool first = !file.hash && stamp.exists; diff --git a/dMasterServer/WorldReloader.cpp b/dMasterServer/WorldReloader.cpp index a94f35b3f..9ae6d39d5 100644 --- a/dMasterServer/WorldReloader.cpp +++ b/dMasterServer/WorldReloader.cpp @@ -6,6 +6,7 @@ #include #include +#include "AssetManager.h" #include "Game.h" #include "GeneralUtils.h" #include "InstanceManager.h" @@ -30,6 +31,59 @@ namespace { uint64_t size{}; }; + // What the worker reads for a job: the file that is read for the path now, or a hash known without reading + struct HashTarget { + HashJob job; + std::string readPath; + std::optional knownHash; + }; + + /** + * A file under maps/ is read from the override folder when it is there (AssetManager::SetOverrideDir), so that is + * the file watched for it, whichever of the two a world reported. name: the file's name under res/ ("maps/..."). + */ + std::optional MapsName(const std::string& path) { + if (!Game::assetManager || Game::assetManager->GetOverrideDir().empty()) return std::nullopt; + std::error_code code; + const std::filesystem::path file(path); + for (const auto& base : { Game::assetManager->GetOverrideDir(), Game::assetManager->GetResPath() }) { + if (!file.is_absolute()) break; + const auto absolute = std::filesystem::absolute(base, code).lexically_normal(); + const auto relative = file.lexically_normal().lexically_relative(absolute).generic_string(); + if (!relative.empty() && !relative.starts_with("..")) return AssetManager::OverrideName(relative); + } + // A name in the client's packs + return file.is_absolute() ? std::nullopt : AssetManager::OverrideName(path); + } + + // The loose file read for a name under maps/ now: the override, else the client's; nullopt when it is in the packs + std::optional LooseFile(const std::string& name) { + if (const auto overridden = Game::assetManager->GetOverridePath(name)) return overridden->string(); + std::error_code code; + const auto client = Game::assetManager->GetResPath() / name; + if (std::filesystem::is_regular_file(client, code)) return client.string(); + return std::nullopt; + } + + FdbSnapshot::Stamp StampNow(const std::string& path) { + const auto name = MapsName(path); + if (!name) return FdbSnapshot::StampOf(path); + if (const auto loose = LooseFile(*name)) return FdbSnapshot::StampOf(*loose); + // Read from the packs: there, and never changing, until an override shows up + return FdbSnapshot::Stamp{ 0, 0, true }; + } + + // What reloads did per zone, for the dashboard (WorldFilesStatus::Zone) + struct ZoneCounters { + uint32_t requests{}; + uint32_t acted{}; + uint32_t playersMoved{}; + uint32_t failed{}; + }; + std::map g_ZoneCounters; + + std::set g_Held; // zones whose automatic reloads wait (SetHold) + Tracker g_Tracker; std::future> g_Job; Clock::time_point g_NextPoll{}; @@ -65,23 +119,50 @@ namespace { } // The worker gets copies of the paths; it only reads the files and never logs or touches shared state - std::vector HashFiles(std::vector jobs) { + std::vector HashFiles(std::vector targets) { std::vector results; - results.reserve(jobs.size()); - for (auto& job : jobs) { + results.reserve(targets.size()); + for (auto& target : targets) { HashResult result; - std::error_code code; - const auto size = std::filesystem::file_size(job.path, code); - if (!code) { - result.size = static_cast(size); - result.hash = FdbSnapshot::HashFile(job.path); + if (target.knownHash) { + result.hash = target.knownHash; + } else { + std::error_code code; + const auto size = std::filesystem::file_size(target.readPath, code); + if (!code) { + result.size = static_cast(size); + result.hash = FdbSnapshot::HashFile(target.readPath); + } } - result.job = std::move(job); + result.job = std::move(target.job); results.push_back(std::move(result)); } return results; } + // Main thread: which file each job reads (the settings and the asset manager stay off the worker) + std::vector Targets(std::vector jobs) { + std::vector targets; + targets.reserve(jobs.size()); + for (auto& job : jobs) { + HashTarget target; + target.readPath = job.path; + if (const auto name = MapsName(job.path)) { + if (const auto loose = LooseFile(*name)) { + target.readPath = *loose; + } else { + // Back to the client's packs: what a world read from them, or (a world only ever read an override + // of it) a hash no world read + const auto file = g_Tracker.Files().find(job.path); + target.knownHash = file != g_Tracker.Files().end() && file->second.packed ? file->second.reportedHash : 0; + } + } + target.job = std::move(job); + targets.push_back(std::move(target)); + } + return targets; + } + std::vector Views() { std::vector views; for (const auto& instance : Game::im->GetInstances()) { @@ -160,6 +241,7 @@ namespace { if (refusal != InstanceMigration::eRefusal::NONE) { g_Migrations.erase(request.requestId); LOG("World reload (%s): zone %u instance %u could not be replaced: %s", by.c_str(), view.zoneId, view.instanceId, InstanceMigration::Describe(refusal)); + g_ZoneCounters[view.zoneId].failed++; counts.second++; continue; } @@ -170,6 +252,7 @@ namespace { } } for (const auto& [zone, counts] : perZone) { + g_ZoneCounters[zone].acted += counts.first; std::string message = "Reload (" + by + "): " + std::to_string(counts.first) + " instance(s) replaced, stopped or left to empty"; if (counts.second) message += ", " + std::to_string(counts.second) + " left alone"; SetZoneMessage(zone, message); @@ -181,6 +264,7 @@ namespace { void ReloadStale() { std::set wanted; for (const auto& key : g_Tracker.Stale()) { + if (g_Held.contains(key.zone)) continue; const auto tried = g_Tried.find(key); if (tried != g_Tried.end() && tried->second == g_Tracker.Signature(key)) continue; wanted.insert(key); @@ -239,6 +323,13 @@ namespace { zone.instances.push_back(row); } if (const auto message = g_ZoneMessages.find(zoneId); message != g_ZoneMessages.end()) zone.message = message->second; + if (const auto counters = g_ZoneCounters.find(zoneId); counters != g_ZoneCounters.end()) { + zone.requests = counters->second.requests; + zone.acted = counters->second.acted; + zone.playersMoved = counters->second.playersMoved; + zone.failed = counters->second.failed; + } + for (const auto& [id, key] : g_Migrations) zone.pending += key.zone == zoneId ? 1 : 0; status.zones.push_back(std::move(zone)); } return status; @@ -282,6 +373,9 @@ void WorldReloader::HandleRequest(const WorldReloadRequest& request, const std:: return; } const auto zone = request.zoneId; + g_ZoneCounters[zone].requests++; + g_Dirty = true; + if (g_Held.erase(zone)) LOG("World reload of zone %u: its automatic reloads are no longer held", zone); const auto choices = Choose(Views(), [zone](const InstanceMigration::InstanceView& view) { return view.zoneId == zone; }, KeepZones()); if (choices.empty()) { LOG("World reload of zone %u asked for by %s: no instance of it runs", zone, who.c_str()); @@ -296,6 +390,13 @@ void WorldReloader::HandleRequest(const WorldReloadRequest& request, const std:: Apply(choices, std::min(request.warnSeconds, InstanceMigrationRequest::MAX_WARN_SECONDS), request.requesterId, who); } +void WorldReloader::SetHold(uint32_t zoneId, bool hold) { + if (zoneId == 0) return; + if (hold ? g_Held.insert(zoneId).second : g_Held.erase(zoneId) > 0) { + LOG("World reload: zone %u's instances are %s when its files change", zoneId, hold ? "left alone" : "replaced again"); + } +} + void WorldReloader::OnMigrationStatus(const MigrationStatus& status) { const auto it = g_Migrations.find(status.migrationId); if (it == g_Migrations.end()) return; @@ -306,6 +407,8 @@ void WorldReloader::OnMigrationStatus(const MigrationStatus& status) { const auto key = it->second; g_Migrations.erase(it); std::string message = "Instance " + std::to_string(key.instance); + if (status.state == InstanceMigration::eState::DONE) g_ZoneCounters[key.zone].playersMoved += status.moved; + else g_ZoneCounters[key.zone].failed++; if (status.state == InstanceMigration::eState::DONE) { message += " replaced by " + std::to_string(status.targetInstance) + " (" + std::to_string(status.moved) + " player(s) moved)"; } else { @@ -319,7 +422,10 @@ void WorldReloader::OnInstanceGone(const Instance& instance) { const InstanceKey key{ instance.GetMapID(), instance.GetInstanceID() }; g_Tracker.Forget(key.zone, key.instance); g_Tried.erase(key); - if (g_Tracker.Zones().count(key.zone) == 0) g_ZoneMessages.erase(key.zone); + if (g_Tracker.Zones().count(key.zone) == 0) { + g_ZoneMessages.erase(key.zone); + g_ZoneCounters.erase(key.zone); + } g_Dirty = true; } @@ -347,8 +453,8 @@ void WorldReloader::Update() { if (seconds > 0) { // Worlds that reported after the last change are caught here too ReloadStale(); - auto due = g_Tracker.Poll([](const std::string& path) { return FdbSnapshot::StampOf(path); }); - if (!due.empty()) g_Job = std::async(std::launch::async, HashFiles, std::move(due)); + auto due = g_Tracker.Poll(StampNow, [](const std::string& path) { return MapsName(path).has_value(); }); + if (!due.empty()) g_Job = std::async(std::launch::async, HashFiles, Targets(std::move(due))); } // Player counts change without telling us; the status is compared before it is sent g_Dirty = true; diff --git a/dMasterServer/WorldReloader.h b/dMasterServer/WorldReloader.h index 9c59201c8..297d09f0d 100644 --- a/dMasterServer/WorldReloader.h +++ b/dMasterServer/WorldReloader.h @@ -14,7 +14,8 @@ struct WorldReloadRequest; /** * Master's side of world hot reload (docs/WorldHotReload.md). Worlds report the zone files they loaded (WORLD_FILES); - * master watches them (size and mtime every world_watch_seconds, then a hash on a worker thread) and, when one changes + * master watches them (size and mtime every world_watch_seconds, then a hash on a worker thread; for a file under + * maps/ that is the file in the override folder, zone_override_dir, when one is there) and, when one changes * or a GM or the dashboard asks (WORLD_RELOAD), replaces the affected instances with new ones on the files on disk, * moving their players with the instance migration (MigrationCoordinator). Main thread only. */ @@ -28,6 +29,9 @@ namespace WorldReloader { // A GM's /reloadworld or the dashboard; who is for the log void HandleRequest(const WorldReloadRequest& request, const std::string& who); + // The dashboard holds (or stops holding) a zone's automatic reloads; a request for the zone ends a hold + void SetHold(uint32_t zoneId, bool hold); + // Every migration's progress (MigrationCoordinator's observer) void OnMigrationStatus(const MigrationStatus& status); diff --git a/dNet/master/WorldFiles.h b/dNet/master/WorldFiles.h index 9c8abd629..e7db562d0 100644 --- a/dNet/master/WorldFiles.h +++ b/dNet/master/WorldFiles.h @@ -102,6 +102,30 @@ struct WorldReloadRequest : public LUBitStream { } }; +/** + * WORLD_RELOAD_HOLD (dashboard -> master): while hold is set, master does not replace zoneId's instances by itself + * when their files change (a set of edited files is in place and its reload waits for someone to say so). A + * WORLD_RELOAD of the zone still reloads it, and ends the hold. + */ +struct WorldReloadHold : public LUBitStream { + WorldReloadHold() : LUBitStream(ServiceType::MASTER, MessageType::Master::WORLD_RELOAD_HOLD) {} + + uint32_t zoneId{}; + bool hold{}; + + void Serialize(RakNet::BitStream& stream) const override { + stream.Write(zoneId); + stream.Write(hold ? 1 : 0); + } + + bool Deserialize(RakNet::BitStream& stream) override { + uint8_t value{}; + if (!stream.Read(zoneId) || !stream.Read(value)) return false; + hold = value != 0; + return true; + } +}; + /** * WORLD_FILES_STATUS (master -> dashboard): every zone that runs, the files its instances loaded as they are on disk * now, and which instances loaded an older version. Sent when something changes and when the dashboard connects. @@ -137,6 +161,12 @@ struct WorldFilesStatus : public LUBitStream { std::vector files; std::vector instances; std::string message; // the last reload of the zone + // Since master started (or the zone last had no instance), so a reader can tell what one request did + uint32_t requests{}; // reloads asked for (WORLD_RELOAD) + uint32_t acted{}; // instances replaced, stopped or left to empty + uint32_t playersMoved{}; + uint32_t failed{}; // instances that could not be replaced (message says why) + uint16_t pending{}; // instances being replaced right now bool operator==(const Zone& other) const = default; }; @@ -172,6 +202,11 @@ struct WorldFilesStatus : public LUBitStream { stream.Write((instance.stale ? 1 : 0) | (instance.reloading ? 2 : 0) | (instance.outdated ? 4 : 0)); } InstanceMigration::WriteText(stream, zone.message, WorldFiles::MAX_MESSAGE); + stream.Write(zone.requests); + stream.Write(zone.acted); + stream.Write(zone.playersMoved); + stream.Write(zone.failed); + stream.Write(zone.pending); } } @@ -205,6 +240,7 @@ struct WorldFilesStatus : public LUBitStream { instance.outdated = flags & 4; } if (!InstanceMigration::ReadText(stream, zone.message, WorldFiles::MAX_MESSAGE)) return false; + if (!stream.Read(zone.requests) || !stream.Read(zone.acted) || !stream.Read(zone.playersMoved) || !stream.Read(zone.failed) || !stream.Read(zone.pending)) return false; } return true; } diff --git a/tests/dCommonTests/MessageIdPinTests.cpp b/tests/dCommonTests/MessageIdPinTests.cpp index a52218b31..b68eec2a7 100644 --- a/tests/dCommonTests/MessageIdPinTests.cpp +++ b/tests/dCommonTests/MessageIdPinTests.cpp @@ -1766,7 +1766,7 @@ static_assert(static_cast(MessageType::Game::CAN_ITEMS_BE_REFORGED) == static_assert(static_cast(MessageType::Game::NOTIFY_CLIENT_RAIL_START_FAILED) == 1771); static_assert(static_cast(MessageType::Game::GET_IS_ON_RAIL) == 1772); -// MessageType::Master: 47 enumerators +// MessageType::Master: 48 enumerators static_assert(static_cast(MessageType::Master::REQUEST_ZONE_TRANSFER) == 1); static_assert(static_cast(MessageType::Master::REQUEST_ZONE_TRANSFER_RESPONSE) == 2); static_assert(static_cast(MessageType::Master::SERVER_INFO) == 3); @@ -1817,6 +1817,7 @@ static_assert(static_cast(MessageType::Master::WORLD_FILES) == 47); static_assert(static_cast(MessageType::Master::WORLD_RELOAD) == 48); static_assert(static_cast(MessageType::Master::WORLD_FILES_STATUS) == 49); static_assert(static_cast(MessageType::Master::UPDATE_STATUS) == 50); +static_assert(static_cast(MessageType::Master::WORLD_RELOAD_HOLD) == 51); // MessageType::Server: 3 enumerators static_assert(static_cast(MessageType::Server::VERSION_CONFIRM) == 0); diff --git a/tests/dCommonTests/WorldFileWatchTests.cpp b/tests/dCommonTests/WorldFileWatchTests.cpp index 01a303900..40b4e5b80 100644 --- a/tests/dCommonTests/WorldFileWatchTests.cpp +++ b/tests/dCommonTests/WorldFileWatchTests.cpp @@ -250,3 +250,40 @@ TEST(WorldFileWatchTests, ChooseKeepsAnInstanceOfKeptZones) { EXPECT_EQ(choices[1].action, eAction::STOP); } } + +// A file a world read from the client's packs is watched once the caller says a folder can replace it: an override +// showing up for it makes the instances that read the pack's version stale +TEST(WorldFileWatchTests, PackedFilesAreWatchedWhenAnOverrideCanReplaceThem) { + Tracker tracker; + tracker.Report(1100, 1, 0, { File("maps/a.lvl", 7, Kind::SCENE, true) }); + const auto inPack = At(0, 0); + Stamp now = inPack; + const auto poll = [&] { + return tracker.Poll([&](const std::string&) { return now; }, [](const std::string& path) { return path.starts_with("maps/"); }); + }; + // Not watched without the second argument + EXPECT_TRUE(tracker.Poll([&](const std::string&) { return now; }).empty()); + + // First read: what the pack holds + auto due = poll(); + ASSERT_EQ(due.size(), 1u); + EXPECT_FALSE(tracker.Hashed(due[0], 7, 100)); + EXPECT_TRUE(tracker.Stale().empty()); + EXPECT_TRUE(poll().empty()); + + // An override appears and holds still for a poll + now = At(50, 9); + EXPECT_TRUE(poll().empty()); + due = poll(); + ASSERT_EQ(due.size(), 1u); + EXPECT_TRUE(tracker.Hashed(due[0], 8, 50)); + EXPECT_EQ(tracker.Stale().size(), 1u); + + // It is taken away again: the pack's version is what is read, and the instance matches it again + now = inPack; + EXPECT_TRUE(poll().empty()); + due = poll(); + ASSERT_EQ(due.size(), 1u); + EXPECT_TRUE(tracker.Hashed(due[0], 7, 100)); + EXPECT_TRUE(tracker.Stale().empty()); +}