feat(master): watch world zone files and replace instances that loaded old ones

Master hashes each reported file on a worker, polls size and mtime every
world_watch_seconds (a change must hold still for one poll), and replaces
stale instances with the instance migration (Mythran shift, or seamless
with world_reload_seamless=1); empty ones are stopped. Also on WORLD_RELOAD.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dMasterServer/CMakeLists.txt

@@ -1,5 +1,6 @@
 set(DMASTERSERVER_SOURCES
 	"CDClientReloader.cpp"
+	"WorldReloader.cpp"
 	"InstanceManager.cpp"
 	"MigrationCoordinator.cpp"
 	"LiveUpdateCoordinator.cpp"

dMasterServer/MasterServer.cpp

@@ -61,6 +61,8 @@
 #include "master/UgcModelsMade.h"
 #include "master/CDClientReload.h"
 #include "CDClientReloader.h"
+#include "WorldReloader.h"
+#include "master/WorldFiles.h"
 #include "BuildInfo.h"
 
 #ifdef DARKFLAME_PLATFORM_UNIX
@@ -629,6 +631,7 @@ int main(int argc, char** argv) {
 		}
 		LiveUpdateCoordinator::Update();
 		CDClientReloader::Update();
+		WorldReloader::Update();
 		CheckPlayerActionTimeouts();
 
 		// Spare instances for busy zones (zone_limits), checked every few seconds
@@ -702,6 +705,7 @@ int main(int argc, char** argv) {
 
 			if (instance->GetShutdownComplete()) {
 				MigrationCoordinator::OnInstanceGone(*instance);
+				WorldReloader::OnInstanceGone(*instance);
 				Game::im->RemoveInstance(instance);
 			}
 		}
@@ -794,6 +798,7 @@ namespace {
 		case ServiceType::DASHBOARD:
 			dashboardServerMasterPeerSysAddr = sysAddr;
 			g_DashboardConnects++;
+			WorldReloader::Republish();
 			// Its traffic isn't a player's; packet captures leave it out
 			PacketCapture::IgnorePeer(sysAddr);
 			break;
@@ -1143,6 +1148,25 @@ namespace {
 		else CDClientReloader::Request("a GM, character " + std::to_string(request.requesterId));
 	}
 
+	// World hot reload (docs/WorldHotReload.md): a world's zone files, and a GM's /reloadworld or the dashboard
+	void OnWorldFiles(const WorldFilesReport& report, const SystemAddress& sysAddr) {
+		WorldReloader::HandleReport(sysAddr, report);
+	}
+
+	void OnWorldReload(const WorldReloadRequest& request, const SystemAddress& sysAddr) {
+		const bool fromDashboard = sysAddr == dashboardServerMasterPeerSysAddr && sysAddr != UNASSIGNED_SYSTEM_ADDRESS;
+		if (!fromDashboard && !Game::im->GetInstanceBySysAddr(sysAddr)) {
+			LOG("Ignoring a world reload request from a server that is neither the dashboard nor a world");
+			return;
+		}
+		if (shutdownSequenceStarted) {
+			LOG("Shutdown sequence has been started. Not reloading zone %u.", request.zoneId);
+			return;
+		}
+		const std::string by = request.requestedBy.empty() ? (fromDashboard ? "the dashboard" : "a GM") : request.requestedBy;
+		WorldReloader::HandleRequest(request, fromDashboard ? "the dashboard, " + by : by);
+	}
+
 	void OnConfigReload(const ConfigReload& reload, const SystemAddress& sysAddr) {
 		if (sysAddr != dashboardServerMasterPeerSysAddr) {
 			LOG("Ignoring config reload from a server that is not the dashboard");
@@ -1247,6 +1271,8 @@ namespace {
 			handlers.On<DataChanged>(Master::DATA_CHANGED, ForwardWorldToDashboard<DataChanged>);
 			handlers.On<MessageCaptureControl>(Master::MESSAGE_CAPTURE_CONTROL, OnMessageCaptureControl);
 			handlers.On<CDClientReload>(Master::CDCLIENT_RELOAD, OnCDClientReload);
+			handlers.On<WorldFilesReport>(Master::WORLD_FILES, OnWorldFiles);
+			handlers.On<WorldReloadRequest>(Master::WORLD_RELOAD, OnWorldReload);
 			handlers.On<MessageCaptureData>(Master::MESSAGE_CAPTURE_DATA, OnMessageCaptureData);
 			handlers.On<RequestServerList>(Master::REQUEST_SERVER_LIST, OnRequestServerList);
 			handlers.On<ServerTraffic>(Master::SERVER_TRAFFIC, OnServerTraffic);
@@ -1281,6 +1307,7 @@ void HandlePacket(Packet* packet) {
 			}
 
 			MigrationCoordinator::OnInstanceGone(*instance);
+			WorldReloader::OnInstanceGone(*instance);
 			Game::im->RemoveInstance(instance);
 		}
 
@@ -1343,6 +1370,7 @@ void HandlePacket(Packet* packet) {
 int ShutdownSequence(int32_t signal) {
 	if (!Game::logger) return -1;
 	CDClientReloader::Shutdown();
+	WorldReloader::Shutdown();
 	LOG("Recieved Signal %d", signal);
 	if (shutdownSequenceStarted) {
 		LOG("Duplicate Shutdown Sequence");
@@ -1532,7 +1560,13 @@ void InitializeLiveUpdates() {
 		}
 	};
 	LiveUpdateCoordinator::Initialize(std::move(hooks));
-	MigrationCoordinator::SetObserver(LiveUpdateCoordinator::OnMigrationStatus);
+	MigrationCoordinator::SetObserver([](const MigrationStatus& status) {
+		LiveUpdateCoordinator::OnMigrationStatus(status);
+		WorldReloader::OnMigrationStatus(status);
+	});
+	WorldReloader::SetPublisher([](const WorldFilesStatus& status) {
+		if (dashboardServerMasterPeerSysAddr != UNASSIGNED_SYSTEM_ADDRESS) MasterPackets::SendTo(dashboardServerMasterPeerSysAddr, status);
+	});
 
 	// The worlds; the UGC and dashboard servers read the current files when they start (docs/CDClientFdb.md)
 	CDClientReloader::SetBroadcast([](const CDClientReload& reload) {

dMasterServer/WorldFileWatch.h

@@ -0,0 +1,273 @@
+#ifndef __WORLDFILEWATCH__H__
+#define __WORLDFILEWATCH__H__
+
+#include <compare>
+#include <cstdint>
+#include <functional>
+#include <map>
+#include <optional>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "FdbSnapshot.h"
+#include "ZoneFileLog.h"
+#include "master/InstanceMigration.h"
+
+/**
+ * World hot reload without master's state (docs/WorldHotReload.md): which zone files the running instances loaded,
+ * which of those changed on disk, and which instances to replace. Header only, so it is unit tested without a master.
+ *
+ * Master reads (hashes) each watched file itself: once when a world first reports it, and again whenever its size or
+ * mtime changed and then held still for one poll (FdbSnapshot::Watcher). An instance is stale when a file it loaded
+ * has another hash on disk now. Files a world read from the client's packs are listed but not watched.
+ */
+namespace WorldFileWatch {
+	using Stamp = FdbSnapshot::Stamp;
+
+	struct InstanceKey {
+		uint32_t zone{};
+		uint32_t instance{};
+
+		auto operator<=>(const InstanceKey&) const = default;
+	};
+
+	struct FileState {
+		ZoneFileLog::eKind kind{};
+		bool packed{};
+		uint64_t reportedSize{};            // what the first world to report it read
+		uint64_t reportedHash{};
+		std::optional<uint64_t> hash;       // master's read of the file on disk
+		uint64_t size{};
+		bool missing{};                     // the last poll or read found no file
+		bool hashing{};
+		FdbSnapshot::Watcher watcher;
+	};
+
+	struct Loaded {
+		uint32_t clone{};
+		std::map<std::string, uint64_t> files; // path -> hash the world read
+	};
+
+	// A file to hash now, with the stamp it had when the poll saw it (accepted once the hash is in)
+	struct HashJob {
+		std::string path;
+		Stamp stamp;
+	};
+
+	class Tracker {
+	public:
+		// A world reported the files it loaded (replaces what that instance reported before)
+		void Report(uint32_t zone, uint32_t instance, uint32_t clone, const std::vector<ZoneFileLog::Entry>& files) {
+			auto& loaded = m_Instances[{ zone, instance }];
+			loaded = {};
+			loaded.clone = clone;
+			for (const auto& file : files) {
+				loaded.files[file.path] = file.hash;
+				auto [it, added] = m_Files.try_emplace(file.path);
+				if (!added) continue;
+				it->second.kind = file.kind;
+				it->second.packed = file.packed;
+				it->second.reportedSize = file.size;
+				it->second.reportedHash = file.hash;
+			}
+		}
+
+		// An instance stopped; files no instance loaded any more stop being watched
+		void Forget(uint32_t zone, uint32_t instance) {
+			m_Instances.erase({ zone, instance });
+			std::erase_if(m_Files, [this](const auto& entry) { return !entry.second.hashing && !IsLoaded(entry.first); });
+		}
+
+		/**
+		 * Looks at every watched file (stampOf: its size and mtime now). Returns the ones to hash: files master hasn't
+		 * read yet, and files whose size or mtime changed and held still since the last poll. Those are marked as
+		 * being hashed until Hashed is called for them.
+		 */
+		std::vector<HashJob> Poll(const std::function<Stamp(const std::string&)>& stampOf) {
+			std::vector<HashJob> due;
+			for (auto& [path, file] : m_Files) {
+				if (file.packed || file.hashing) continue;
+				const auto stamp = stampOf(path);
+				file.missing = !stamp.exists;
+				const bool first = !file.hash && stamp.exists;
+				if (!file.watcher.Poll(stamp) && !first) continue;
+				file.hashing = true;
+				due.push_back({ path, stamp });
+			}
+			return due;
+		}
+
+		/**
+		 * A hash finished (hash nullopt: the file couldn't be read). Returns true when the file on disk is another
+		 * version than master read before.
+		 */
+		bool Hashed(const HashJob& job, std::optional<uint64_t> hash, uint64_t size) {
+			const auto it = m_Files.find(job.path);
+			if (it == m_Files.end()) return false;
+			auto& file = it->second;
+			file.hashing = false;
+			// Taken either way, so an unreadable file isn't read again every poll; the next change to it is
+			file.watcher.Accept(job.stamp);
+			if (!hash) {
+				file.missing = true;
+				if (!IsLoaded(job.path)) m_Files.erase(it);
+				return false;
+			}
+			file.missing = false;
+			const bool changed = file.hash && *file.hash != *hash;
+			file.hash = hash;
+			file.size = size;
+			if (!IsLoaded(job.path)) m_Files.erase(it);
+			return changed;
+		}
+
+		// A file this instance loaded is another version on disk now
+		bool IsStale(const InstanceKey& key) const {
+			const auto it = m_Instances.find(key);
+			if (it == m_Instances.end()) return false;
+			for (const auto& [path, loadedHash] : it->second.files) {
+				const auto file = m_Files.find(path);
+				if (file != m_Files.end() && file->second.hash && *file->second.hash != loadedHash) return true;
+			}
+			return false;
+		}
+
+		std::vector<InstanceKey> Stale() const {
+			std::vector<InstanceKey> stale;
+			for (const auto& [key, loaded] : m_Instances) {
+				if (IsStale(key)) stale.push_back(key);
+			}
+			return stale;
+		}
+
+		// The versions on disk of the files this instance loaded; an automatic reload is tried once per signature
+		uint64_t Signature(const InstanceKey& key) const {
+			uint64_t signature = 14695981039346656037ULL;
+			const auto it = m_Instances.find(key);
+			if (it == m_Instances.end()) return signature;
+			for (const auto& [path, loadedHash] : it->second.files) {
+				const auto file = m_Files.find(path);
+				const uint64_t onDisk = file != m_Files.end() && file->second.hash ? *file->second.hash : loadedHash;
+				signature = FdbSnapshot::Hash(reinterpret_cast<const uint8_t*>(path.data()), path.size(), signature);
+				signature = FdbSnapshot::Hash(reinterpret_cast<const uint8_t*>(&onDisk), sizeof(onDisk), signature);
+			}
+			return signature;
+		}
+
+		// The files of a stale instance that changed (for the log)
+		std::vector<std::string> ChangedFiles(const InstanceKey& key) const {
+			std::vector<std::string> changed;
+			const auto it = m_Instances.find(key);
+			if (it == m_Instances.end()) return changed;
+			for (const auto& [path, loadedHash] : it->second.files) {
+				const auto file = m_Files.find(path);
+				if (file != m_Files.end() && file->second.hash && *file->second.hash != loadedHash) changed.push_back(path);
+			}
+			return changed;
+		}
+
+		// Any instance of zone loaded another version of path than is on disk
+		bool IsChanged(uint32_t zone, const std::string& path) const {
+			const auto file = m_Files.find(path);
+			if (file == m_Files.end() || !file->second.hash) return false;
+			for (const auto& [key, loaded] : m_Instances) {
+				if (key.zone != zone) continue;
+				const auto it = loaded.files.find(path);
+				if (it != loaded.files.end() && it->second != *file->second.hash) return true;
+			}
+			return false;
+		}
+
+		// Every file some instance of zone loaded
+		std::set<std::string> FilesOf(uint32_t zone) const {
+			std::set<std::string> files;
+			for (const auto& [key, loaded] : m_Instances) {
+				if (key.zone != zone) continue;
+				for (const auto& [path, hash] : loaded.files) files.insert(path);
+			}
+			return files;
+		}
+
+		std::set<uint32_t> Zones() const {
+			std::set<uint32_t> zones;
+			for (const auto& [key, loaded] : m_Instances) zones.insert(key.zone);
+			return zones;
+		}
+
+		bool Knows(const InstanceKey& key) const { return m_Instances.contains(key); }
+		const std::map<std::string, FileState>& Files() const { return m_Files; }
+
+	private:
+		bool IsLoaded(const std::string& path) const {
+			for (const auto& [key, loaded] : m_Instances) {
+				if (loaded.files.contains(path)) return true;
+			}
+			return false;
+		}
+
+		std::map<std::string, FileState> m_Files;
+		std::map<InstanceKey, Loaded> m_Instances;
+	};
+
+	enum class eAction : uint8_t {
+		REPLACE,          // start a new instance (same clone, same password), move its players there, stop it
+		START_THEN_STOP,  // nobody there, but the zone always has an instance: start a new one, stop this one
+		STOP,             // nobody there: stop it (a new instance starts when someone goes there)
+		SKIP,
+	};
+
+	inline const char* ActionName(eAction action) {
+		switch (action) {
+		case eAction::REPLACE: return "replace";
+		case eAction::START_THEN_STOP: return "start a new one, then stop it";
+		case eAction::STOP: return "stop";
+		case eAction::SKIP: return "skip";
+		}
+		return "skip";
+	}
+
+	struct Choice {
+		InstanceMigration::InstanceView view;
+		eAction action{ eAction::SKIP };
+		std::string reason;
+	};
+
+	/**
+	 * What to do with each instance wanted() picks: replace it when players are there, stop it when it is empty (a zone
+	 * in keepZones gets one new public instance first). Character selection loads no zone; instances still starting
+	 * load the files on disk now (and report them); instances shutting down or being emptied already are left alone.
+	 */
+	inline std::vector<Choice> Choose(const std::vector<InstanceMigration::InstanceView>& instances,
+		const std::function<bool(const InstanceMigration::InstanceView&)>& wanted, const std::set<uint32_t>& keepZones) {
+		std::vector<Choice> choices;
+		for (const auto& view : instances) {
+			if (!wanted(view)) continue;
+			Choice choice;
+			choice.view = view;
+			if (view.zoneId == 0) choice.reason = "Character selection loads no zone";
+			else if (view.shuttingDown) choice.reason = "Already shutting down";
+			else if (view.draining) choice.reason = "Its players are already being moved";
+			else if (!view.ready) choice.reason = "Still starting; it loads the files on disk now";
+			else if (view.players > 0) choice.action = eAction::REPLACE;
+			else choice.action = eAction::STOP;
+			choices.push_back(std::move(choice));
+		}
+		// A zone that always has an instance keeps one: a public instance replaced for its players, or else one new
+		// instance started for the first empty public one
+		std::set<uint32_t> keptZones;
+		for (const auto& choice : choices) {
+			if (choice.action == eAction::REPLACE && !choice.view.isPrivate && choice.view.cloneId == 0) keptZones.insert(choice.view.zoneId);
+		}
+		for (auto& choice : choices) {
+			const auto& view = choice.view;
+			if (choice.action != eAction::STOP || !keepZones.contains(view.zoneId) || view.isPrivate || view.cloneId != 0 || keptZones.contains(view.zoneId)) continue;
+			choice.action = eAction::START_THEN_STOP;
+			keptZones.insert(view.zoneId);
+		}
+		return choices;
+	}
+}
+
+#endif  //!__WORLDFILEWATCH__H__

dMasterServer/WorldReloader.cpp

@@ -0,0 +1,349 @@
+#include "WorldReloader.h"
+
+#include <cctype>
+#include <chrono>
+#include <future>
+#include <map>
+#include <set>
+
+#include "Game.h"
+#include "GeneralUtils.h"
+#include "InstanceManager.h"
+#include "Logger.h"
+#include "MigrationCoordinator.h"
+#include "WorldFileWatch.h"
+#include "dConfig.h"
+#include "master/InstanceMigration.h"
+#include "master/WorldFiles.h"
+
+using namespace WorldFileWatch;
+
+namespace {
+	using Clock = std::chrono::steady_clock;
+	// World reload migrations get IDs of their own, far from the worlds' (time-based) and the live update's ones
+	constexpr uint32_t MIGRATION_ID_BASE = 0x80000000;
+	constexpr auto PUBLISH_INTERVAL = std::chrono::seconds(1);
+
+	struct HashResult {
+		HashJob job;
+		std::optional<uint64_t> hash;
+		uint64_t size{};
+	};
+
+	Tracker g_Tracker;
+	std::future<std::vector<HashResult>> g_Job;
+	Clock::time_point g_NextPoll{};
+	Clock::time_point g_NextPublish{};
+	bool g_Dirty = true;
+	uint32_t g_NextMigrationId = 0;
+	std::map<InstanceKey, uint64_t> g_Tried;           // automatic reloads: the signature last tried per instance
+	std::map<uint32_t, InstanceKey> g_Migrations;       // our migrations -> the instance being replaced
+	std::map<uint32_t, std::string> g_ZoneMessages;     // the last reload of each zone, for the dashboard
+	std::function<void(const WorldFilesStatus&)> g_Publisher;
+	WorldFilesStatus g_Published;
+
+	uint32_t WatchSeconds() {
+		return GeneralUtils::TryParse<uint32_t>(Game::config->GetValue("world_watch_seconds")).value_or(5);
+	}
+
+	bool Seamless() {
+		return Game::config->GetValue("world_reload_seamless") == "1";
+	}
+
+	std::set<uint32_t> KeepZones() {
+		std::set<uint32_t> zones;
+		for (auto part : GeneralUtils::SplitString(Game::config->GetValue("prestart_worlds", "0,1000"), ',')) {
+			std::erase_if(part, [](const char c) { return std::isspace(static_cast<unsigned char>(c)); });
+			if (const auto zone = GeneralUtils::TryParse<uint32_t>(part)) zones.insert(*zone);
+		}
+		return zones;
+	}
+
+	std::string FileName(const std::string& path) {
+		const auto slash = path.find_last_of("/\\");
+		return slash == std::string::npos ? path : path.substr(slash + 1);
+	}
+
+	// The worker gets copies of the paths; it only reads the files and never logs or touches shared state
+	std::vector<HashResult> HashFiles(std::vector<HashJob> jobs) {
+		std::vector<HashResult> results;
+		results.reserve(jobs.size());
+		for (auto& job : jobs) {
+			HashResult result;
+			std::error_code code;
+			const auto size = std::filesystem::file_size(job.path, code);
+			if (!code) {
+				result.size = static_cast<uint64_t>(size);
+				result.hash = FdbSnapshot::HashFile(job.path);
+			}
+			result.job = std::move(job);
+			results.push_back(std::move(result));
+		}
+		return results;
+	}
+
+	std::vector<InstanceMigration::InstanceView> Views() {
+		std::vector<InstanceMigration::InstanceView> views;
+		for (const auto& instance : Game::im->GetInstances()) {
+			if (instance) views.push_back(instance->View());
+		}
+		return views;
+	}
+
+	void SetZoneMessage(uint32_t zone, const std::string& message) {
+		g_ZoneMessages[zone] = message;
+		g_Dirty = true;
+	}
+
+	/**
+	 * Replaces what Choose picked. warnSeconds: how long players are warned; requesterId: the GM told how each move
+	 * goes. Returns how many instances are being replaced or stopped.
+	 */
+	uint32_t Apply(const std::vector<Choice>& choices, uint16_t warnSeconds, LWOOBJID requesterId, const std::string& by) {
+		uint32_t acted = 0;
+		std::map<uint32_t, std::pair<uint32_t, uint32_t>> perZone; // zone -> (acted, skipped)
+		for (const auto& choice : choices) {
+			const auto& view = choice.view;
+			auto& counts = perZone[view.zoneId];
+			switch (choice.action) {
+			case eAction::SKIP:
+				LOG("World reload (%s): zone %u instance %u left alone: %s", by.c_str(), view.zoneId, view.instanceId, choice.reason.c_str());
+				counts.second++;
+				continue;
+			case eAction::STOP:
+			case eAction::START_THEN_STOP: {
+				const auto& instance = Game::im->FindInstanceWithPrivate(static_cast<LWOMAPID>(view.zoneId), static_cast<LWOINSTANCEID>(view.instanceId));
+				if (!instance) continue;
+				if (choice.action == eAction::START_THEN_STOP) {
+					// Started first: the stopped one takes nobody new meanwhile
+					instance->SetIsDraining(true);
+					const auto& started = Game::im->StartNewInstance(static_cast<LWOMAPID>(view.zoneId), 0);
+					if (started) LOG("World reload (%s): started instance %u of zone %u", by.c_str(), started->GetInstanceID(), view.zoneId);
+				}
+				// Look it up again: starting one grew the instance list
+				const auto& stopping = Game::im->FindInstanceWithPrivate(static_cast<LWOMAPID>(view.zoneId), static_cast<LWOINSTANCEID>(view.instanceId));
+				if (!stopping) continue;
+				LOG("World reload (%s): zone %u instance %u is empty; stopping it", by.c_str(), view.zoneId, view.instanceId);
+				stopping->Shutdown();
+				counts.first++;
+				acted++;
+				continue;
+			}
+			case eAction::REPLACE: {
+				InstanceMigrationRequest request;
+				request.requestId = MIGRATION_ID_BASE | (++g_NextMigrationId & 0x3FFFFFFF);
+				request.kind = InstanceMigration::eKind::REPLACE;
+				request.zoneId = view.zoneId;
+				request.sourceInstance = view.instanceId;
+				request.warnSeconds = warnSeconds;
+				request.shutdownSource = true;
+				request.seamless = Seamless();
+				request.requesterId = requesterId;
+				request.requestedBy = ("world reload (" + by + ")").substr(0, InstanceMigrationRequest::MAX_BY);
+				MigrationCoordinator::Options options;
+				// Properties, private instances and activity zones are moved too; a property is saved and frozen first
+				options.liveUpdate = true;
+				options.prepare = view.cloneId != 0;
+				g_Migrations[request.requestId] = { view.zoneId, view.instanceId };
+				const auto refusal = MigrationCoordinator::Start(request, options);
+				if (refusal != InstanceMigration::eRefusal::NONE) {
+					g_Migrations.erase(request.requestId);
+					LOG("World reload (%s): zone %u instance %u could not be replaced: %s", by.c_str(), view.zoneId, view.instanceId, InstanceMigration::Describe(refusal));
+					counts.second++;
+					continue;
+				}
+				counts.first++;
+				acted++;
+				continue;
+			}
+			}
+		}
+		for (const auto& [zone, counts] : perZone) {
+			std::string message = "Reload (" + by + "): " + std::to_string(counts.first) + " instance(s) replaced or stopped";
+			if (counts.second) message += ", " + std::to_string(counts.second) + " left alone";
+			SetZoneMessage(zone, message);
+		}
+		return acted;
+	}
+
+	// Instances that loaded a file that changed since, each tried once per version of its files
+	void ReloadStale() {
+		std::set<InstanceKey> wanted;
+		for (const auto& key : g_Tracker.Stale()) {
+			const auto tried = g_Tried.find(key);
+			if (tried != g_Tried.end() && tried->second == g_Tracker.Signature(key)) continue;
+			wanted.insert(key);
+		}
+		if (wanted.empty()) return;
+		auto choices = Choose(Views(), [&wanted](const InstanceMigration::InstanceView& view) {
+			return wanted.contains({ view.zoneId, view.instanceId });
+		}, KeepZones());
+		// Instances still starting (or already moving) are tried again at the next poll
+		std::erase_if(choices, [](const Choice& choice) { return choice.action == eAction::SKIP; });
+		if (choices.empty()) return;
+		for (const auto& choice : choices) {
+			const InstanceKey key{ choice.view.zoneId, choice.view.instanceId };
+			g_Tried[key] = g_Tracker.Signature(key);
+			std::string files;
+			for (const auto& path : g_Tracker.ChangedFiles(key)) files += (files.empty() ? "" : ", ") + FileName(path);
+			LOG("World reload: zone %u instance %u loaded files that changed on disk (%s): %s", choice.view.zoneId, choice.view.instanceId,
+				files.c_str(), ActionName(choice.action));
+		}
+		Apply(choices, WorldReloadRequest::DEFAULT_WARN_SECONDS, LWOOBJID_EMPTY, "files changed");
+	}
+
+	WorldFilesStatus BuildStatus() {
+		WorldFilesStatus status;
+		const auto seconds = WatchSeconds();
+		status.watching = seconds > 0;
+		status.watchSeconds = static_cast<uint16_t>(std::min<uint32_t>(seconds, UINT16_MAX));
+		status.seamless = Seamless();
+		for (const auto zoneId : g_Tracker.Zones()) {
+			WorldFilesStatus::Zone zone;
+			zone.zoneId = zoneId;
+			for (const auto& path : g_Tracker.FilesOf(zoneId)) {
+				const auto it = g_Tracker.Files().find(path);
+				if (it == g_Tracker.Files().end()) continue;
+				const auto& state = it->second;
+				WorldFilesStatus::File file;
+				file.disk.kind = state.kind;
+				file.disk.packed = state.packed;
+				file.disk.path = path;
+				file.hashed = state.hash.has_value();
+				file.disk.size = state.hash ? state.size : state.reportedSize;
+				file.disk.hash = state.hash ? *state.hash : state.reportedHash;
+				file.missing = state.missing;
+				file.changed = g_Tracker.IsChanged(zoneId, path);
+				zone.files.push_back(std::move(file));
+			}
+			for (const auto& instance : Game::im->GetInstances()) {
+				if (!instance || instance->GetMapID() != zoneId) continue;
+				WorldFilesStatus::Instance row;
+				row.instanceId = instance->GetInstanceID();
+				row.cloneId = instance->GetCloneID();
+				row.players = instance->GetCurrentClientCount();
+				row.stale = g_Tracker.IsStale({ zoneId, row.instanceId });
+				row.reloading = instance->GetIsShuttingDown() || instance->GetIsDraining();
+				zone.instances.push_back(row);
+			}
+			if (const auto message = g_ZoneMessages.find(zoneId); message != g_ZoneMessages.end()) zone.message = message->second;
+			status.zones.push_back(std::move(zone));
+		}
+		return status;
+	}
+
+	void Publish(bool force) {
+		if (!g_Publisher) return;
+		const auto now = Clock::now();
+		if (!force && (!g_Dirty || now < g_NextPublish)) return;
+		g_Dirty = false;
+		g_NextPublish = now + PUBLISH_INTERVAL;
+		auto status = BuildStatus();
+		const bool same = status.watching == g_Published.watching && status.watchSeconds == g_Published.watchSeconds &&
+			status.seamless == g_Published.seamless && status.zones == g_Published.zones;
+		if (same && !force) return;
+		g_Published = status;
+		g_Publisher(status);
+	}
+}
+
+void WorldReloader::SetPublisher(std::function<void(const WorldFilesStatus&)> publisher) {
+	g_Publisher = std::move(publisher);
+}
+
+void WorldReloader::HandleReport(const SystemAddress& from, const WorldFilesReport& report) {
+	const auto& instance = Game::im->GetInstanceBySysAddr(from);
+	if (!instance) {
+		LOG("Ignoring a world file list from a server that is not a world");
+		return;
+	}
+	uint32_t watched = 0;
+	for (const auto& file : report.files) watched += file.packed ? 0 : 1;
+	LOG("Zone %u instance %u loaded %zu zone file(s) (%u loose, watched)", instance->GetMapID(), instance->GetInstanceID(), report.files.size(), watched);
+	g_Tracker.Report(instance->GetMapID(), instance->GetInstanceID(), instance->GetCloneID(), report.files);
+	g_Dirty = true;
+}
+
+void WorldReloader::HandleRequest(const WorldReloadRequest& request, const std::string& who) {
+	if (request.zoneId == 0) {
+		LOG("World reload asked for by %s without a zone; nothing to do", who.c_str());
+		return;
+	}
+	const auto zone = request.zoneId;
+	const auto choices = Choose(Views(), [zone](const InstanceMigration::InstanceView& view) { return view.zoneId == zone; }, KeepZones());
+	if (choices.empty()) {
+		LOG("World reload of zone %u asked for by %s: no instance of it runs", zone, who.c_str());
+		SetZoneMessage(zone, "Reload (" + who + "): no instance was running");
+		return;
+	}
+	LOG("World reload of zone %u asked for by %s: %zu instance(s)", zone, who.c_str(), choices.size());
+	for (const auto& choice : choices) {
+		// A replaced instance is not tried again automatically for the files on disk now
+		g_Tried[{ choice.view.zoneId, choice.view.instanceId }] = g_Tracker.Signature({ choice.view.zoneId, choice.view.instanceId });
+	}
+	Apply(choices, std::min(request.warnSeconds, InstanceMigrationRequest::MAX_WARN_SECONDS), request.requesterId, who);
+}
+
+void WorldReloader::OnMigrationStatus(const MigrationStatus& status) {
+	const auto it = g_Migrations.find(status.migrationId);
+	if (it == g_Migrations.end()) return;
+	if (!status.Finished()) {
+		g_Dirty = true;
+		return;
+	}
+	const auto key = it->second;
+	g_Migrations.erase(it);
+	std::string message = "Instance " + std::to_string(key.instance);
+	if (status.state == InstanceMigration::eState::DONE) {
+		message += " replaced by " + std::to_string(status.targetInstance) + " (" + std::to_string(status.moved) + " player(s) moved)";
+	} else {
+		message += " was not replaced: " + (status.message.empty() ? std::string("the move failed") : status.message);
+		// Tried again once its files change again, or when asked
+	}
+	SetZoneMessage(key.zone, message);
+}
+
+void WorldReloader::OnInstanceGone(const Instance& instance) {
+	const InstanceKey key{ instance.GetMapID(), instance.GetInstanceID() };
+	g_Tracker.Forget(key.zone, key.instance);
+	g_Tried.erase(key);
+	if (g_Tracker.Zones().count(key.zone) == 0) g_ZoneMessages.erase(key.zone);
+	g_Dirty = true;
+}
+
+void WorldReloader::Republish() {
+	g_Dirty = true;
+	Publish(true);
+}
+
+void WorldReloader::Update() {
+	if (g_Job.valid()) {
+		if (g_Job.wait_for(std::chrono::seconds(0)) != std::future_status::ready) return Publish(false);
+		for (const auto& result : g_Job.get()) {
+			const bool changed = g_Tracker.Hashed(result.job, result.hash, result.size);
+			if (changed) LOG("World reload: %s changed on disk", result.job.path.c_str());
+			else if (!result.hash) LOG("World reload: could not read %s", result.job.path.c_str());
+		}
+		g_Dirty = true;
+		ReloadStale();
+	}
+
+	const auto now = Clock::now();
+	if (now >= g_NextPoll) {
+		const auto seconds = WatchSeconds();
+		g_NextPoll = now + std::chrono::seconds(seconds > 0 ? seconds : 60);
+		if (seconds > 0) {
+			// Worlds that reported after the last change are caught here too
+			ReloadStale();
+			auto due = g_Tracker.Poll([](const std::string& path) { return FdbSnapshot::StampOf(path); });
+			if (!due.empty()) g_Job = std::async(std::launch::async, HashFiles, std::move(due));
+		}
+		// Player counts change without telling us; the status is compared before it is sent
+		g_Dirty = true;
+	}
+	Publish(false);
+}
+
+void WorldReloader::Shutdown() {
+	if (g_Job.valid()) g_Job.wait();
+}

dMasterServer/WorldReloader.h

@@ -0,0 +1,47 @@
+#ifndef WORLDRELOADER_H
+#define WORLDRELOADER_H
+
+#include <functional>
+#include <string>
+
+#include "RakNetTypes.h"
+
+class Instance;
+struct MigrationStatus;
+struct WorldFilesReport;
+struct WorldFilesStatus;
+struct WorldReloadRequest;
+
+/**
+ * Master's side of world hot reload (docs/WorldHotReload.md). Worlds report the zone files they loaded (WORLD_FILES);
+ * master watches them (size and mtime every world_watch_seconds, then a hash on a worker thread) and, when one changes
+ * or a GM or the dashboard asks (WORLD_RELOAD), replaces the affected instances with new ones on the files on disk,
+ * moving their players with the instance migration (MigrationCoordinator). Main thread only.
+ */
+namespace WorldReloader {
+	// Where the dashboard's status goes (WORLD_FILES_STATUS)
+	void SetPublisher(std::function<void(const WorldFilesStatus&)> publisher);
+
+	// A world reported its files
+	void HandleReport(const SystemAddress& from, const WorldFilesReport& report);
+
+	// A GM's /reloadworld or the dashboard; who is for the log
+	void HandleRequest(const WorldReloadRequest& request, const std::string& who);
+
+	// Every migration's progress (MigrationCoordinator's observer)
+	void OnMigrationStatus(const MigrationStatus& status);
+
+	// A world server went away
+	void OnInstanceGone(const Instance& instance);
+
+	// The dashboard connected: send it the status again
+	void Republish();
+
+	// Every master frame: polls the files, finishes hashes and reloads what changed
+	void Update();
+
+	// Waits for a running hash (at shutdown)
+	void Shutdown();
+};
+
+#endif // WORLDRELOADER_H