feat: live updates move every server onto a new build without a restart

With new binaries in place, master moves everything onto them while the
server keeps running (the dashboard's Live update, /liveupdate or SIGUSR2 to
master):

- database migrations of the new build first; a failure stops there
- UGC finishes the jobs it is running (queued rows stay pending), auth
  restarts, chat hands its teams to master for the next chat server; master
  starts the new processes and retries ones that don't come back
- once the new chat server is up (CHAT_SERVER_READY) every world connects at
  once and sends its players again (LoginSessionNotify resync, no login logged)
- every world instance is replaced with an instance migration: public worlds
  and private ones (same password) at once, properties after the old instance
  saved and froze the property (MIGRATE_PREPARE: no building, claiming or
  saving there any more), activity zones and character select once their
  players left or after a wait; empty instances just stop, zones in
  prestart_worlds get a new one first
- the dashboard restarts last and picks the status up again

Players land where they stood (position carried in CarriedPlayerState, also on
properties and Moon Base). Draining instances get no new players
(InstanceMigration::AcceptsNewPlayers) and show as "Moving players" in the
world list. The order lives in LiveUpdateMachine.h without master state and is
unit tested; master's glue is LiveUpdateCoordinator. Master itself is not
replaced. Message IDs are appended only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dMasterServer/CMakeLists.txt

@@ -1,6 +1,7 @@
 set(DMASTERSERVER_SOURCES
 	"InstanceManager.cpp"
 	"MigrationCoordinator.cpp"
+	"LiveUpdateCoordinator.cpp"
 	"Start.cpp"
 )
 

dMasterServer/InstanceManager.cpp

@@ -240,7 +240,7 @@ const InstancePtr& InstanceManager::GetInstanceBySysAddr(const SystemAddress& sy
 
 const InstancePtr& InstanceManager::FindInstance(LWOMAPID mapID, bool isFriendTransfer, LWOCLONEID cloneId) {
 	for (const auto& i : m_Instances) {
-		if (i && i->GetMapID() == mapID && i->GetCloneID() == cloneId && !i->IsFull(isFriendTransfer) && !i->GetIsPrivate() && !i->GetShutdownComplete() && !i->GetIsShuttingDown() && !i->GetIsDraining()) {
+		if (i && InstanceMigration::AcceptsNewPlayers(i->View(), mapID, cloneId, isFriendTransfer)) {
 			return i;
 		}
 	}
@@ -304,7 +304,8 @@ const InstancePtr& InstanceManager::FindPrivateInstance(const std::string& passw
 	for (const auto& instance : m_Instances) {
 		if (!instance) continue;
 
-		if (!instance->GetIsPrivate()) {
+		// A private instance being replaced (live update) takes nobody new: its replacement has the same password
+		if (!instance->GetIsPrivate() || instance->GetIsDraining() || instance->GetIsShuttingDown() || instance->GetShutdownComplete()) {
 			continue;
 		}
 

dMasterServer/InstanceManager.h

@@ -11,6 +11,7 @@
 #include "Stamps.h"
 #include "dZMCommon.h"
 #include "Logger.h"
+#include "master/InstanceMigration.h"
 
 struct Player {
 	LWOOBJID id;
@@ -98,6 +99,23 @@ public:
 
 	bool IsFull(bool isFriendTransfer) const;
 
+	// What instance migrations and live updates plan with
+	InstanceMigration::InstanceView View() const {
+		InstanceMigration::InstanceView view;
+		view.zoneId = GetMapID();
+		view.instanceId = GetInstanceID();
+		view.cloneId = GetCloneID();
+		view.players = GetCurrentClientCount();
+		view.softCap = GetSoftCap();
+		view.hardCap = GetHardCap();
+		view.reserved = GetReserved();
+		view.ready = GetIsReady();
+		view.isPrivate = GetIsPrivate();
+		view.shuttingDown = GetIsShuttingDown() || GetShutdownComplete();
+		view.draining = GetIsDraining();
+		return view;
+	}
+
 private:
 	std::string m_IP{};
 	uint32_t m_Port{};

dMasterServer/LiveUpdateCoordinator.cpp

@@ -0,0 +1,315 @@
+#include "LiveUpdateCoordinator.h"
+
+#include <cctype>
+#include <ctime>
+#include <filesystem>
+#include <map>
+
+#include "BinaryPathFinder.h"
+#include "CDActivitiesTable.h"
+#include "CDClientManager.h"
+#include "Database.h"
+#include "Game.h"
+#include "GeneralUtils.h"
+#include "InstanceManager.h"
+#include "Logger.h"
+#include "MigrationCoordinator.h"
+#include "MigrationRunner.h"
+#include "dConfig.h"
+
+using namespace LiveUpdate;
+
+namespace {
+	// How often the machine runs (master runs at 30 frames a second; nothing here needs that)
+	constexpr auto TICK_INTERVAL = std::chrono::milliseconds(250);
+	// Live update migrations get IDs of their own, far from the worlds' (time-based) ones
+	constexpr uint32_t MIGRATION_ID_BASE = 0xC0000000;
+
+	LiveUpdateCoordinator::Hooks g_Hooks;
+	Machine g_Machine;
+	LiveUpdateStatus g_Status; // the last one sent
+	uint32_t g_UpdateId = 0;
+	uint32_t g_NextMigrationId = 0;
+	Clock::time_point g_NextTick{};
+	std::map<size_t, std::pair<eUnitState, std::string>> g_Logged; // unit index -> what was logged last
+	ePhase g_LoggedPhase = ePhase::IDLE;
+
+	int64_t Now() {
+		return static_cast<int64_t>(std::time(nullptr));
+	}
+
+	template<typename T>
+	T Setting(const std::string& key, T fallback) {
+		return GeneralUtils::TryParse<T>(Game::config->GetValue(key)).value_or(fallback);
+	}
+
+	std::set<uint32_t> KeepZones() {
+		std::set<uint32_t> zones;
+		for (auto part : GeneralUtils::SplitString(Game::config->GetValue("prestart_worlds", "0,1000"), ',')) {
+			std::erase_if(part, [](const char c) { return std::isspace(static_cast<unsigned char>(c)); });
+			if (const auto zone = GeneralUtils::TryParse<uint32_t>(part)) zones.insert(*zone);
+		}
+		zones.insert(0);
+		return zones;
+	}
+
+	Settings ReadSettings(int32_t warnSeconds) {
+		Settings settings;
+		settings.warnSeconds = warnSeconds >= 0 ? static_cast<uint16_t>(warnSeconds) :
+			std::min<uint16_t>(Setting<uint16_t>("live_update_warn_seconds", 10), InstanceMigrationRequest::MAX_WARN_SECONDS);
+		settings.parallelWorlds = std::max<uint32_t>(Setting<uint32_t>("live_update_parallel_worlds", 4), 1);
+		settings.serviceTimeout = std::chrono::seconds(std::max<uint32_t>(Setting<uint32_t>("live_update_service_timeout", 30), 10));
+		settings.ugcDrainTimeout = std::chrono::seconds(Setting<uint32_t>("live_update_ugc_drain_timeout", 300));
+		settings.charSelectWait = std::chrono::seconds(Setting<uint32_t>("live_update_char_select_wait", 60));
+		settings.activityWait = std::chrono::seconds(Setting<uint32_t>("live_update_activity_wait", 1800));
+		settings.playerWaitSeconds = std::min<uint16_t>(Setting<uint16_t>("live_update_player_wait", 30), MigratePlayersOrder::MAX_MAX_WAIT_SECONDS);
+		settings.propertyBuildWaitSeconds = std::min<uint16_t>(Setting<uint16_t>("live_update_property_build_wait", 60), MigratePrepare::MAX_WAIT_SECONDS);
+		settings.runMigrations = Game::config->GetValue("live_update_run_migrations") != "0";
+		settings.keepZones = KeepZones();
+		return settings;
+	}
+
+	// Races, minigames and other activities run in their own zones with state that only lives in that world
+	bool IsActivityZone(uint32_t zone) {
+		static std::map<uint32_t, bool> cache;
+		if (const auto it = cache.find(zone); it != cache.end()) return it->second;
+		auto* activities = CDClientManager::GetTable<CDActivitiesTable>();
+		const bool activity = activities && !activities->Query([zone](const CDActivities& row) { return row.instanceMapID == zone; }).empty();
+		return cache[zone] = activity;
+	}
+
+	Observed Observe() {
+		Observed observed;
+		observed.ugc = g_Hooks.service(eService::UGC);
+		observed.auth = g_Hooks.service(eService::AUTH);
+		observed.chat = g_Hooks.service(eService::CHAT);
+		observed.dashboard = g_Hooks.service(eService::DASHBOARD);
+		for (const auto& instance : Game::im->GetInstances()) {
+			if (!instance) continue;
+			observed.worlds.push_back({ instance->View(), IsActivityZone(instance->GetMapID()) });
+		}
+		return observed;
+	}
+
+	class Actions final : public IActions {
+	public:
+		bool RunMigrations(std::string& error) override {
+			LOG("Live update: running the new build's database migrations");
+			try {
+				MigrationRunner::RunMigrations();
+				MigrationRunner::RunSQLiteMigrations();
+				Database::Get()->Commit();
+			} catch (const std::exception& ex) {
+				error = std::string("The database migrations failed: ") + ex.what();
+				LOG("Live update: %s", error.c_str());
+				return false;
+			}
+			return true;
+		}
+
+		void RetireService(eService service) override {
+			LOG("Live update: retiring the %s", ServiceName(service));
+			g_Hooks.retire(service);
+		}
+
+		void StopService(eService service) override {
+			LOG("Live update: stopping the %s", ServiceName(service));
+			g_Hooks.stop(service);
+		}
+
+		void StartService(eService service) override {
+			LOG("Live update: starting the %s", ServiceName(service));
+			g_Hooks.start(service);
+		}
+
+		void ChatReady() override {
+			LOG("Live update: the new chat server is up; telling the worlds");
+			g_Hooks.chatReady();
+		}
+
+		std::optional<uint32_t> StartWorld(uint32_t zone) override {
+			const auto& started = Game::im->StartNewInstance(static_cast<LWOMAPID>(zone), 0);
+			if (!started) return std::nullopt;
+			LOG("Live update: started instance %u of zone %u", started->GetInstanceID(), zone);
+			return started->GetInstanceID();
+		}
+
+		void SetDraining(uint32_t zone, uint32_t instance, bool draining) override {
+			const auto& found = Game::im->FindInstanceWithPrivate(static_cast<LWOMAPID>(zone), static_cast<LWOINSTANCEID>(instance));
+			if (!found) return;
+			LOG("Live update: zone %u instance %u %s", zone, instance, draining ? "takes no new players" : "takes players again");
+			found->SetIsDraining(draining);
+		}
+
+		std::optional<uint32_t> Migrate(const Move& move, std::string& refusal) override {
+			InstanceMigrationRequest request;
+			request.requestId = MIGRATION_ID_BASE | (++g_NextMigrationId & 0x3FFFFFFF);
+			request.kind = move.kind;
+			request.zoneId = move.zone;
+			request.sourceInstance = move.instance;
+			request.targetInstance = move.target;
+			request.warnSeconds = move.warnSeconds;
+			request.shutdownSource = true;
+			request.requestedBy = "live update";
+			MigrationCoordinator::Options options;
+			options.liveUpdate = true;
+			options.prepare = move.prepare;
+			options.prepareWaitSeconds = move.prepareWaitSeconds;
+			options.playerWaitSeconds = move.playerWaitSeconds;
+			const auto result = MigrationCoordinator::Start(request, options);
+			if (result != InstanceMigration::eRefusal::NONE) {
+				refusal = InstanceMigration::Describe(result);
+				return std::nullopt;
+			}
+			return request.requestId;
+		}
+
+		void StopWorld(uint32_t zone, uint32_t instance) override {
+			const auto& found = Game::im->FindInstanceWithPrivate(static_cast<LWOMAPID>(zone), static_cast<LWOINSTANCEID>(instance));
+			if (found) found->Shutdown();
+		}
+	};
+
+	Actions g_Actions;
+
+	void LogChanges() {
+		const auto& units = g_Machine.Units();
+		for (size_t i = 0; i < units.size(); i++) {
+			const auto& unit = units[i];
+			auto& logged = g_Logged[i];
+			if (logged.first == unit.state && logged.second == unit.message) continue;
+			logged = { unit.state, unit.message };
+			if (unit.kind == eUnitKind::WORLD) {
+				LOG("Live update %u: zone %u clone %u instance %u%s: %s%s%s", g_UpdateId, unit.zone, unit.clone, unit.instance,
+					unit.replacement ? (" -> " + std::to_string(unit.replacement)).c_str() : "", StateName(unit.state),
+					unit.message.empty() ? "" : ": ", unit.message.c_str());
+			} else {
+				LOG("Live update %u: %s: %s%s%s", g_UpdateId, KindName(unit.kind), StateName(unit.state), unit.message.empty() ? "" : ": ", unit.message.c_str());
+			}
+		}
+		if (g_LoggedPhase != g_Machine.Phase()) {
+			g_LoggedPhase = g_Machine.Phase();
+			LOG("Live update %u: %s%s%s", g_UpdateId, PhaseName(g_LoggedPhase), g_Machine.Message().empty() ? "" : ": ", g_Machine.Message().c_str());
+		}
+	}
+
+	void Publish(bool phaseOrFinishChanged) {
+		g_Machine.FillStatus(g_Status);
+		g_Status.updateId = g_UpdateId;
+		if (IsFinished(g_Status.phase) && g_Status.finishedAt == 0) g_Status.finishedAt = Now();
+		LogChanges();
+		if (g_Hooks.publish) g_Hooks.publish(g_Status, phaseOrFinishChanged);
+	}
+
+	size_t FinishedUnits() {
+		return static_cast<size_t>(std::count_if(g_Machine.Units().begin(), g_Machine.Units().end(), [](const Unit& unit) { return IsFinished(unit.state); }));
+	}
+
+	// The binaries the update starts; a build still being written would fail to start
+	bool CheckBinaries(std::string& error) {
+		const auto dir = BinaryPathFinder::GetBinaryDir();
+		std::vector<std::string> names{ "WorldServer", "AuthServer", "ChatServer" };
+		if (Game::config->GetValue("enable_dashboard") == "1") names.push_back("DashboardServer");
+		if (Game::config->GetValue("enable_ugc_server") == "1") names.push_back("UgcServer");
+		for (auto name : names) {
+#ifdef _WIN32
+			name += ".exe";
+#endif
+			std::error_code ec;
+			const auto path = dir / name;
+			if (!std::filesystem::is_regular_file(path, ec) || std::filesystem::file_size(path, ec) == 0) {
+				error = name + " is missing from " + dir.string() + " (is a build still running?)";
+				return false;
+			}
+		}
+		return true;
+	}
+}
+
+void LiveUpdateCoordinator::Initialize(Hooks hooks) {
+	g_Hooks = std::move(hooks);
+}
+
+bool LiveUpdateCoordinator::Start(const std::string& by, LWOOBJID requesterId, int32_t warnSeconds, std::string& error) {
+	if (g_Machine.IsRunning()) {
+		error = "A live update is already running";
+		return false;
+	}
+	if (!CheckBinaries(error)) return false;
+	const auto settings = ReadSettings(warnSeconds);
+	if (!g_Machine.Start(settings, Observe(), Clock::now(), error)) return false;
+	g_UpdateId++;
+	g_Logged.clear();
+	g_LoggedPhase = ePhase::IDLE;
+	g_Status = LiveUpdateStatus{};
+	g_Status.startedAt = Now();
+	g_Status.by = by;
+	g_Status.requesterId = requesterId;
+	g_NextTick = {};
+	LOG("Live update %u started by %s: %zu world instance(s), warn %u s, %u at a time", g_UpdateId, by.c_str(),
+		g_Machine.Units().size() - 5, settings.warnSeconds, settings.parallelWorlds);
+	Publish(true);
+	return true;
+}
+
+void LiveUpdateCoordinator::HandleRequest(const LiveUpdateRequest& request) {
+	switch (request.action) {
+	case eAction::START: {
+		std::string error;
+		const auto by = request.requestedBy.empty() ? std::string("unknown") : request.requestedBy;
+		if (!Start(by, request.requesterId, request.warnSeconds, error)) {
+			LOG("Live update requested by %s refused: %s", by.c_str(), error.c_str());
+			// The status tells whoever asked why (the running one's, or the last one's with the reason)
+			auto status = Status();
+			status.requesterId = request.requesterId;
+			status.message = error;
+			if (g_Hooks.publish) g_Hooks.publish(status, true);
+		}
+		return;
+	}
+	case eAction::CANCEL:
+		if (g_Machine.Cancel()) {
+			LOG("Live update %u cancelled by %s", g_UpdateId, request.requestedBy.c_str());
+			Publish(true);
+		}
+		return;
+	case eAction::STATUS: {
+		auto status = Status();
+		if (request.requesterId != LWOOBJID_EMPTY) status.requesterId = request.requesterId;
+		if (g_Hooks.publish) g_Hooks.publish(status, request.requesterId != LWOOBJID_EMPTY);
+		return;
+	}
+	}
+}
+
+void LiveUpdateCoordinator::OnMigrationStatus(const MigrationStatus& status) {
+	if (!g_Machine.IsRunning()) return;
+	g_Machine.OnMigration(status);
+}
+
+void LiveUpdateCoordinator::Update() {
+	if (!g_Machine.IsRunning()) return;
+	const auto now = Clock::now();
+	if (now < g_NextTick) return;
+	g_NextTick = now + TICK_INTERVAL;
+	const auto phaseBefore = g_Machine.Phase();
+	const auto finishedBefore = FinishedUnits();
+	if (g_Machine.Tick(Observe(), now, g_Actions)) Publish(phaseBefore != g_Machine.Phase() || finishedBefore != FinishedUnits());
+}
+
+void LiveUpdateCoordinator::Abort(const std::string& why) {
+	if (!g_Machine.IsRunning()) return;
+	g_Machine.Abort(why);
+	Publish(true);
+}
+
+bool LiveUpdateCoordinator::IsRunning() {
+	return g_Machine.IsRunning();
+}
+
+LiveUpdateStatus LiveUpdateCoordinator::Status() {
+	auto status = g_Status;
+	g_Machine.FillStatus(status);
+	status.updateId = g_UpdateId;
+	return status;
+}

dMasterServer/LiveUpdateCoordinator.h

@@ -0,0 +1,48 @@
+#ifndef __LIVEUPDATECOORDINATOR__H__
+#define __LIVEUPDATECOORDINATOR__H__
+
+#include <functional>
+#include <string>
+
+#include "LiveUpdateMachine.h"
+#include "master/LiveUpdate.h"
+
+/**
+ * Master's side of live updates (docs/LiveUpdate.md): runs LiveUpdate::Machine against the instance manager, the
+ * instance migrations and the other servers. Started from the dashboard (LIVE_UPDATE_REQUEST), a GM's /liveupdate or
+ * SIGUSR2 to master. Main thread only.
+ */
+namespace LiveUpdateCoordinator {
+	// What MasterServer.cpp provides: the other servers, and where statuses go
+	struct Hooks {
+		std::function<LiveUpdate::ServiceView(LiveUpdate::eService)> service;
+		std::function<void(LiveUpdate::eService)> retire;  // LIVE_UPDATE_RETIRE (chat, UGC) or SHUTDOWN (auth, dashboard)
+		std::function<void(LiveUpdate::eService)> stop;    // SHUTDOWN
+		std::function<void(LiveUpdate::eService)> start;   // start its process
+		std::function<void()> chatReady;                   // CHAT_SERVER_READY to every world
+		std::function<void(const LiveUpdateStatus&, bool toWorlds)> publish;
+	};
+
+	void Initialize(Hooks hooks);
+
+	// Start one; false (and why) when it can't
+	bool Start(const std::string& by, LWOOBJID requesterId, int32_t warnSeconds, std::string& error);
+
+	// LIVE_UPDATE_REQUEST (the sender is checked by master: the dashboard, or a world for a GM)
+	void HandleRequest(const LiveUpdateRequest& request);
+
+	// Every instance migration's progress (MigrationCoordinator's observer)
+	void OnMigrationStatus(const MigrationStatus& status);
+
+	// Every master frame
+	void Update();
+
+	// Master is shutting down
+	void Abort(const std::string& why);
+
+	bool IsRunning();
+
+	LiveUpdateStatus Status();
+}
+
+#endif  //!__LIVEUPDATECOORDINATOR__H__

dMasterServer/LiveUpdateMachine.h

@@ -0,0 +1,580 @@
+#ifndef __LIVEUPDATEMACHINE__H__
+#define __LIVEUPDATEMACHINE__H__
+
+#include <algorithm>
+#include <chrono>
+#include <cstdint>
+#include <optional>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "master/InstanceMigration.h"
+#include "master/LiveUpdate.h"
+
+/**
+ * The order of a live update (docs/LiveUpdate.md), without master's state: it sees the servers and instances master
+ * knows (Observed), asks for things to be done through IActions, and hears how instance migrations go (OnMigration).
+ * Master's LiveUpdateCoordinator runs it every frame. Header only, so it is unit tested without a master server.
+ *
+ * Order: database migrations; then the UGC server, auth and chat together; once chat is back, the world instances
+ * (a few at a time); the dashboard last, so it shows the progress until then.
+ */
+namespace LiveUpdate {
+	using Clock = std::chrono::steady_clock;
+
+	enum class eService : uint8_t { UGC, AUTH, CHAT, DASHBOARD };
+
+	inline const char* ServiceName(eService service) {
+		switch (service) {
+		case eService::UGC: return "UGC server";
+		case eService::AUTH: return "auth server";
+		case eService::CHAT: return "chat server";
+		case eService::DASHBOARD: return "dashboard";
+		}
+		return "server";
+	}
+
+	struct Settings {
+		uint16_t warnSeconds{ 10 };                       // players are warned this long before they are moved
+		uint32_t parallelWorlds{ 4 };                     // world instances being replaced at once
+		std::chrono::seconds serviceTimeout{ 60 };        // a server to stop, or its new process to connect
+		std::chrono::seconds ugcDrainTimeout{ 300 };      // the UGC server finishing the jobs it is running
+		std::chrono::seconds charSelectWait{ 60 };        // players at character selection to pick a character
+		std::chrono::seconds activityWait{ 1800 };        // players in an activity zone (races, minigames) to finish
+		std::chrono::seconds worldStartTimeout{ 180 };    // a replacement instance to be ready
+		std::chrono::seconds worldTimeout{ 900 };         // backstop for one world's whole replacement
+		std::chrono::seconds chatSettle{ 3 };             // worlds to reconnect to the new chat server
+		uint16_t playerWaitSeconds{ 30 };                 // dead or building players before they are moved anyway
+		uint16_t propertyBuildWaitSeconds{ 60 };          // builders on a property before it is saved anyway
+		bool runMigrations{ true };
+		std::set<uint32_t> keepZones{ 0, 1000 };          // zones that always have an instance (prestart_worlds)
+		uint32_t maxServiceStarts{ 3 };                   // tries to start a server that didn't come back
+	};
+
+	struct ServiceView {
+		bool enabled{ true };
+		bool online{};
+		uint32_t connects{}; // counts every time one connected to master
+	};
+
+	struct WorldView {
+		InstanceMigration::InstanceView view;
+		bool activityZone{};
+	};
+
+	struct Observed {
+		ServiceView ugc, auth, chat, dashboard;
+		std::vector<WorldView> worlds;
+
+		const WorldView* Find(uint32_t zone, uint32_t instance) const {
+			for (const auto& world : worlds) {
+				if (world.view.zoneId == zone && world.view.instanceId == instance) return &world;
+			}
+			return nullptr;
+		}
+
+		const ServiceView& Service(eService service) const {
+			switch (service) {
+			case eService::UGC: return ugc;
+			case eService::AUTH: return auth;
+			case eService::CHAT: return chat;
+			case eService::DASHBOARD: return dashboard;
+			}
+			return auth;
+		}
+	};
+
+	// What the machine has master do
+	class IActions {
+	public:
+		virtual ~IActions() = default;
+		// The new build's database migrations; false (and why) when they failed
+		virtual bool RunMigrations(std::string& error) = 0;
+		// Chat and UGC: LIVE_UPDATE_RETIRE; auth and the dashboard: SHUTDOWN. Master starts the new one when it disconnects.
+		virtual void RetireService(eService service) = 0;
+		// SHUTDOWN right away (a UGC server that took too long to finish its jobs)
+		virtual void StopService(eService service) = 0;
+		virtual void StartService(eService service) = 0;
+		// CHAT_SERVER_READY to every world
+		virtual void ChatReady() = 0;
+		// Start a new public instance of zone (clone 0); its instance ID, or none when master refused
+		virtual std::optional<uint32_t> StartWorld(uint32_t zone) = 0;
+		virtual void SetDraining(uint32_t zone, uint32_t instance, bool draining) = 0;
+
+		struct Move {
+			uint32_t zone{};
+			uint32_t instance{};
+			InstanceMigration::eKind kind{ InstanceMigration::eKind::REPLACE };
+			uint32_t target{};         // merges only
+			uint16_t warnSeconds{};
+			uint16_t playerWaitSeconds{};
+			bool prepare{};            // properties: freeze and save before the new instance loads it
+			uint16_t prepareWaitSeconds{};
+		};
+		// An instance migration (MigrationCoordinator); its ID, or none (and why) when refused
+		virtual std::optional<uint32_t> Migrate(const Move& move, std::string& refusal) = 0;
+		virtual void StopWorld(uint32_t zone, uint32_t instance) = 0;
+	};
+
+	// What happens to one world instance, decided when its turn comes
+	enum class eWorldPlan : uint8_t {
+		STOP,           // nobody there: shut it down (a new one starts when someone goes there)
+		REPLACE_EMPTY,  // nobody there, but the zone should always have one: start the new one, then stop this one
+		MOVE,           // replace it and move its players (properties save and freeze first)
+		WAIT_THEN_MOVE, // activity zones and character selection: nobody new goes there; its players leave by
+		                // themselves or, after a while, are moved
+	};
+
+	inline const char* PlanName(eWorldPlan plan) {
+		switch (plan) {
+		case eWorldPlan::STOP: return "stop";
+		case eWorldPlan::REPLACE_EMPTY: return "replace_empty";
+		case eWorldPlan::MOVE: return "move";
+		case eWorldPlan::WAIT_THEN_MOVE: return "wait_then_move";
+		}
+		return "unknown";
+	}
+
+	// zoneReplaced: this update already started a new instance of the zone (keeping one is then taken care of)
+	inline eWorldPlan PlanWorld(const InstanceMigration::InstanceView& view, bool activityZone, bool keepZone, bool zoneReplaced) {
+		if (view.players <= 0) {
+			if ((keepZone || view.zoneId == 0) && !view.isPrivate && view.cloneId == 0 && !zoneReplaced) return eWorldPlan::REPLACE_EMPTY;
+			return eWorldPlan::STOP;
+		}
+		if (view.zoneId == 0 || activityZone) return eWorldPlan::WAIT_THEN_MOVE;
+		return eWorldPlan::MOVE;
+	}
+
+	struct Unit {
+		eUnitKind kind{};
+		eUnitState state{ eUnitState::PENDING };
+		eWorldPlan plan{ eWorldPlan::STOP };
+		uint32_t zone{};
+		uint32_t instance{};
+		uint32_t clone{};
+		bool isPrivate{};
+		uint32_t replacement{};
+		uint32_t players{};
+		uint16_t moved{};
+		uint32_t migrationId{};
+		bool drained{};         // we set it draining (so we undo that if it fails)
+		uint32_t connectsAtStart{};
+		uint32_t starts{};
+		Clock::time_point deadline{};
+		std::string message;
+	};
+
+	class Machine {
+	public:
+		/**
+		 * Plans an update of what runs now. False (and why) when one is already running. Instances that are shutting
+		 * down are left out; instances started later are the new build's already.
+		 */
+		bool Start(const Settings& settings, const Observed& observed, Clock::time_point now, std::string& error) {
+			if (m_Phase == ePhase::RUNNING || m_Phase == ePhase::CANCELLING) {
+				error = "A live update is already running";
+				return false;
+			}
+			m_Settings = settings;
+			m_Units.clear();
+			m_ReplacedZones.clear();
+			m_Phase = ePhase::RUNNING;
+			m_Message.clear();
+			m_Dirty = true;
+			m_Units.push_back(ServiceUnit(eUnitKind::DATABASE));
+			m_Units.push_back(ServiceUnit(eUnitKind::UGC));
+			m_Units.push_back(ServiceUnit(eUnitKind::AUTH));
+			m_Units.push_back(ServiceUnit(eUnitKind::CHAT));
+			// Character selection first (logins go to the new one soonest), then the busiest
+			std::vector<WorldView> worlds;
+			for (const auto& world : observed.worlds) {
+				if (!world.view.shuttingDown) worlds.push_back(world);
+			}
+			std::stable_sort(worlds.begin(), worlds.end(), [](const WorldView& a, const WorldView& b) {
+				if ((a.view.zoneId == 0) != (b.view.zoneId == 0)) return a.view.zoneId == 0;
+				return a.view.players > b.view.players;
+			});
+			for (const auto& world : worlds) {
+				Unit unit;
+				unit.kind = eUnitKind::WORLD;
+				unit.zone = world.view.zoneId;
+				unit.instance = world.view.instanceId;
+				unit.clone = world.view.cloneId;
+				unit.isPrivate = world.view.isPrivate;
+				unit.players = static_cast<uint32_t>(std::max(world.view.players, 0));
+				m_Units.push_back(unit);
+			}
+			m_Units.push_back(ServiceUnit(eUnitKind::DASHBOARD));
+			(void)now;
+			return true;
+		}
+
+		// Start nothing new; what is under way finishes. False when nothing is running.
+		bool Cancel() {
+			if (m_Phase != ePhase::RUNNING) return false;
+			m_Phase = ePhase::CANCELLING;
+			for (auto& unit : m_Units) {
+				if (unit.state == eUnitState::PENDING) SetState(unit, eUnitState::SKIPPED, "Cancelled");
+			}
+			m_Message = "Cancelled; waiting for what is under way";
+			m_Dirty = true;
+			return true;
+		}
+
+		// Master is shutting down: forget the rest
+		void Abort(const std::string& why) {
+			if (!IsRunning()) return;
+			for (auto& unit : m_Units) {
+				if (!IsFinished(unit.state)) SetState(unit, eUnitState::SKIPPED, why);
+			}
+			Finish(ePhase::CANCELLED, why);
+		}
+
+		/**
+		 * Moves things along. Master calls it every frame (with what it knows now); it returns whether anything changed
+		 * (the status is sent again then).
+		 */
+		bool Tick(const Observed& observed, Clock::time_point now, IActions& actions) {
+			if (!IsRunning()) return TakeDirty();
+
+			auto& database = m_Units[0];
+			if (database.state == eUnitState::PENDING) {
+				if (!m_Settings.runMigrations) {
+					SetState(database, eUnitState::SKIPPED, "Switched off (live_update_run_migrations)");
+				} else {
+					std::string error;
+					if (actions.RunMigrations(error)) {
+						SetState(database, eUnitState::STOPPED, "Up to date");
+					} else {
+						SetState(database, eUnitState::FAILED, error.empty() ? "The migrations failed" : error);
+						for (auto& unit : m_Units) {
+							if (unit.state == eUnitState::PENDING) SetState(unit, eUnitState::SKIPPED, "Not started: the database migrations failed");
+						}
+						Finish(ePhase::FAILED, "The database migrations failed; nothing else was touched");
+						return TakeDirty();
+					}
+				}
+			}
+
+			for (auto& unit : m_Units) {
+				switch (unit.kind) {
+				case eUnitKind::UGC: TickService(unit, eService::UGC, observed, now, actions); break;
+				case eUnitKind::AUTH: TickService(unit, eService::AUTH, observed, now, actions); break;
+				case eUnitKind::CHAT: TickService(unit, eService::CHAT, observed, now, actions); break;
+				default: break;
+				}
+			}
+
+			// Worlds once chat is back (their players are sent to the new chat server first)
+			if (IsFinished(Find(eUnitKind::CHAT).state)) {
+				for (auto& unit : m_Units) {
+					if (unit.kind != eUnitKind::WORLD || IsFinished(unit.state)) continue;
+					if (unit.state == eUnitState::PENDING) {
+						if (m_Phase != ePhase::RUNNING || Busy() >= std::max<uint32_t>(m_Settings.parallelWorlds, 1)) continue;
+						StartWorld(unit, observed, now, actions);
+					} else {
+						TickWorld(unit, observed, now, actions);
+					}
+				}
+			}
+
+			// The dashboard last: it shows the progress until then
+			auto& dashboard = Find(eUnitKind::DASHBOARD);
+			const bool othersDone = std::all_of(m_Units.begin(), m_Units.end(), [](const Unit& unit) {
+				return unit.kind == eUnitKind::DASHBOARD || IsFinished(unit.state);
+			});
+			if (othersDone && (dashboard.state != eUnitState::PENDING || m_Phase == ePhase::RUNNING)) {
+				TickService(dashboard, eService::DASHBOARD, observed, now, actions);
+			}
+
+			if (std::all_of(m_Units.begin(), m_Units.end(), [](const Unit& unit) { return IsFinished(unit.state); })) {
+				const auto failed = std::count_if(m_Units.begin(), m_Units.end(), [](const Unit& unit) { return unit.state == eUnitState::FAILED; });
+				if (m_Phase == ePhase::CANCELLING) Finish(ePhase::CANCELLED, "Cancelled");
+				else Finish(ePhase::DONE, failed == 0 ? "Everything runs on the new build" :
+					std::to_string(failed) + " part(s) failed and still run on the old build");
+			}
+			return TakeDirty();
+		}
+
+		// How an instance migration this update started is going
+		void OnMigration(const MigrationStatus& status) {
+			for (auto& unit : m_Units) {
+				if (unit.kind != eUnitKind::WORLD || unit.migrationId == 0 || unit.migrationId != status.migrationId || IsFinished(unit.state)) continue;
+				using InstanceMigration::eState;
+				if (status.targetInstance != 0) unit.replacement = status.targetInstance;
+				unit.moved = status.moved;
+				switch (status.state) {
+				case eState::PREPARING: SetState(unit, eUnitState::PREPARING, "Saving the property"); break;
+				case eState::PREPARED: SetState(unit, eUnitState::STARTING, "Property saved"); break;
+				case eState::STARTING_TARGET: SetState(unit, eUnitState::STARTING, "Starting instance " + std::to_string(unit.replacement)); break;
+				case eState::WARNING: SetState(unit, eUnitState::READY, "Instance " + std::to_string(unit.replacement) + " is ready; players were warned"); break;
+				case eState::MOVING: SetState(unit, eUnitState::DRAINING, std::to_string(status.moved) + " moved, " + std::to_string(status.remaining) + " left"); break;
+				case eState::DONE:
+					unit.migrationId = 0;
+					SetState(unit, eUnitState::STOPPING, std::to_string(status.moved) + " player(s) moved" + (status.message.empty() ? "" : "; " + status.message));
+					unit.deadline = Clock::time_point::max(); // TickWorld sets the stop deadline
+					break;
+				case eState::FAILED:
+					unit.migrationId = 0;
+					SetState(unit, eUnitState::FAILED, status.message.empty() ? "The move failed" : status.message);
+					break;
+				}
+				m_Dirty = true;
+			}
+		}
+
+		bool IsRunning() const { return m_Phase == ePhase::RUNNING || m_Phase == ePhase::CANCELLING; }
+		ePhase Phase() const { return m_Phase; }
+		const std::vector<Unit>& Units() const { return m_Units; }
+		const std::string& Message() const { return m_Message; }
+		const Settings& GetSettings() const { return m_Settings; }
+
+		// Units taking a slot of parallelWorlds: worlds being prepared, started or emptied
+		uint32_t Busy() const {
+			uint32_t busy = 0;
+			for (const auto& unit : m_Units) {
+				if (unit.kind != eUnitKind::WORLD) continue;
+				if (unit.state == eUnitState::PREPARING || unit.state == eUnitState::STARTING || unit.state == eUnitState::READY ||
+					unit.state == eUnitState::DRAINING) busy++;
+			}
+			return busy;
+		}
+
+		void FillStatus(LiveUpdateStatus& status) const {
+			status.phase = m_Phase;
+			status.message = m_Message;
+			status.units.clear();
+			for (const auto& unit : m_Units) {
+				LiveUpdateStatus::Unit row;
+				row.kind = unit.kind;
+				row.state = unit.state;
+				row.zoneId = unit.zone;
+				row.instanceId = unit.instance;
+				row.cloneId = unit.clone;
+				row.replacement = unit.replacement;
+				row.players = unit.players;
+				row.moved = unit.moved;
+				row.isPrivate = unit.isPrivate ? 1 : 0;
+				row.message = unit.message;
+				status.units.push_back(std::move(row));
+			}
+		}
+
+	private:
+		static Unit ServiceUnit(eUnitKind kind) {
+			Unit unit;
+			unit.kind = kind;
+			return unit;
+		}
+
+		Unit& Find(eUnitKind kind) {
+			for (auto& unit : m_Units) {
+				if (unit.kind == kind) return unit;
+			}
+			return m_Units.front();
+		}
+
+		void SetState(Unit& unit, eUnitState state, const std::string& message) {
+			if (unit.state != state || unit.message != message) m_Dirty = true;
+			unit.state = state;
+			unit.message = message;
+		}
+
+		void Finish(ePhase phase, const std::string& message) {
+			m_Phase = phase;
+			m_Message = message;
+			m_Dirty = true;
+		}
+
+		bool TakeDirty() {
+			const bool dirty = m_Dirty;
+			m_Dirty = false;
+			return dirty;
+		}
+
+		// A server: asked to stop (the UGC server first finishes what it is making); master starts the new one when the
+		// old one disconnects; done once a new one connected
+		void TickService(Unit& unit, eService service, const Observed& observed, Clock::time_point now, IActions& actions) {
+			const auto& view = observed.Service(service);
+			const bool replaced = view.connects > unit.connectsAtStart && view.online;
+			switch (unit.state) {
+			case eUnitState::PENDING:
+				if (m_Phase != ePhase::RUNNING) return SetState(unit, eUnitState::SKIPPED, "Cancelled");
+				if (!view.enabled) return SetState(unit, eUnitState::SKIPPED, "Not enabled");
+				if (!view.online) return SetState(unit, eUnitState::SKIPPED, "Not running (master starts it from the new build when it comes back)");
+				unit.connectsAtStart = view.connects;
+				actions.RetireService(service);
+				if (service == eService::UGC) {
+					unit.deadline = now + m_Settings.ugcDrainTimeout;
+					return SetState(unit, eUnitState::DRAINING, "Finishing the jobs it is running");
+				}
+				unit.deadline = now + m_Settings.serviceTimeout;
+				return SetState(unit, eUnitState::STOPPING, service == eService::CHAT ? "Handing its teams over" : "Stopping");
+			case eUnitState::DRAINING:
+			case eUnitState::STOPPING:
+				if (replaced) return Replaced(unit, service, now, actions);
+				if (!view.online) {
+					unit.deadline = now + m_Settings.serviceTimeout;
+					return SetState(unit, eUnitState::STARTING, "Starting the new " + std::string(ServiceName(service)));
+				}
+				if (now < unit.deadline) return;
+				if (unit.state == eUnitState::DRAINING) {
+					actions.StopService(service);
+					unit.deadline = now + m_Settings.serviceTimeout;
+					return SetState(unit, eUnitState::STOPPING, "Took too long; stopped (the jobs it was making are made again)");
+				}
+				return SetState(unit, eUnitState::FAILED, "It didn't stop; the old one keeps running");
+			case eUnitState::STARTING:
+				if (replaced) return Replaced(unit, service, now, actions);
+				if (now < unit.deadline) return;
+				if (unit.starts >= m_Settings.maxServiceStarts) {
+					return SetState(unit, eUnitState::FAILED, "The new " + std::string(ServiceName(service)) + " didn't come up");
+				}
+				unit.starts++;
+				actions.StartService(service);
+				unit.deadline = now + m_Settings.serviceTimeout;
+				return SetState(unit, eUnitState::STARTING, "Starting the new " + std::string(ServiceName(service)) + " again (try " + std::to_string(unit.starts) + ")");
+			case eUnitState::READY:
+				// Chat: worlds had a moment to reconnect and send their players
+				if (now >= unit.deadline) SetState(unit, eUnitState::STOPPED, "Replaced; worlds sent it their players");
+				return;
+			default:
+				return;
+			}
+		}
+
+		void Replaced(Unit& unit, eService service, Clock::time_point now, IActions& actions) {
+			if (service != eService::CHAT) return SetState(unit, eUnitState::STOPPED, "Replaced");
+			actions.ChatReady();
+			unit.deadline = now + m_Settings.chatSettle;
+			SetState(unit, eUnitState::READY, "New chat server is up; worlds are reconnecting");
+		}
+
+		IActions::Move MoveFor(const Unit& unit, InstanceMigration::eKind kind, uint32_t target) const {
+			IActions::Move move;
+			move.zone = unit.zone;
+			move.instance = unit.instance;
+			move.kind = kind;
+			move.target = target;
+			move.warnSeconds = m_Settings.warnSeconds;
+			move.playerWaitSeconds = m_Settings.playerWaitSeconds;
+			move.prepare = unit.clone != 0;
+			move.prepareWaitSeconds = m_Settings.propertyBuildWaitSeconds;
+			return move;
+		}
+
+		void StartMigration(Unit& unit, InstanceMigration::eKind kind, uint32_t target, Clock::time_point now, IActions& actions) {
+			std::string refusal;
+			const auto id = actions.Migrate(MoveFor(unit, kind, target), refusal);
+			if (!id) {
+				if (unit.drained) actions.SetDraining(unit.zone, unit.instance, false);
+				return SetState(unit, eUnitState::FAILED, refusal.empty() ? "Master refused to move its players" : refusal);
+			}
+			unit.migrationId = *id;
+			unit.deadline = now + m_Settings.worldTimeout;
+			if (unit.clone != 0) SetState(unit, eUnitState::PREPARING, "Saving the property");
+			else SetState(unit, eUnitState::STARTING, "Starting its replacement");
+		}
+
+		void StopOld(Unit& unit, Clock::time_point now, IActions& actions, const std::string& message) {
+			if (!unit.drained) {
+				actions.SetDraining(unit.zone, unit.instance, true);
+				unit.drained = true;
+			}
+			actions.StopWorld(unit.zone, unit.instance);
+			unit.deadline = now + m_Settings.serviceTimeout;
+			SetState(unit, eUnitState::STOPPING, message);
+		}
+
+		void StartWorld(Unit& unit, const Observed& observed, Clock::time_point now, IActions& actions) {
+			const auto* world = observed.Find(unit.zone, unit.instance);
+			if (!world || world->view.shuttingDown) return SetState(unit, eUnitState::STOPPED, "It stopped by itself");
+			unit.players = static_cast<uint32_t>(std::max(world->view.players, 0));
+			unit.plan = PlanWorld(world->view, world->activityZone, m_Settings.keepZones.contains(unit.zone), m_ReplacedZones.contains(unit.zone));
+			switch (unit.plan) {
+			case eWorldPlan::STOP:
+				return StopOld(unit, now, actions, "Nobody there; stopping it");
+			case eWorldPlan::REPLACE_EMPTY: {
+				const auto started = actions.StartWorld(unit.zone);
+				if (!started) return SetState(unit, eUnitState::FAILED, "Master refused to start a new instance");
+				unit.replacement = *started;
+				m_ReplacedZones.insert(unit.zone);
+				unit.deadline = now + m_Settings.worldStartTimeout;
+				return SetState(unit, eUnitState::STARTING, "Starting instance " + std::to_string(unit.replacement));
+			}
+			case eWorldPlan::MOVE:
+				return StartMigration(unit, InstanceMigration::eKind::REPLACE, 0, now, actions);
+			case eWorldPlan::WAIT_THEN_MOVE: {
+				// Character selection: logins go to a new one at once
+				if (unit.zone == 0 && !m_ReplacedZones.contains(0)) {
+					if (const auto started = actions.StartWorld(0)) {
+						unit.replacement = *started;
+						m_ReplacedZones.insert(0);
+					}
+				}
+				actions.SetDraining(unit.zone, unit.instance, true);
+				unit.drained = true;
+				unit.deadline = now + (unit.zone == 0 ? m_Settings.charSelectWait : m_Settings.activityWait);
+				return SetState(unit, eUnitState::WAITING, unit.zone == 0 ? "Waiting for players to pick a character" : "Waiting for the activity to end");
+			}
+			}
+		}
+
+		void TickWorld(Unit& unit, const Observed& observed, Clock::time_point now, IActions& actions) {
+			const auto* world = observed.Find(unit.zone, unit.instance);
+			switch (unit.state) {
+			case eUnitState::STOPPING:
+				if (!world) return SetState(unit, eUnitState::STOPPED, unit.message.empty() ? "Stopped" : unit.message);
+				if (unit.deadline == Clock::time_point::max()) unit.deadline = now + m_Settings.worldStartTimeout;
+				if (now >= unit.deadline) SetState(unit, eUnitState::FAILED, "It didn't stop");
+				return;
+			case eUnitState::WAITING: {
+				if (!world) return SetState(unit, eUnitState::STOPPED, "Everyone left; it stopped");
+				if (world->view.players <= 0) return StopOld(unit, now, actions, "Everyone left; stopping it");
+				if (now < unit.deadline) return;
+				// Whoever is still there is moved: from character selection into the new one, from an activity into a
+				// fresh instance (the activity is lost)
+				const auto* replacement = unit.replacement ? observed.Find(unit.zone, unit.replacement) : nullptr;
+				if (replacement && replacement->view.ready) return StartMigration(unit, InstanceMigration::eKind::MERGE, unit.replacement, now, actions);
+				return StartMigration(unit, InstanceMigration::eKind::REPLACE, 0, now, actions);
+			}
+			case eUnitState::STARTING:
+			case eUnitState::READY:
+				if (unit.plan == eWorldPlan::REPLACE_EMPTY) {
+					const auto* replacement = observed.Find(unit.zone, unit.replacement);
+					if (!world) return SetState(unit, eUnitState::STOPPED, "It stopped by itself");
+					if (!replacement) return SetState(unit, eUnitState::FAILED, "Instance " + std::to_string(unit.replacement) + " stopped while starting");
+					if (!replacement->view.ready) {
+						if (now >= unit.deadline) SetState(unit, eUnitState::FAILED, "Instance " + std::to_string(unit.replacement) + " didn't start in time");
+						return;
+					}
+					// Someone may have arrived before it was ready: they are moved over
+					if (world->view.players > 0) {
+						actions.SetDraining(unit.zone, unit.instance, true);
+						unit.drained = true;
+						return StartMigration(unit, InstanceMigration::eKind::MERGE, unit.replacement, now, actions);
+					}
+					return StopOld(unit, now, actions, "Instance " + std::to_string(unit.replacement) + " took over; stopping it");
+				}
+				[[fallthrough]];
+			case eUnitState::PREPARING:
+			case eUnitState::DRAINING:
+				// The migration reports how it goes (OnMigration); this is only a backstop
+				if (now >= unit.deadline) SetState(unit, eUnitState::FAILED, "Timed out");
+				return;
+			default:
+				return;
+			}
+		}
+
+		Settings m_Settings;
+		ePhase m_Phase{ ePhase::IDLE };
+		std::vector<Unit> m_Units;
+		std::set<uint32_t> m_ReplacedZones;
+		std::string m_Message;
+		bool m_Dirty{};
+	};
+}
+
+#endif  //!__LIVEUPDATEMACHINE__H__

dMasterServer/MasterServer.cpp

@@ -5,6 +5,7 @@
 #include <ctime>
 #include <iostream>
 #include <map>
+#include <optional>
 #include <set>
 #include <string>
 #include <thread>
@@ -39,6 +40,7 @@
 #include "Game.h"
 #include "InstanceManager.h"
 #include "MigrationCoordinator.h"
+#include "LiveUpdateCoordinator.h"
 #include "MasterPackets.h"
 #include "FdbToSqlite.h"
 #include "BitStreamUtils.h"
@@ -51,6 +53,7 @@
 #include "master/DataChanged.h"
 #include "master/MessageCapture.h"
 #include "master/InstanceMigration.h"
+#include "master/LiveUpdate.h"
 #include "master/ServerTraffic.h"
 #include "master/UgcModelsMade.h"
 
@@ -88,6 +91,18 @@ SystemAddress ugcServerMasterPeerSysAddr;
 // The UGC server's process id from its last start (0: not started), for the dashboard
 uint32_t ugcServerPid = 0;
 
+namespace {
+	// How many times each server connected (a live update waits for the new process to connect)
+	uint32_t g_AuthConnects = 0;
+	uint32_t g_ChatConnects = 0;
+	uint32_t g_UgcConnects = 0;
+	uint32_t g_DashboardConnects = 0;
+	// The teams a chat server retired by a live update handed over, for the next chat server that connects
+	std::optional<ChatHandoff> g_ChatHandoff;
+	// SIGUSR2 starts a live update (docs/LiveUpdate.md)
+	volatile std::sig_atomic_t g_LiveUpdateSignal = 0;
+}
+
 namespace {
 	// Every server for the dashboard: auth, chat, the UGC server and the worlds, including those launched but not
 	// connected yet (starting) and those shutting down
@@ -110,7 +125,8 @@ namespace {
 			entry.ip = LUString(inst->GetIP());
 			entry.port = inst->GetPort();
 			entry.isPrivate = inst->GetIsPrivate() ? 1 : 0;
-			entry.state = inst->GetIsShuttingDown() ? eState::STOPPING : inst->GetIsReady() ? eState::READY : eState::STARTING;
+			entry.state = inst->GetIsShuttingDown() ? eState::STOPPING : !inst->GetIsReady() ? eState::STARTING :
+				inst->GetIsDraining() ? eState::DRAINING : eState::READY;
 		}
 		return response;
 	}
@@ -157,6 +173,8 @@ namespace {
 	}
 }
 
+void InitializeLiveUpdates();
+
 int GenerateBCryptPassword(const std::string& password, const int workFactor, char salt[BCRYPT_HASHSIZE], char hash[BCRYPT_HASHSIZE]) {
 	int32_t bcryptState = ::bcrypt_gensalt(workFactor, salt);
 	assert(bcryptState == 0);
@@ -182,6 +200,10 @@ int main(int argc, char** argv) {
 	std::atexit([]() { ShutdownSequence(); });
 	std::signal(SIGINT, Game::OnSignal);
 	std::signal(SIGTERM, Game::OnSignal);
+#ifdef SIGUSR2
+	// kill -USR2 <master pid>: live update onto the binaries on disk now (docs/LiveUpdate.md)
+	std::signal(SIGUSR2, [](int) { g_LiveUpdateSignal = 1; });
+#endif
 
 	Game::config = new dConfig("masterconfig.ini");
 
@@ -472,6 +494,8 @@ int main(int argc, char** argv) {
 	});
 	Game::im->LoadZoneLimits();
 
+	InitializeLiveUpdates();
+
 	//Depending on the config, start up servers:
 	if (Game::config->GetValue("prestart_servers") != "0") {
 		StartChatServer();
@@ -525,6 +549,14 @@ int main(int argc, char** argv) {
 		}
 
 		MigrationCoordinator::Update();
+		if (g_LiveUpdateSignal) {
+			g_LiveUpdateSignal = 0;
+			std::string error;
+			if (!LiveUpdateCoordinator::Start("signal (SIGUSR2)", LWOOBJID_EMPTY, LiveUpdateRequest::DEFAULT_WARN, error)) {
+				LOG("Live update from SIGUSR2 refused: %s", error.c_str());
+			}
+		}
+		LiveUpdateCoordinator::Update();
 		CheckPlayerActionTimeouts();
 
 		// Spare instances for busy zones (zone_limits), checked every few seconds
@@ -673,15 +705,25 @@ namespace {
 			break;
 		case ServiceType::CHAT:
 			chatServerMasterPeerSysAddr = sysAddr;
+			g_ChatConnects++;
+			// A live update retired the last one: this one gets its teams
+			if (g_ChatHandoff) {
+				LOG("Handing %zu team(s) over to the new chat server", g_ChatHandoff->teams.size());
+				MasterPackets::SendTo(sysAddr, *g_ChatHandoff);
+				g_ChatHandoff.reset();
+			}
 			break;
 		case ServiceType::AUTH:
 			authServerMasterPeerSysAddr = sysAddr;
+			g_AuthConnects++;
 			break;
 		case ServiceType::DASHBOARD:
 			dashboardServerMasterPeerSysAddr = sysAddr;
+			g_DashboardConnects++;
 			break;
 		case ServiceType::UGC:
 			ugcServerMasterPeerSysAddr = sysAddr;
+			g_UgcConnects++;
 			break;
 		default:
 			break;
@@ -864,6 +906,30 @@ namespace {
 		MigrationCoordinator::Start(request);
 	}
 
+	// The dashboard, or a world for a GM's /liveupdate (the world checked the GM level)
+	void OnLiveUpdateRequest(const LiveUpdateRequest& request, const SystemAddress& sysAddr) {
+		const bool fromDashboard = sysAddr == dashboardServerMasterPeerSysAddr && sysAddr != UNASSIGNED_SYSTEM_ADDRESS;
+		if (!fromDashboard && !Game::im->GetInstanceBySysAddr(sysAddr)) {
+			LOG("Ignoring a live update request from a server that is neither the dashboard nor a world");
+			return;
+		}
+		if (shutdownSequenceStarted && request.action == LiveUpdate::eAction::START) {
+			LOG("Shutdown sequence has been started. Not starting a live update.");
+			return;
+		}
+		LiveUpdateCoordinator::HandleRequest(request);
+	}
+
+	// A chat server retired by a live update hands its teams over
+	void OnChatHandoff(const ChatHandoff& handoff, const SystemAddress& sysAddr) {
+		if (sysAddr != chatServerMasterPeerSysAddr) {
+			LOG("Ignoring a chat handoff from a server that is not the chat server");
+			return;
+		}
+		LOG("The chat server handed over %zu team(s)", handoff.teams.size());
+		g_ChatHandoff = handoff;
+	}
+
 	void OnPlayerAction(const PlayerActionRequest& request, const SystemAddress& sysAddr) {
 		// Only the dashboard may ask worlds to act on players
 		if (sysAddr != dashboardServerMasterPeerSysAddr) {
@@ -1028,6 +1094,8 @@ namespace {
 			handlers.On<RequestServerList>(Master::REQUEST_SERVER_LIST, OnRequestServerList);
 			handlers.On<ServerTraffic>(Master::SERVER_TRAFFIC, OnServerTraffic);
 			handlers.On<UgcModelsMade>(Master::UGC_MODELS_MADE, OnUgcModelsMade);
+			handlers.On<LiveUpdateRequest>(Master::LIVE_UPDATE_REQUEST, OnLiveUpdateRequest);
+			handlers.On<ChatHandoff>(Master::CHAT_HANDOFF, OnChatHandoff);
 			return handlers;
 		}();
 		return handlers;
@@ -1125,6 +1193,7 @@ int ShutdownSequence(int32_t signal) {
 		FinalizeShutdown(EXIT_FAILURE);
 	}
 
+	LiveUpdateCoordinator::Abort("Master is shutting down");
 	Game::im->SetIsShuttingDown(true);
 	shutdownSequenceStarted = true;
 	Game::lastSignal = -1;
@@ -1231,3 +1300,77 @@ int32_t FinalizeShutdown(int32_t signal) {
 	if (signal != EXIT_SUCCESS) exit(signal);
 	return signal;
 }
+
+// Live updates (docs/LiveUpdate.md): what the coordinator needs from master
+void InitializeLiveUpdates() {
+	using LiveUpdate::eService;
+	const auto peer = [](eService service) -> const SystemAddress& {
+		switch (service) {
+		case eService::UGC: return ugcServerMasterPeerSysAddr;
+		case eService::AUTH: return authServerMasterPeerSysAddr;
+		case eService::CHAT: return chatServerMasterPeerSysAddr;
+		case eService::DASHBOARD: return dashboardServerMasterPeerSysAddr;
+		}
+		return authServerMasterPeerSysAddr;
+	};
+
+	LiveUpdateCoordinator::Hooks hooks;
+	hooks.service = [peer](eService service) {
+		LiveUpdate::ServiceView view;
+		view.online = peer(service) != UNASSIGNED_SYSTEM_ADDRESS;
+		switch (service) {
+		case eService::UGC:
+			view.enabled = Game::config->GetValue("enable_ugc_server") == "1";
+			view.connects = g_UgcConnects;
+			break;
+		case eService::AUTH:
+			view.connects = g_AuthConnects;
+			break;
+		case eService::CHAT:
+			view.connects = g_ChatConnects;
+			break;
+		case eService::DASHBOARD:
+			view.enabled = Game::config->GetValue("enable_dashboard") == "1";
+			view.connects = g_DashboardConnects;
+			break;
+		}
+		return view;
+	};
+	// Chat hands its teams over and the UGC server finishes its jobs first; auth and the dashboard just stop. Either
+	// way master starts the new one when the old one disconnects (HandlePacket).
+	hooks.retire = [peer](eService service) {
+		const auto& address = peer(service);
+		if (address == UNASSIGNED_SYSTEM_ADDRESS) return;
+		if (service == eService::CHAT || service == eService::UGC) MasterPackets::SendTo(address, LiveUpdateRetire());
+		else MasterPackets::SendTo(address, MasterPackets::Shutdown());
+	};
+	hooks.stop = [peer](eService service) {
+		const auto& address = peer(service);
+		if (address != UNASSIGNED_SYSTEM_ADDRESS) MasterPackets::SendTo(address, MasterPackets::Shutdown());
+	};
+	hooks.start = [](eService service) {
+		switch (service) {
+		case eService::UGC: ugcServerPid = StartUgcServer(); break;
+		case eService::AUTH: StartAuthServer(); break;
+		case eService::CHAT: StartChatServer(); break;
+		case eService::DASHBOARD: StartDashboardServer(); break;
+		}
+	};
+	hooks.chatReady = [] {
+		for (const auto& instance : Game::im->GetInstances()) {
+			if (instance && instance->GetIsReady() && !instance->GetIsShuttingDown()) MasterPackets::SendTo(instance->GetSysAddr(), ChatServerReady());
+		}
+	};
+	hooks.publish = [](const LiveUpdateStatus& status, bool toWorlds) {
+		if (dashboardServerMasterPeerSysAddr != UNASSIGNED_SYSTEM_ADDRESS) MasterPackets::SendTo(dashboardServerMasterPeerSysAddr, status);
+		// The worlds' states (draining) changed with it
+		PushServerListToDashboard();
+		// The GM who asked in game hears how it goes, wherever they are by then
+		if (!toWorlds || status.requesterId == LWOOBJID_EMPTY) return;
+		for (const auto& instance : Game::im->GetInstances()) {
+			if (instance && instance->GetIsReady() && !instance->GetIsShuttingDown()) MasterPackets::SendTo(instance->GetSysAddr(), status);
+		}
+	};
+	LiveUpdateCoordinator::Initialize(std::move(hooks));
+	MigrationCoordinator::SetObserver(LiveUpdateCoordinator::OnMigrationStatus);
+}

dMasterServer/MigrationCoordinator.cpp

@@ -36,6 +36,10 @@ namespace {
 		bool shutdownSource{};
 		bool startedTarget{}; // we started the target for this migration
 		bool seamless{};
+		bool liveUpdate{};
+		bool prepare{};
+		uint16_t prepareWaitSeconds{};
+		uint16_t playerWaitSeconds{ MigratePlayersOrder::DEFAULT_MAX_WAIT_SECONDS };
 		LWOOBJID requester{};
 		int reserved{};
 		uint16_t moved{};
@@ -46,25 +50,14 @@ namespace {
 
 	std::map<uint32_t, Migration> g_Active;
 	std::function<void(const MigrationStatus&)> g_Reporter;
+	std::function<void(const MigrationStatus&)> g_Observer;
 
 	const InstancePtr& FindInstance(uint32_t zone, uint32_t instance) {
 		return Game::im->FindInstanceWithPrivate(static_cast<LWOMAPID>(zone), static_cast<LWOINSTANCEID>(instance));
 	}
 
 	InstanceView View(const Instance& instance) {
-		InstanceView view;
-		view.zoneId = instance.GetMapID();
-		view.instanceId = instance.GetInstanceID();
-		view.cloneId = instance.GetCloneID();
-		view.players = instance.GetCurrentClientCount();
-		view.softCap = instance.GetSoftCap();
-		view.hardCap = instance.GetHardCap();
-		view.reserved = instance.GetReserved();
-		view.ready = instance.GetIsReady();
-		view.isPrivate = instance.GetIsPrivate();
-		view.shuttingDown = instance.GetIsShuttingDown() || instance.GetShutdownComplete();
-		view.draining = instance.GetIsDraining();
-		return view;
+		return instance.View();
 	}
 
 	// Races, minigames and other activities run in their own zones with state that only lives in that world
@@ -84,7 +77,7 @@ namespace {
 	void Report(const Migration& migration, eState state, const std::string& message, uint16_t remaining = 0) {
 		LOG("Migration %u (%s zone %u instance %u -> %u): %s%s%s", migration.id, KindName(migration.kind), migration.zone, migration.source,
 			migration.target, StateName(state), message.empty() ? "" : ": ", message.c_str());
-		if (!g_Reporter) return;
+		if (!g_Reporter && !g_Observer) return;
 		MigrationStatus status;
 		status.migrationId = migration.id;
 		status.state = state;
@@ -97,7 +90,9 @@ namespace {
 		status.requesterId = migration.requester;
 		status.remaining = remaining;
 		status.message = message;
-		g_Reporter(status);
+		if (g_Observer) g_Observer(status);
+		// In game only the GM who asked hears about it
+		if (g_Reporter && status.requesterId != LWOOBJID_EMPTY) g_Reporter(status);
 	}
 
 	void ReleaseSeats(Migration& migration) {
@@ -126,6 +121,10 @@ namespace {
 		if (success && migration.shutdownSource && source) {
 			// Draining stays set: nothing is sent to it while it shuts down
 			source->Shutdown();
+		} else if (source && migration.prepare && migration.moved > 0) {
+			// A property some players already went to: the new instance owns it now, and the old one stays frozen
+			// (it is shut down once empty); nobody new goes to the old one
+			LOG("Migration %u: the property's old instance %u keeps draining (players already went to %u)", migration.id, migration.source, migration.target);
 		} else if (source) {
 			source->SetIsDraining(false);
 		}
@@ -150,7 +149,9 @@ namespace {
 		order.playersPerSecond = PLAYERS_PER_SECOND;
 		// Without a loading screen the "dimensional shift" notice would be the only sign; leave it out then
 		order.seamless = migration.seamless;
-		order.mythranShift = !migration.seamless;
+		// Character selection has no maintenance notice to show
+		order.mythranShift = !migration.seamless && source.GetMapID() != 0;
+		order.maxWaitSeconds = migration.playerWaitSeconds;
 		MasterPackets::SendTo(source.GetSysAddr(), order);
 
 		migration.state = migration.warnSeconds > 0 ? eState::WARNING : eState::MOVING;
@@ -164,7 +165,42 @@ void MigrationCoordinator::SetReporter(std::function<void(const MigrationStatus&
 	g_Reporter = std::move(reporter);
 }
 
-eRefusal MigrationCoordinator::Start(const InstanceMigrationRequest& request) {
+void MigrationCoordinator::SetObserver(std::function<void(const MigrationStatus&)> observer) {
+	g_Observer = std::move(observer);
+}
+
+namespace {
+	// The new instance (REPLACE): a fresh world server from the binary on disk now, so an update takes effect. A private
+	// instance's replacement gets the same password; the source is draining already, so the password finds the new one.
+	Instance* StartTarget(Instance& source) {
+		if (source.GetIsPrivate()) {
+			source.SetIsDraining(true);
+			const auto& started = Game::im->CreatePrivateInstance(source.GetMapID(), source.GetCloneID(), source.GetPassword());
+			return started ? started.get() : nullptr;
+		}
+		const auto& started = Game::im->StartNewInstance(source.GetMapID(), source.GetCloneID());
+		return started ? started.get() : nullptr;
+	}
+
+	// Seats held, the source draining, and the order sent once the target is ready
+	void Begin(Migration& migration, Instance& source, Instance& target) {
+		migration.target = target.GetInstanceID();
+		// Seats for everyone there now; a few may still arrive while draining, the hard cap is the real limit
+		migration.reserved = source.GetCurrentClientCount();
+		target.SetReserved(target.GetReserved() + migration.reserved);
+		source.SetIsDraining(true);
+		if (target.GetIsReady()) {
+			SendOrder(migration, source, target);
+		} else {
+			migration.state = eState::STARTING_TARGET;
+			migration.deadline = Clock::now() + TARGET_START_TIMEOUT;
+			Report(migration, eState::STARTING_TARGET, "Starting instance " + std::to_string(migration.target),
+				static_cast<uint16_t>(source.GetCurrentClientCount()));
+		}
+	}
+}
+
+eRefusal MigrationCoordinator::Start(const InstanceMigrationRequest& request, const Options& options) {
 	Migration migration;
 	migration.id = request.requestId;
 	migration.kind = request.kind;
@@ -175,6 +211,10 @@ eRefusal MigrationCoordinator::Start(const InstanceMigrationRequest& request) {
 	migration.seamless = request.seamless;
 	migration.requester = request.requesterId;
 	migration.by = request.requestedBy;
+	migration.liveUpdate = options.liveUpdate;
+	migration.prepare = options.prepare;
+	migration.prepareWaitSeconds = std::min(options.prepareWaitSeconds, MigratePrepare::MAX_WAIT_SECONDS);
+	migration.playerWaitSeconds = std::min(options.playerWaitSeconds, MigratePlayersOrder::MAX_MAX_WAIT_SECONDS);
 
 	const auto refuse = [&migration](eRefusal refusal) {
 		Report(migration, eState::FAILED, Describe(refusal));
@@ -186,7 +226,8 @@ eRefusal MigrationCoordinator::Start(const InstanceMigrationRequest& request) {
 	Instance* source = FindInstance(request.zoneId, request.sourceInstance).get();
 	if (!source) return refuse(eRefusal::NOT_RUNNING);
 	auto sourceView = View(*source);
-	if (const auto refusal = CheckSource(sourceView, IsActivityZone(request.zoneId)); refusal != eRefusal::NONE) return refuse(refusal);
+	const auto sourceRefusal = options.liveUpdate ? CheckLiveUpdateSource(sourceView) : CheckSource(sourceView, IsActivityZone(request.zoneId));
+	if (sourceRefusal != eRefusal::NONE) return refuse(sourceRefusal);
 	if (IsTakingPart(request.zoneId, request.sourceInstance)) return refuse(eRefusal::ALREADY_MIGRATING);
 	migration.clone = source->GetCloneID();
 
@@ -208,32 +249,36 @@ eRefusal MigrationCoordinator::Start(const InstanceMigrationRequest& request) {
 		if (IsTakingPart(request.zoneId, targetId)) return refuse(eRefusal::ALREADY_MIGRATING);
 		if (const auto refusal = CheckMergeTarget(View(*found), sourceView); refusal != eRefusal::NONE) return refuse(refusal);
 		target = found.get();
+	} else if (migration.prepare) {
+		// A property: the new instance loads it from the database when it starts, so the source saves and freezes it
+		// first (HandleStatus starts the target once it is PREPARED). Visitors still go to the source meanwhile.
+		LOG("Migration %u requested by %s: preparing zone %u clone %u instance %u (%i player(s))", migration.id, migration.by.c_str(),
+			migration.zone, migration.clone, migration.source, source->GetCurrentClientCount());
+		auto& active = g_Active[migration.id] = migration;
+		active.state = eState::PREPARING;
+		active.deadline = Clock::now() + std::chrono::seconds(active.prepareWaitSeconds) + TARGET_START_TIMEOUT;
+		MigratePrepare prepare;
+		prepare.migrationId = active.id;
+		prepare.maxWaitSeconds = active.prepareWaitSeconds;
+		MasterPackets::SendTo(source->GetSysAddr(), prepare);
+		Report(active, eState::PREPARING, "Saving the property", static_cast<uint16_t>(source->GetCurrentClientCount()));
+		return eRefusal::NONE;
 	} else {
 		// A fresh world server, started from the binary on disk now: this is how a live update takes over
-		const auto& started = Game::im->StartNewInstance(source->GetMapID(), source->GetCloneID());
-		if (!started) return refuse(eRefusal::MASTER_SHUTTING_DOWN);
-		target = started.get();
+		target = StartTarget(*source);
+		if (!target) {
+			source->SetIsDraining(false);
+			return refuse(eRefusal::MASTER_SHUTTING_DOWN);
+		}
 		migration.startedTarget = true;
+		// Starting it grew the instance list, which moves the InstancePtrs but not the Instances: source is still good
 	}
 
-	migration.target = target->GetInstanceID();
-	// Seats for everyone there now; a few may still arrive while draining, the hard cap is the real limit
-	migration.reserved = source->GetCurrentClientCount();
-	target->SetReserved(target->GetReserved() + migration.reserved);
-	source->SetIsDraining(true);
-
 	LOG("Migration %u requested by %s: %s zone %u instance %u (%i player(s)) -> instance %u", migration.id, migration.by.c_str(),
-		KindName(migration.kind), migration.zone, migration.source, source->GetCurrentClientCount(), migration.target);
+		KindName(migration.kind), migration.zone, migration.source, source->GetCurrentClientCount(), target->GetInstanceID());
 
 	auto& active = g_Active[migration.id] = migration;
-	if (target->GetIsReady()) {
-		SendOrder(active, *source, *target);
-	} else {
-		active.state = eState::STARTING_TARGET;
-		active.deadline = Clock::now() + TARGET_START_TIMEOUT;
-		Report(active, eState::STARTING_TARGET, "Starting instance " + std::to_string(active.target),
-			static_cast<uint16_t>(source->GetCurrentClientCount()));
-	}
+	Begin(active, *source, *target);
 	return eRefusal::NONE;
 }
 
@@ -248,6 +293,20 @@ void MigrationCoordinator::HandleStatus(const SystemAddress& from, const Migrati
 	migration.failed = status.failed;
 	if (status.state == eState::DONE) return Finish(it, true, status.message);
 	if (status.state == eState::FAILED) return Finish(it, false, status.message);
+	if (status.state == eState::PREPARED) {
+		if (migration.state != eState::PREPARING) return;
+		// Saved and frozen: now the new instance may load the property
+		Instance* sourceInstance = source.get();
+		Instance* target = StartTarget(*sourceInstance);
+		if (!target) {
+			SendCancel(migration);
+			return Finish(it, false, "Master refused to start the new instance");
+		}
+		migration.startedTarget = true;
+		LOG("Migration %u: property of zone %u clone %u saved; instance %u -> %u", migration.id, migration.zone, migration.clone, migration.source, target->GetInstanceID());
+		Report(migration, eState::PREPARED, status.message, static_cast<uint16_t>(sourceInstance->GetCurrentClientCount()));
+		return Begin(migration, *sourceInstance, *target);
+	}
 	migration.state = status.state;
 	Report(migration, status.state, status.message, status.remaining);
 }
@@ -289,7 +348,14 @@ void MigrationCoordinator::Update() {
 	for (auto it = g_Active.begin(); it != g_Active.end();) {
 		auto& migration = it->second;
 		auto current = it++;
-		if (migration.state == eState::STARTING_TARGET) {
+		if (migration.state == eState::PREPARING) {
+			if (!FindInstance(migration.zone, migration.source)) {
+				Finish(current, false, "The instance being emptied stopped");
+			} else if (now > migration.deadline) {
+				SendCancel(migration);
+				Finish(current, false, "The property wasn't saved in time");
+			}
+		} else if (migration.state == eState::STARTING_TARGET) {
 			const auto& source = FindInstance(migration.zone, migration.source);
 			const auto& target = FindInstance(migration.zone, migration.target);
 			if (!source || !target) {

dMasterServer/MigrationCoordinator.h

@@ -17,9 +17,24 @@ namespace MigrationCoordinator {
 	// Where statuses go (every world, so the GM who asked hears about it wherever they are); set once at startup
 	void SetReporter(std::function<void(const MigrationStatus&)> reporter);
 
-	// A migration was asked for (a GM command, or later a dashboard); anything but NONE means it was refused (and
+	// Also told every status, including those nobody asked for in game (live updates follow their migrations here)
+	void SetObserver(std::function<void(const MigrationStatus&)> observer);
+
+	// What a live update (LiveUpdateCoordinator) changes about a migration
+	struct Options {
+		// Moves what CheckSource refuses (see CheckLiveUpdateSource): character selection, private instances,
+		// properties and activity zones
+		bool liveUpdate{};
+		// Properties: the source saves and freezes the property (MIGRATE_PREPARE) before the new instance is started
+		bool prepare{};
+		uint16_t prepareWaitSeconds{ 60 };
+		// Players who are dead or building wait this long before they are moved anyway
+		uint16_t playerWaitSeconds{ MigratePlayersOrder::DEFAULT_MAX_WAIT_SECONDS };
+	};
+
+	// A migration was asked for (a GM command, or a live update); anything but NONE means it was refused (and
 	// reported as FAILED)
-	InstanceMigration::eRefusal Start(const InstanceMigrationRequest& request);
+	InstanceMigration::eRefusal Start(const InstanceMigrationRequest& request, const Options& options = {});
 
 	// A world reported progress on a migration it is running
 	void HandleStatus(const SystemAddress& from, const MigrationStatus& status);