feat: live updates move every server onto a new build without a restart

With new binaries in place, master moves everything onto them while the
server keeps running (the dashboard's Live update, /liveupdate or SIGUSR2 to
master):

- database migrations of the new build first; a failure stops there
- UGC finishes the jobs it is running (queued rows stay pending), auth
  restarts, chat hands its teams to master for the next chat server; master
  starts the new processes and retries ones that don't come back
- once the new chat server is up (CHAT_SERVER_READY) every world connects at
  once and sends its players again (LoginSessionNotify resync, no login logged)
- every world instance is replaced with an instance migration: public worlds
  and private ones (same password) at once, properties after the old instance
  saved and froze the property (MIGRATE_PREPARE: no building, claiming or
  saving there any more), activity zones and character select once their
  players left or after a wait; empty instances just stop, zones in
  prestart_worlds get a new one first
- the dashboard restarts last and picks the status up again

Players land where they stood (position carried in CarriedPlayerState, also on
properties and Moon Base). Draining instances get no new players
(InstanceMigration::AcceptsNewPlayers) and show as "Moving players" in the
world list. The order lives in LiveUpdateMachine.h without master state and is
unit tested; master's glue is LiveUpdateCoordinator. Master itself is not
replaced. Message IDs are appended only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dGame/UserManager.h

@@ -47,6 +47,8 @@ public:
 	void SaveAllActiveCharacters();
 
 	size_t GetUserCount() const { return m_Users.size(); }
+	// Everyone connected, by address (character selection moves them one by one in a live update)
+	const std::map<SystemAddress, User*>& GetUsers() const { return m_Users; }
 
 	// Access cached config values
 	bool GetMuteAutoRejectNames() const { return m_MuteAutoRejectNames; }

dGame/dComponents/PropertyManagementComponent.cpp

@@ -117,6 +117,8 @@ bool PropertyManagementComponent::IsBestFriend(const LWOOBJID player) const {
 }
 
 bool PropertyManagementComponent::CanBuild(const Entity& player) const {
+	// Handed off to a new instance: building happens there
+	if (frozen) return false;
 	const bool bestFriendsBuild = PropertyBuilders::BestFriendsBuild();
 	PropertyBuilders::Player state{ .id = player.GetObjectID(), .isBuilding = builders.contains(player.GetObjectID()) };
 	// Only a best friend needs the lookup
@@ -267,7 +269,7 @@ void PropertyManagementComponent::UpdatePropertyDetails(std::string name, std::s
 }
 
 bool PropertyManagementComponent::Claim(const LWOOBJID playerId) {
-	if (owner != LWOOBJID_EMPTY) {
+	if (owner != LWOOBJID_EMPTY || frozen) {
 		return false;
 	}
 
@@ -510,6 +512,8 @@ void PropertyManagementComponent::SendModelsOnProperty() const {
 }
 
 void PropertyManagementComponent::UpdateModelPosition(Entity& builder, const LWOOBJID id, const NiPoint3 position, NiQuaternion rotation) {
+	// Handed off to a new instance (live update): nothing changes here any more
+	if (frozen) return;
 	LOG("Placing model <%f, %f, %f>", position.x, position.y, position.z);
 
 	auto* entity = &builder;
@@ -560,6 +564,7 @@ void PropertyManagementComponent::UpdateModelPosition(Entity& builder, const LWO
 }
 
 void PropertyManagementComponent::DeleteModel(Entity& builder, const LWOOBJID id, const int deleteReason) {
+	if (frozen) return;
 	LOG("Delete model: (%llu) (%i)", id, deleteReason);
 
 	auto* model = Game::entityManager->GetEntity(id);
@@ -761,6 +766,9 @@ void PropertyManagementComponent::Load() {
 }
 
 void PropertyManagementComponent::Save() {
+	// Handed off to a new instance (live update): what it saves is what counts
+	if (frozen) return;
+
 	// From the property's own ids: it saves whoever is here (the owner may not be)
 	if (propertyId == LWOOBJID_EMPTY || owner == LWOOBJID_EMPTY) {
 		return;
@@ -833,6 +841,29 @@ void PropertyManagementComponent::Save() {
 	DashboardNotify::Changed("properties", propertyId);
 }
 
+void PropertyManagementComponent::FreezeForHandOff() {
+	if (frozen) return;
+	// Everyone still building stops (their models are kept as they are now), and nobody can start again
+	std::vector<LWOOBJID> building(builders.begin(), builders.end());
+	for (const auto id : building) {
+		auto* builder = Game::entityManager->GetEntity(id);
+		if (builder) EndBuilding(*builder);
+		else builders.erase(id);
+	}
+	Save();
+	frozen = true;
+	// Takes the build rights the clients were given away (CanBuild is false for everyone now)
+	UpdateBuildRights();
+	LOG("Property %llu (clone %u) saved and handed off to a new instance", propertyId, clone_Id);
+}
+
+void PropertyManagementComponent::Unfreeze() {
+	if (!frozen) return;
+	frozen = false;
+	UpdateBuildRights();
+	LOG("Property %llu (clone %u) is no longer handed off", propertyId, clone_Id);
+}
+
 void PropertyManagementComponent::AddModel(LWOOBJID modelId, LWOOBJID spawnerId) {
 	models[modelId] = spawnerId;
 }

dGame/dComponents/PropertyManagementComponent.h

@@ -203,6 +203,21 @@ public:
 
 
 	void OnChatMessageReceived(const std::string& sMessage) const;
+
+	/**
+	 * Live updates (docs/LiveUpdate.md): a new instance of this property is about to load it from the database. Takes
+	 * everyone out of build mode, saves, and from then on nobody can build or claim it here and it is never saved again,
+	 * so the new instance's saves can't be overwritten by this one.
+	 */
+	void FreezeForHandOff();
+
+	// The hand-off was called off before anyone went to the new instance: building and saving work again
+	void Unfreeze();
+
+	bool IsFrozen() const noexcept { return frozen; }
+
+	// Players in build mode here now
+	size_t GetBuilderCount() const noexcept { return builders.size(); }
 private:
 	// Sends the property's data to one player; a player who can build is told they own it, which is the client's only
 	// check before editing (the owner's name stays the owner's)
@@ -219,6 +234,9 @@ private:
 	 */
 	static PropertyManagementComponent* instance;
 
+	// Handed off to a new instance for a live update (FreezeForHandOff)
+	bool frozen = false;
+
 	/**
 	 * The ID of the owner of this property
 	 */

dGame/dUtilities/SlashCommandHandler.cpp

@@ -1714,6 +1714,15 @@ void SlashCommandHandler::Startup() {
 		.requiredLevel = eGameMasterLevel::DEVELOPER
 	});
 
+	RegisterCommand({
+		.help = "Moves the whole server onto the binaries on disk now",
+		.info = "A live update: with a new build in place, master restarts the UGC, auth and chat servers, replaces every world instance with a new one and moves its players there (with the game's Mythran maintenance warning and a short loading screen), then restarts the dashboard. Master itself keeps running. Usage: /liveupdate [start [warn seconds, 0-300] | cancel | status]",
+		.aliases = {"liveupdate"},
+		.handle = WorldMigration::LiveUpdateCommand,
+		.requiredLevel = eGameMasterLevel::OPERATOR,
+		.dashboardPermission = "server_live_update"
+	});
+
 	RegisterCommand({
 		.help = "[claim] Community challenges and live events",
 		.info = "Shows the server-wide community challenges running now, how far along they are and what you added, and the live events in this world. Also gives you the coins waiting from challenges you helped complete",

dGame/dUtilities/WorldMigration.cpp

@@ -20,6 +20,9 @@
 #include "MessageType/Game.h"
 #include "MessageType/Master.h"
 #include "PetComponent.h"
+#include "PropertyManagementComponent.h"
+#include "ControllablePhysicsComponent.h"
+#include "UserManager.h"
 #include "PlayerManager.h"
 #include "ServiceType.h"
 #include "TradingManager.h"
@@ -30,8 +33,6 @@
 using namespace InstanceMigration;
 
 namespace {
-	// Players who are dead or building wait this long to finish before they go anyway
-	constexpr float MAX_WAIT_SECONDS = 15.0f;
 	// A client that got its transfer and is still connected after this is dropped (it is saved already)
 	constexpr float LEAVE_TIMEOUT_SECONDS = 20.0f;
 	// Nobody left to move for this long: done (players still loading in finish first)
@@ -60,6 +61,17 @@ namespace {
 	};
 
 	std::optional<Active> g_Active;
+
+	// A property being saved before its new instance loads it (MIGRATE_PREPARE)
+	struct Preparing {
+		uint32_t migrationId{};
+		float waitLeft{};
+		bool told{};
+	};
+	std::optional<Preparing> g_Preparing;
+	// The migration this property was frozen for (0: not frozen)
+	uint32_t g_FrozenFor = 0;
+	std::string g_LiveUpdateTold; // the last live update summary the GM who asked was told
 	std::map<SystemAddress, Leaving> g_Leaving;
 	std::map<LWOOBJID, std::pair<CarriedPlayerState, float>> g_Carried;
 	std::function<void(const SystemAddress&)> g_Cleanup;
@@ -93,14 +105,38 @@ namespace {
 		announcement.SendToClient(player->GetSystemAddress());
 	}
 
+	// Character selection has users but no player entities
+	bool IsCharacterSelect() {
+		return Game::server->GetZoneID() == 0;
+	}
+
 	uint16_t Remaining() {
 		uint16_t remaining = 0;
+		if (IsCharacterSelect()) {
+			for (const auto& [sysAddr, user] : UserManager::Instance()->GetUsers()) {
+				if (user && !g_Leaving.contains(sysAddr)) remaining++;
+			}
+			return remaining;
+		}
 		for (auto* player : PlayerManager::GetAllPlayers()) {
 			if (player && !g_Leaving.contains(player->GetSystemAddress())) remaining++;
 		}
 		return remaining;
 	}
 
+	// Someone at character selection: nothing to save, they just connect to the new one, which sends them their
+	// characters again
+	void MoveUser(const SystemAddress& sysAddr) {
+		const auto& order = g_Active->order;
+		g_Leaving[sysAddr] = { LWOOBJID_EMPTY, 0.0f };
+		ClientPackets::TransferToWorld transfer;
+		transfer.serverIP = LUString(order.targetIp);
+		transfer.serverPort = order.targetPort;
+		transfer.mythranShift = order.mythranShift;
+		transfer.Send(sysAddr);
+		LOG("Migration %u: sent %s from character selection to instance %u", order.migrationId, sysAddr.ToString(), order.targetInstance);
+	}
+
 	// Save, lock and send one player over. Returns false when they couldn't be sent.
 	bool MovePlayer(Entity* player) {
 		auto* character = player->GetCharacter();
@@ -120,6 +156,19 @@ namespace {
 		carried.characterId = playerId;
 		carried.seamless = order.seamless;
 		if (auto* pet = PetComponent::GetActivePet(playerId)) carried.petItemId = pet->GetItemId();
+		// Where they stand: the saved character doesn't keep it on properties (or Moon Base)
+		if (auto* physics = player->GetComponent<ControllablePhysicsComponent>()) {
+			const auto& position = physics->GetPosition();
+			const auto& rotation = physics->GetRotation();
+			carried.hasPosition = true;
+			carried.x = position.x;
+			carried.y = position.y;
+			carried.z = position.z;
+			carried.rotW = rotation.w;
+			carried.rotX = rotation.x;
+			carried.rotY = rotation.y;
+			carried.rotZ = rotation.z;
+		}
 
 		// Same zone, so they spawn where they stood (Entity.cpp only uses a spawn point when the zone changes)
 		character->SetZoneID(order.targetZone);
@@ -153,6 +202,19 @@ namespace {
 	void MovePlayers(float deltaTime) {
 		auto& active = *g_Active;
 		active.budget = std::min(active.budget + active.order.playersPerSecond * deltaTime, static_cast<float>(active.order.playersPerSecond));
+		if (IsCharacterSelect()) {
+			std::vector<SystemAddress> users;
+			for (const auto& [sysAddr, user] : UserManager::Instance()->GetUsers()) {
+				if (user && !g_Leaving.contains(sysAddr)) users.push_back(sysAddr);
+			}
+			for (const auto& sysAddr : users) {
+				if (active.budget < 1.0f) break;
+				active.budget -= 1.0f;
+				MoveUser(sysAddr);
+				active.moved++;
+			}
+			return;
+		}
 		// Copy: moving a player can change the list (a cancelled trade, a pet)
 		const auto players = PlayerManager::GetAllPlayers();
 		for (auto* player : players) {
@@ -161,7 +223,7 @@ namespace {
 			auto* character = player->GetCharacter();
 			const bool inTrade = TradingManager::Instance()->GetPlayerTrade(player->GetObjectID()) != nullptr;
 			auto& waited = active.waiting[player->GetObjectID()];
-			const auto decision = DecidePlayer(inTrade, character && character->GetBuildMode(), player->GetIsDead(), waited, MAX_WAIT_SECONDS);
+			const auto decision = DecidePlayer(inTrade, character && character->GetBuildMode(), player->GetIsDead(), waited, active.order.maxWaitSeconds);
 			if (decision == ePlayerDecision::WAIT) {
 				waited += deltaTime;
 				continue;
@@ -195,6 +257,15 @@ void WorldMigration::SetCleanupHandler(std::function<void(const SystemAddress&)>
 
 void WorldMigration::HandleOrder(const MigratePlayersOrder& order) {
 	if (order.targetPort == 0) {
+		if (g_Preparing && g_Preparing->migrationId == order.migrationId) {
+			LOG("Migration %u cancelled while the property was being saved", order.migrationId);
+			g_Preparing.reset();
+		}
+		// A property nobody went to the new instance of is ours again; once someone did, it stays with them
+		if (g_FrozenFor == order.migrationId && (!g_Active || g_Active->moved == 0)) {
+			if (auto* property = PropertyManagementComponent::Instance()) property->Unfreeze();
+			g_FrozenFor = 0;
+		}
 		if (!g_Active || g_Active->order.migrationId != order.migrationId) return;
 		LOG("Migration %u cancelled; %u player(s) were moved", order.migrationId, g_Active->moved);
 		SendStatus(eState::FAILED, Remaining(), "Cancelled");
@@ -219,6 +290,27 @@ void WorldMigration::HandleOrder(const MigratePlayersOrder& order) {
 	}
 }
 
+void WorldMigration::HandlePrepare(const MigratePrepare& prepare) {
+	if (g_Active || g_Preparing) {
+		LOG("Migration %u refused: another migration is running here", prepare.migrationId);
+		SendStatus(prepare.migrationId, 0, eState::FAILED, 0, 0, Remaining(), "Another migration is running on this instance");
+		return;
+	}
+	g_Preparing = Preparing{ prepare.migrationId, static_cast<float>(prepare.maxWaitSeconds), false };
+	LOG("Migration %u: saving this instance's property for its replacement (waiting up to %u s for builders)", prepare.migrationId, prepare.maxWaitSeconds);
+}
+
+void WorldMigration::ApplyCarriedPosition(Entity* player) {
+	if (!player) return;
+	const auto it = g_Carried.find(player->GetObjectID());
+	if (it == g_Carried.end() || !it->second.first.hasPosition) return;
+	const auto& state = it->second.first;
+	auto* physics = player->GetComponent<ControllablePhysicsComponent>();
+	if (!physics) return;
+	physics->SetPosition(NiPoint3(state.x, state.y, state.z));
+	physics->SetRotation(NiQuaternion(state.rotW, state.rotX, state.rotY, state.rotZ));
+}
+
 void WorldMigration::StoreCarriedState(const CarriedPlayerState& state) {
 	if (state.targetZone != Game::server->GetZoneID() || state.targetInstance != Game::server->GetInstanceID()) return;
 	g_Carried[state.characterId] = { state, 0.0f };
@@ -231,6 +323,28 @@ void WorldMigration::Update(float deltaTime) {
 		else ++it;
 	}
 	DropStuckClients(deltaTime);
+
+	if (g_Preparing) {
+		auto* property = PropertyManagementComponent::Instance();
+		if (property && property->GetBuilderCount() > 0 && g_Preparing->waitLeft > 0.0f) {
+			if (!g_Preparing->told) {
+				g_Preparing->told = true;
+				const auto seconds = std::to_string(static_cast<int>(g_Preparing->waitLeft));
+				for (auto* player : PlayerManager::GetAllPlayers()) {
+					if (player) ChatPackets::SendSystemMessage(player->GetSystemAddress(),
+						GeneralUtils::ASCIIToUTF16("The server is being updated: building on this property ends in " + seconds + " seconds."));
+				}
+			}
+			g_Preparing->waitLeft -= deltaTime;
+		} else {
+			const auto migrationId = g_Preparing->migrationId;
+			g_Preparing.reset();
+			if (property) property->FreezeForHandOff();
+			g_FrozenFor = migrationId;
+			SendStatus(migrationId, 0, eState::PREPARED, 0, 0, Remaining(), property ? "Property saved" : "Nothing to save");
+		}
+	}
+
 	if (!g_Active) return;
 
 	auto& active = *g_Active;
@@ -401,3 +515,61 @@ void WorldMigration::MergeInstanceCommand(Entity* entity, const SystemAddress& s
 		(target ? u"instance " + GeneralUtils::ASCIIToUTF16(std::to_string(target)) : std::u16string(u"the best fit")) + u".");
 	RequestMigration(entity, eKind::MERGE, target, warnSeconds, seamless);
 }
+
+namespace {
+	std::string LiveUpdateSummary(const LiveUpdateStatus& status) {
+		using namespace LiveUpdate;
+		size_t done = 0, failed = 0;
+		for (const auto& unit : status.units) {
+			if (IsFinished(unit.state)) done++;
+			if (unit.state == eUnitState::FAILED) failed++;
+		}
+		std::string text = "Live update " + std::to_string(status.updateId) + ": " + PhaseName(status.phase);
+		if (!status.units.empty()) text += ", " + std::to_string(done) + " of " + std::to_string(status.units.size()) + " done";
+		if (failed) text += ", " + std::to_string(failed) + " failed";
+		if (!status.message.empty()) text += ". " + status.message;
+		return text;
+	}
+}
+
+void WorldMigration::LiveUpdateCommand(Entity* entity, const SystemAddress& sysAddr, const std::string args) {
+	const auto words = GeneralUtils::SplitString(args, ' ');
+	const std::string verb = words.empty() || words[0].empty() ? "status" : words[0];
+	LiveUpdateRequest request;
+	if (verb == "start") {
+		request.action = LiveUpdate::eAction::START;
+		if (words.size() > 1) {
+			const auto warn = GeneralUtils::TryParse<int32_t>(words[1]);
+			if (!warn || *warn < 0 || *warn > InstanceMigrationRequest::MAX_WARN_SECONDS) {
+				ChatPackets::SendSystemMessage(sysAddr, u"Usage: /liveupdate start [warn seconds, 0-300]");
+				return;
+			}
+			request.warnSeconds = *warn;
+		}
+	} else if (verb == "cancel") {
+		request.action = LiveUpdate::eAction::CANCEL;
+	} else if (verb == "status") {
+		request.action = LiveUpdate::eAction::STATUS;
+	} else {
+		ChatPackets::SendSystemMessage(sysAddr, u"Usage: /liveupdate [start [warn seconds] | cancel | status]");
+		return;
+	}
+	if (entity) {
+		request.requesterId = entity->GetObjectID();
+		if (auto* character = entity->GetCharacter()) request.requestedBy = character->GetName();
+	}
+	g_LiveUpdateTold.clear();
+	LOG("Asking master to %s a live update (requested by %s)", verb.c_str(), request.requestedBy.c_str());
+	MasterPackets::SendToMaster(request);
+	if (request.action == LiveUpdate::eAction::START) ChatPackets::SendSystemMessage(sysAddr, u"Asking master to move everything onto the binaries on disk now.");
+}
+
+void WorldMigration::HandleLiveUpdateStatus(const LiveUpdateStatus& status) {
+	if (status.requesterId == LWOOBJID_EMPTY) return;
+	auto* requester = PlayerManager::GetPlayer(status.requesterId);
+	if (!requester) return;
+	const auto text = LiveUpdateSummary(status);
+	if (text == g_LiveUpdateTold) return;
+	g_LiveUpdateTold = text;
+	ChatPackets::SendSystemMessage(requester->GetSystemAddress(), GeneralUtils::ASCIIToUTF16(text));
+}

dGame/dUtilities/WorldMigration.h

@@ -4,6 +4,7 @@
 #include <functional>
 
 #include "master/InstanceMigration.h"
+#include "master/LiveUpdate.h"
 #include "RakNetTypes.h"
 #include "dCommonVars.h"
 
@@ -28,6 +29,12 @@ namespace WorldMigration {
 	// MIGRATE_PLAYER_STATE from master: a player on their way here
 	void StoreCarriedState(const CarriedPlayerState& state);
 
+	// MIGRATE_PREPARE from master (live update of a property): wait for builders, save the property and freeze it
+	void HandlePrepare(const MigratePrepare& prepare);
+
+	// The player entity of someone moved here was just made: put them where they stood (properties don't save it)
+	void ApplyCarriedPosition(Entity* player);
+
 	// Every world frame
 	void Update(float deltaTime);
 
@@ -59,6 +66,12 @@ namespace WorldMigration {
 
 	// /mergeinstance [target instance, 0 for the best fit] [warn seconds] [seamless]
 	void MergeInstanceCommand(Entity* entity, const SystemAddress& sysAddr, const std::string args);
+
+	// /liveupdate [start [warn seconds] | cancel | status]: a live update of the whole server (docs/LiveUpdate.md)
+	void LiveUpdateCommand(Entity* entity, const SystemAddress& sysAddr, const std::string args);
+
+	// LIVE_UPDATE_STATUS from master: tell the GM who asked, if they are here
+	void HandleLiveUpdateStatus(const LiveUpdateStatus& status);
 }
 
 #endif  //!__WORLDMIGRATION__H__