feat: live updates move every server onto a new build without a restart

With new binaries in place, master moves everything onto them while the
server keeps running (the dashboard's Live update, /liveupdate or SIGUSR2 to
master):

- database migrations of the new build first; a failure stops there
- UGC finishes the jobs it is running (queued rows stay pending), auth
  restarts, chat hands its teams to master for the next chat server; master
  starts the new processes and retries ones that don't come back
- once the new chat server is up (CHAT_SERVER_READY) every world connects at
  once and sends its players again (LoginSessionNotify resync, no login logged)
- every world instance is replaced with an instance migration: public worlds
  and private ones (same password) at once, properties after the old instance
  saved and froze the property (MIGRATE_PREPARE: no building, claiming or
  saving there any more), activity zones and character select once their
  players left or after a wait; empty instances just stop, zones in
  prestart_worlds get a new one first
- the dashboard restarts last and picks the status up again

Players land where they stood (position carried in CarriedPlayerState, also on
properties and Moon Base). Draining instances get no new players
(InstanceMigration::AcceptsNewPlayers) and show as "Moving players" in the
world list. The order lives in LiveUpdateMachine.h without master state and is
unit tested; master's glue is LiveUpdateCoordinator. Master itself is not
replaced. Message IDs are appended only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dDashboardServer/DashboardServer.cpp

@@ -59,6 +59,7 @@
 #include "ModerationTools.h"
 #include "ModeratorHelper.h"
 #include "LiveWorld.h"
+#include "LiveUpdateRoutes.h"
 #include "Scenery.h"
 #include "Workers.h"
 #include "ReportRoutes.h"
@@ -222,7 +223,9 @@ namespace {
 			info.zoneName = GetZoneDisplayName(info.mapID);
 			AddPropertyDetails(info);
 			using eState = MasterPackets::ServerListResponse::eState;
-			if (instance.state == eState::READY) {
+			if (instance.state == eState::READY || instance.state == eState::DRAINING) {
+				// Draining: still up with its players, who are being moved to a new instance
+				if (instance.state == eState::DRAINING) info.state = "draining";
 				ServerState::g_WorldInstances.push_back(info);
 			} else {
 				info.state = instance.state == eState::STARTING ? "starting" : "stopping";
@@ -340,6 +343,7 @@ namespace {
 			handlers.On<PlayerActionResult>(Master::PLAYER_ACTION_RESULT, [](const PlayerActionResult& result, const SystemAddress&) { PlayerActions::HandleResult(result); });
 			handlers.On<MasterPackets::WorldShutDown>(Master::SHUTDOWN_RESPONSE, OnWorldShutDown);
 			handlers.On<ServerTraffic>(Master::SERVER_TRAFFIC, [](const ServerTraffic& report, const SystemAddress&) { Traffic::Ingest(report); });
+			handlers.On<LiveUpdateStatus>(Master::LIVE_UPDATE_STATUS, [](const LiveUpdateStatus& status, const SystemAddress&) { LiveUpdateRoutes::HandleStatus(status); });
 			return handlers;
 		}();
 		return handlers;
@@ -540,6 +544,7 @@ int main(int argc, char** argv) {
 	RegisterModerationToolRoutes();
 	ModeratorHelper::RegisterRoutes();
 	LiveWorld::RegisterRoutes();
+	LiveUpdateRoutes::RegisterRoutes();
 	Inspector::RegisterRoutes();
 	RegisterCDClientBrowserRoutes();
 	RegisterSettingsRoutes();
@@ -606,6 +611,7 @@ int main(int argc, char** argv) {
 			Scheduler::Update();
 			ServerRoutes::Update();
 			LiveWorld::Update();
+			LiveUpdateRoutes::Update();
 			Inspector::Update();
 			Announcements::Update();
 			EventsCalendar::Update();

dDashboardServer/routes/CMakeLists.txt

@@ -16,6 +16,7 @@ set(DASHBOARDROUTES_SOURCES
 	"WebhookRoutes.cpp"
 	"EconomyJobs.cpp"
 	"LiveWorld.cpp"
+	"LiveUpdateRoutes.cpp"
 	"SettingsRoutes.cpp"
 	"Background.cpp"
 	"Scheduler.cpp"

dDashboardServer/routes/LiveUpdateRoutes.cpp

@@ -0,0 +1,138 @@
+#include "LiveUpdateRoutes.h"
+
+#include <chrono>
+#include <ctime>
+
+#include "Alerts.h"
+#include "DashboardRoutes.h"
+#include "Game.h"
+#include "Logger.h"
+#include "MasterPackets.h"
+#include "Permissions.h"
+#include "RouteUtils.h"
+#include "Web.h"
+#include "dServer.h"
+#include "eHTTPMethod.h"
+#include "master/LiveUpdate.h"
+
+using namespace RouteUtils;
+
+namespace {
+	constexpr const char* TOPIC = "live_update";
+	constexpr const char* PERMISSION = "server_live_update";
+
+	LiveUpdateStatus g_Status;
+	bool g_Asked = false;           // asked master for the status since connecting
+	bool g_WasConnected = false;
+	LiveUpdate::ePhase g_AlertedPhase = LiveUpdate::ePhase::IDLE;
+	uint32_t g_AlertedUpdate = 0;
+
+	nlohmann::json ToJson(const LiveUpdateStatus& status) {
+		using namespace LiveUpdate;
+		const auto& zones = ZoneNames();
+		nlohmann::json units = nlohmann::json::array();
+		size_t done = 0, failed = 0;
+		for (const auto& unit : status.units) {
+			if (IsFinished(unit.state)) done++;
+			if (unit.state == eUnitState::FAILED) failed++;
+			nlohmann::json row{ {"kind", KindName(unit.kind)}, {"state", StateName(unit.state)}, {"message", unit.message} };
+			if (unit.kind == eUnitKind::WORLD) {
+				const auto zone = std::to_string(unit.zoneId);
+				row["zone"] = unit.zoneId;
+				row["zoneName"] = unit.zoneId == 0 ? "Character Select" : zones.contains(zone) ? zones[zone].get<std::string>() : "Zone " + zone;
+				row["instance"] = unit.instanceId;
+				row["clone"] = unit.cloneId;
+				row["replacement"] = unit.replacement;
+				row["players"] = unit.players;
+				row["moved"] = unit.moved;
+				row["isPrivate"] = unit.isPrivate != 0;
+			}
+			units.push_back(std::move(row));
+		}
+		return {
+			{"updateId", status.updateId}, {"phase", PhaseName(status.phase)}, {"running", !IsFinished(status.phase) && status.phase != ePhase::IDLE},
+			{"startedAt", status.startedAt}, {"finishedAt", status.finishedAt}, {"by", status.by}, {"message", status.message},
+			{"done", done}, {"failed", failed}, {"total", status.units.size()}, {"units", units}
+		};
+	}
+
+	bool SendRequest(LiveUpdate::eAction action, int32_t warnSeconds, const std::string& by) {
+		if (!Game::server || !Game::server->GetIsConnectedToMaster()) return false;
+		LiveUpdateRequest request;
+		request.action = action;
+		request.warnSeconds = warnSeconds;
+		request.requestedBy = by.substr(0, LiveUpdateRequest::MAX_BY);
+		MasterPackets::SendToMaster(request);
+		return true;
+	}
+}
+
+namespace LiveUpdateRoutes {
+	void HandleStatus(const LiveUpdateStatus& status) {
+		g_Status = status;
+		auto json = ToJson(status);
+		Game::web.SendWSMessage(TOPIC, json);
+		// Webhooks hear about the start and the end
+		using LiveUpdate::ePhase;
+		if (status.updateId != 0 && (status.updateId != g_AlertedUpdate || status.phase != g_AlertedPhase)) {
+			const bool started = status.phase == ePhase::RUNNING && status.updateId != g_AlertedUpdate;
+			const bool finished = LiveUpdate::IsFinished(status.phase);
+			if (started) Alerts::Emit("server", "Live update started", "The servers are being moved onto the new build.", { { "By", status.by } }, "/#liveUpdate");
+			// Not for an old one a restarted dashboard hears about
+			const bool recent = status.finishedAt == 0 || std::time(nullptr) - status.finishedAt < 600;
+			if (finished && g_AlertedPhase != status.phase && recent) {
+				Alerts::Emit("server", std::string("Live update ") + LiveUpdate::PhaseName(status.phase), status.message, { { "By", status.by } }, "/#liveUpdate");
+			}
+			g_AlertedUpdate = status.updateId;
+			g_AlertedPhase = status.phase;
+		}
+	}
+
+	void Update() {
+		const bool connected = Game::server && Game::server->GetIsConnectedToMaster();
+		if (connected && !g_WasConnected) g_Asked = false;
+		g_WasConnected = connected;
+		if (!connected || g_Asked) return;
+		g_Asked = SendRequest(LiveUpdate::eAction::STATUS, LiveUpdateRequest::DEFAULT_WARN, "dashboard");
+	}
+
+	nlohmann::json StatusJson() {
+		return ToJson(g_Status);
+	}
+
+	void RegisterRoutes() {
+		Game::web.RegisterWSSubscription(TOPIC, std::function<uint8_t()>([] { return Permissions::Level(PERMISSION); }), PERMISSION);
+
+		Route(eHTTPMethod::GET, "/api/server/live_update", Perm(PERMISSION),
+			"The last live update (or the running one): phase, who started it, and every server and world instance with its state (also pushed on the live_update socket topic)",
+			[](HTTPReply& reply, const HTTPContext&) {
+				JsonSuccess(reply, { {"liveUpdate", StatusJson()} });
+			});
+
+		Route(eHTTPMethod::POST, "/api/server/live_update", Perm(PERMISSION),
+			"Start a live update onto the binaries on disk now: UGC, auth and chat restart, every world instance is replaced and its players moved, the dashboard restarts last. Body: {warnSeconds (0-300, optional)}",
+			[](HTTPReply& reply, const HTTPContext& context) {
+				const auto body = ParseBody(context);
+				int32_t warn = LiveUpdateRequest::DEFAULT_WARN;
+				if (body && body->contains("warnSeconds") && !(*body)["warnSeconds"].is_null()) {
+					warn = body->value("warnSeconds", -1);
+					if (warn < 0 || warn > InstanceMigrationRequest::MAX_WARN_SECONDS) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Warn players 0 to 300 seconds before they are moved");
+				}
+				if (!SendRequest(LiveUpdate::eAction::START, warn, context.authenticatedUser)) {
+					return JsonError(reply, eHTTPStatusCode::SERVICE_UNAVAILABLE, "Not connected to the master server");
+				}
+				Audit(context, "live_update", warn >= 0 ? "Started a live update (players warned " + std::to_string(warn) + " s before they are moved)" : "Started a live update");
+				JsonSuccess(reply, { {"message", "Asked master to start a live update"} });
+			});
+
+		Route(eHTTPMethod::POST, "/api/server/live_update/cancel", Perm(PERMISSION),
+			"Cancel the running live update: nothing new is started, what is under way finishes",
+			[](HTTPReply& reply, const HTTPContext& context) {
+				if (!SendRequest(LiveUpdate::eAction::CANCEL, LiveUpdateRequest::DEFAULT_WARN, context.authenticatedUser)) {
+					return JsonError(reply, eHTTPStatusCode::SERVICE_UNAVAILABLE, "Not connected to the master server");
+				}
+				Audit(context, "live_update_cancel", "Cancelled the live update");
+				JsonSuccess(reply, { {"message", "Asked master to cancel the live update"} });
+			});
+	}
+}

dDashboardServer/routes/LiveUpdateRoutes.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "json.hpp"
+
+struct LiveUpdateStatus;
+
+/**
+ * The dashboard's "Live update" (docs/LiveUpdate.md): starts one on master and shows how it goes. The status comes
+ * from master (LIVE_UPDATE_STATUS) and is pushed to browsers on the live_update socket topic.
+ */
+namespace LiveUpdateRoutes {
+	void RegisterRoutes();
+
+	// Master sent a status
+	void HandleStatus(const LiveUpdateStatus& status);
+
+	// Main loop: ask master for the status once connected (a restarted dashboard picks a running update up again)
+	void Update();
+
+	nlohmann::json StatusJson();
+}

dDashboardServer/routes/LiveWorld.cpp

@@ -245,14 +245,14 @@ namespace LiveWorld {
 				JsonSuccess(reply, { {"message", "Shutting it down"} });
 			});
 
-		Route(eHTTPMethod::GET, "/api/worlds", Perm("players_view"), "Every world with its instance and clone IDs, address, players and, for property instances, the property and owner; worlds still starting or shutting down have state: starting|stopping",
+		Route(eHTTPMethod::GET, "/api/worlds", Perm("players_view"), "Every world with its instance and clone IDs, address, players and, for property instances, the property and owner; worlds still starting or shutting down have state: starting|stopping, and those whose players are being moved to a new instance state: draining",
 			[](HTTPReply& reply, const HTTPContext&) {
 				nlohmann::json worlds = nlohmann::json::array();
 				std::lock_guard lock(ServerState::g_StatusMutex);
 				for (const auto& w : ServerState::g_WorldInstances) {
 					worlds.push_back({ {"mapID", w.mapID}, {"zoneName", w.zoneName}, {"instanceID", w.instanceID}, {"cloneID", w.cloneID}, {"players", w.players},
 						{"isPrivate", w.isPrivate}, {"ip", w.ip}, {"port", w.port}, {"propertyId", w.propertyId}, {"propertyName", w.propertyName},
-						{"ownerId", w.ownerId}, {"ownerName", w.ownerName} });
+						{"ownerId", w.ownerId}, {"ownerName", w.ownerName}, {"state", w.state} });
 				}
 				// Launched but not connected yet, or shutting down
 				for (const auto& w : ServerState::g_PendingWorlds) {

dDashboardServer/routes/ServerState.h

@@ -39,7 +39,8 @@ struct WorldInstanceInfo {
 	std::string propertyName{};
 	std::string ownerId{};
 	std::string ownerName{};
-	// "starting" (launched, not connected yet) or "stopping"; worlds that are up are in g_WorldInstances instead
+	// "starting" (launched, not connected yet) or "stopping" (in g_PendingWorlds), or "draining" for a world that is up
+	// but whose players are being moved to a new instance (in g_WorldInstances); empty for one that is just up
 	std::string state{};
 };
 
@@ -77,7 +78,8 @@ namespace ServerState {
 				{"cloneID", world.cloneID},
 				{"players", world.players},
 				{"isPrivate", world.isPrivate},
-				{"zoneName", world.zoneName}
+				{"zoneName", world.zoneName},
+				{"state", world.state}
 			});
 		}
 
@@ -103,7 +105,7 @@ namespace ServerState {
 			const auto mapID = world.value("mapID", 0u);
 			if (world.value("cloneID", 0u) == 0) {
 				worlds.push_back({ {"mapID", mapID}, {"instanceID", world.value("instanceID", 0u)}, {"players", world.value("players", 0u)},
-					{"zoneName", world.value("zoneName", "")} });
+					{"zoneName", world.value("zoneName", "")}, {"state", world.value("state", "")} });
 				continue;
 			}
 			const auto it = propertyRows.find(mapID);

dDashboardServer/routes/SettingsCatalog.cpp

@@ -137,6 +137,17 @@ namespace {
 		c.Add(Bool(MASTER, "enable_dashboard", "Start the web dashboard", "", false, true));
 		c.Add(Bool(SHARED, "skip_account_creation", "Skip the first-account prompt", "For non-interactive setups: master doesn't ask for an account when there are none.", false, true));
 
+		c.AddSection("Live updates", "Moving every server and world instance onto a new build without a restart (the dashboard's Live update, /liveupdate or SIGUSR2 to master). Read when one starts.");
+		c.Add(Unit(Int(MASTER, "live_update_warn_seconds", "Warning before moving players", "The game's Mythran maintenance warning is shown this long before players are moved. The dashboard can pick another for one update.", "10", 0, 300), "seconds"));
+		c.Add(Int(MASTER, "live_update_parallel_worlds", "Worlds at once", "World instances replaced at the same time.", "4", 1, 64));
+		c.Add(Unit(Int(MASTER, "live_update_player_wait", "Wait for busy players", "Players who are dead or building are moved once they are done, or after this long.", "30", 0, 600), "seconds"));
+		c.Add(Unit(Int(MASTER, "live_update_property_build_wait", "Wait for property builders", "A property is saved for its new instance once nobody builds there, or after this long (they leave build mode then).", "60", 0, 600), "seconds"));
+		c.Add(Unit(Int(MASTER, "live_update_char_select_wait", "Wait at character select", "Players picking a character get this long to go in by themselves; then they are moved to the new character select.", "60", 0, 3600), "seconds"));
+		c.Add(Unit(Int(MASTER, "live_update_activity_wait", "Wait for activities", "Races, minigames and other activity zones get this long to finish; then their players are moved (the activity is lost).", "1800", 0, 86400), "seconds"));
+		c.Add(Unit(Int(MASTER, "live_update_ugc_drain_timeout", "Wait for the UGC server", "The UGC server finishes the models it is making; after this long it is stopped (they are made again).", "300", 0, 3600), "seconds"));
+		c.Add(Unit(Int(MASTER, "live_update_service_timeout", "Server restart timeout", "How long auth, chat, the UGC server or the dashboard may take to stop or come back before master starts it again (3 tries).", "30", 10, 600), "seconds"));
+		c.Add(Bool(MASTER, "live_update_run_migrations", "Run database migrations", "Run the new build's database migrations first. The running servers must cope with the new schema until they are replaced.", true));
+
 		c.AddSection("Game client");
 		c.Add(Format(Text(SHARED, "client_location", "Client folder", "The folder with res/, or the one with client/ and versions/.", "", true), eFormat::PATH));
 		c.Add(Int(SHARED, "client_net_version", "Network version", "Clients reporting another version are refused. 171022 for the retail client, 171023 for Darkflame Universe clients.", "171022", 0, std::nullopt, true));

dDashboardServer/templates/index.jinja2

@@ -48,6 +48,44 @@
 
 {% include "world_instances.jinja2" %}
 
+{% if can.server_live_update %}
+<div class="row g-3 mt-1">
+	<div class="col-12" id="liveUpdate">
+		<div class="card">
+			<div class="card-header d-flex justify-content-between align-items-center">
+				<h5 class="mb-0">Live update</h5>
+				<span id="liveUpdatePhase"></span>
+			</div>
+			<div class="card-body">
+				<p class="small text-body-secondary">Moves everything onto the server binaries on disk now, without a restart: the UGC server finishes its jobs
+					and restarts, auth restarts (logins pause for a few seconds), chat hands its teams over and restarts, every world instance is
+					replaced by a new one and its players are moved there (the game's Mythran maintenance warning, then a short loading screen), and
+					the dashboard restarts last. Properties are saved before their new instance loads them; races and minigames finish first.
+					Master keeps running: updating master itself takes a normal restart.</p>
+				<div class="d-flex flex-wrap gap-2 align-items-center mb-2" id="liveUpdateForm">
+					<div class="input-group input-group-sm" style="max-width: 20rem">
+						<span class="input-group-text">Warn players</span>
+						<select class="form-select" id="liveUpdateWarn">
+							<option value="" selected>as set (live_update_warn_seconds)</option><option value="0">not at all</option><option value="10">10 seconds before</option>
+							<option value="30">30 seconds before</option><option value="60">1 minute before</option><option value="120">2 minutes before</option>
+						</select>
+					</div>
+					<button class="btn btn-sm btn-warning" id="liveUpdateStart">Start live update</button>
+					<button class="btn btn-sm btn-outline-danger d-none" id="liveUpdateCancel">Cancel</button>
+				</div>
+				<p class="mb-2" id="liveUpdateSummary"></p>
+				<div class="table-responsive d-none" id="liveUpdateTableWrap">
+					<table class="table table-sm align-middle mb-0">
+						<thead><tr><th>What</th><th>Instance</th><th>Players</th><th>State</th><th></th></tr></thead>
+						<tbody id="liveUpdateRows"></tbody>
+					</table>
+				</div>
+			</div>
+		</div>
+	</div>
+</div>
+{% endif %}
+
 {% if can.server_announce or can.server_restart %}
 <div class="row g-3 mt-1">
 	{% if can.server_announce %}
@@ -122,12 +160,60 @@ $(function () {
 		if (window.Live) Live.onStatus(function (s) { showRestart(s.restart); });
 	}
 
+	// Live update: start or cancel it, and show every server and world as it goes (pushed on the live_update topic)
+	if (document.getElementById('liveUpdate')) {
+		var stateColors = { pending: 'secondary', preparing: 'info', starting: 'info', ready: 'primary', waiting: 'warning', draining: 'warning',
+			stopping: 'info', stopped: 'success', failed: 'danger', skipped: 'secondary' };
+		var stateNames = { pending: 'Waiting its turn', preparing: 'Saving property', starting: 'Starting new one', ready: 'New one ready', waiting: 'Waiting for players',
+			draining: 'Moving players', stopping: 'Stopping old one', stopped: 'Done', failed: 'Failed', skipped: 'Skipped' };
+		var phaseColors = { idle: 'secondary', running: 'warning', cancelling: 'warning', done: 'success', failed: 'danger', cancelled: 'secondary' };
+		var kindNames = { database: 'Database migrations', ugc: 'UGC server', auth: 'Auth server', chat: 'Chat server', dashboard: 'Dashboard' };
+		var showLiveUpdate = function (u) {
+			if (!u) return;
+			var running = !!u.running;
+			document.getElementById('liveUpdateStart').classList.toggle('d-none', running);
+			document.getElementById('liveUpdateWarn').closest('.input-group').classList.toggle('d-none', running);
+			document.getElementById('liveUpdateCancel').classList.toggle('d-none', !running || u.phase === 'cancelling');
+			document.getElementById('liveUpdatePhase').innerHTML = u.updateId ? fmt.badge(u.phase, phaseColors[u.phase] || 'secondary') : '';
+			var summary = '';
+			if (u.updateId) {
+				summary = 'Started ' + esc(fmt.unix(u.startedAt)) + (u.by ? ' by ' + esc(u.by) : '') + (u.finishedAt ? ', finished ' + esc(fmt.unix(u.finishedAt)) : '') +
+					'. ' + esc(u.done) + ' of ' + esc(u.total) + ' done' + (u.failed ? ', ' + esc(u.failed) + ' failed' : '') + '.';
+			}
+			if (u.message) summary += (summary ? ' ' : '') + '<strong>' + esc(u.message) + '</strong>';
+			document.getElementById('liveUpdateSummary').innerHTML = summary;
+			var rows = (u.units || []).map(function (unit) {
+				var what = unit.kind === 'world' ? fmt.zone(unit.zone, unit.zoneName) + (unit.clone ? ' <span class="small text-body-secondary">property clone ' + esc(unit.clone) + '</span>' : '') +
+					(unit.isPrivate ? ' ' + fmt.badge('Private', 'warning') : '') : esc(kindNames[unit.kind] || unit.kind);
+				var instance = unit.kind === 'world' ? esc(unit.instance) + (unit.replacement ? ' &rarr; ' + esc(unit.replacement) : '') : '';
+				var players = unit.kind === 'world' ? esc(unit.players) + (unit.moved ? ' <span class="small text-body-secondary">(' + esc(unit.moved) + ' moved)</span>' : '') : '';
+				return '<tr><td>' + what + '</td><td>' + instance + '</td><td>' + players + '</td><td>' + fmt.badge(stateNames[unit.state] || unit.state, stateColors[unit.state] || 'secondary') +
+					'</td><td class="small">' + esc(unit.message || '') + '</td></tr>';
+			});
+			document.getElementById('liveUpdateRows').innerHTML = rows.join('');
+			document.getElementById('liveUpdateTableWrap').classList.toggle('d-none', rows.length === 0);
+		};
+		document.getElementById('liveUpdateStart').addEventListener('click', function () {
+			if (!confirm('Start a live update onto the binaries on disk now? Everyone online is moved to new instances (a short loading screen), and the dashboard restarts at the end.')) return;
+			var warn = document.getElementById('liveUpdateWarn').value;
+			api.action('/api/server/live_update', warn === '' ? {} : { warnSeconds: parseInt(warn, 10) }, 'Live update requested').catch(function () {});
+		});
+		document.getElementById('liveUpdateCancel').addEventListener('click', function () {
+			if (!confirm('Cancel the live update? Nothing new is started; what is under way finishes.')) return;
+			api.action('/api/server/live_update/cancel', {}, 'Cancelling').catch(function () {});
+		});
+		var loadLiveUpdate = function () { api.get('/api/server/live_update').then(function (d) { if (d.success) showLiveUpdate(d.liveUpdate); }).catch(function () {}); };
+		loadLiveUpdate();
+		if (window.Live) Live.onTopic('live_update', showLiveUpdate);
+	}
+
 	// Staff get the detailed list (address, which property a clone is); everyone else the basic one from the status
 	var detailedWorlds = DASH.can('players_view');
 	// Worlds master launched that haven't connected yet, and those shutting down
 	function stateBadge(row) {
 		if (row.state === 'starting') return ' ' + fmt.badge('Starting', 'info');
 		if (row.state === 'stopping') return ' ' + fmt.badge('Shutting down', 'secondary');
+		if (row.state === 'draining') return ' ' + fmt.badge('Moving players', 'warning');
 		return '';
 	}
 	function withPending(data) {