feat: web dashboard and playground work

The NexusDashboard-parity dashboard (dDashboardServer) and everything built on it on the experimental branch:
accounts, characters, properties and moderation tools, permissions shared with in-game slash commands, economy
reports, World 3D and property 3D views with client scenery, scheduled events (features, vanity changes, live
events, announcements, restarts), vanity files and events, the CDClient browser, the message inspector with saved
captures, chat filter tools, community challenges, live ops, the AI moderator helper, and the server-side changes
they need.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dMasterServer/InstanceManager.cpp

@@ -1,5 +1,7 @@
 #include "InstanceManager.h"
 #include <string>
+#include <algorithm>
+#include <chrono>
 #include "Game.h"
 #include "dServer.h"
 #include "Logger.h"
@@ -7,6 +9,7 @@
 #include "CDClientDatabase.h"
 #include "CDClientManager.h"
 #include "CDZoneTableTable.h"
+#include "Database.h"
 #include "MasterPackets.h"
 #include "BitStreamUtils.h"
 #include "ServiceType.h"
@@ -323,19 +326,79 @@ const InstancePtr& InstanceManager::FindPrivateInstance(const std::string& passw
 }
 
 int InstanceManager::GetSoftCap(LWOMAPID mapID) {
+	const auto limit = m_ZoneLimits.find(mapID);
+	if (limit != m_ZoneLimits.end() && limit->second.softCap) return std::min(static_cast<int>(*limit->second.softCap), GetHardCap(mapID));
+
 	const CDZoneTable* zone = CDZoneTableTable::Query(mapID);
 
 	// Default to 8 which is the cap for most worlds.
-	return zone ? zone->population_soft_cap : 8;
+	return std::min(zone ? static_cast<int>(zone->population_soft_cap) : 8, GetHardCap(mapID));
 }
 
 int InstanceManager::GetHardCap(LWOMAPID mapID) {
+	const auto limit = m_ZoneLimits.find(mapID);
+	if (limit != m_ZoneLimits.end() && limit->second.hardCap) return static_cast<int>(*limit->second.hardCap);
+
 	const CDZoneTable* zone = CDZoneTableTable::Query(mapID);
 
 	// Default to 12 which is the cap for most worlds.
 	return zone ? zone->population_hard_cap : 12;
 }
 
+void InstanceManager::LoadZoneLimits() {
+	std::vector<IServerOperations::ZoneLimit> rows;
+	try {
+		rows = Database::Get()->GetZoneLimits();
+	} catch (const std::exception& ex) {
+		LOG("Could not load zone limits, using the client's caps: %s", ex.what());
+		return;
+	}
+	m_ZoneLimits.clear();
+	for (auto& row : rows) {
+		if (row.zoneId == 0) continue; // character selection is never capped
+		LOG("Zone %u: soft cap %s, hard cap %s, %u spare instance(s)", row.zoneId, row.softCap ? std::to_string(*row.softCap).c_str() : "default",
+			row.hardCap ? std::to_string(*row.hardCap).c_str() : "default", row.spareInstances);
+		m_ZoneLimits[row.zoneId] = std::move(row);
+	}
+	for (const auto& instance : m_Instances) {
+		if (!instance || instance->GetIsPrivate() || instance->GetMapID() == 0) continue;
+		instance->SetCaps(GetSoftCap(instance->GetMapID()), GetHardCap(instance->GetMapID()));
+	}
+}
+
+void InstanceManager::KeepSpareInstances() {
+	if (m_IsShuttingDown) return;
+	const auto now = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
+	for (const auto& [mapID, limit] : m_ZoneLimits) {
+		if (limit.spareInstances == 0) continue;
+		// Follow the spare we started last: one that stops soon after starting counts as a failure and delays the next
+		auto& backoff = m_SpareBackoff[mapID];
+		if (const auto watched = backoff.Watched()) {
+			const auto it = std::find_if(m_Instances.begin(), m_Instances.end(), [&](const InstancePtr& instance) {
+				return instance && instance->GetMapID() == mapID && instance->GetInstanceID() == watched && !instance->GetShutdownComplete();
+			});
+			if (it == m_Instances.end()) {
+				backoff.Lost(now);
+				LOG("Spare instance %u of zone %u stopped soon after starting; waiting %lld s before starting another", watched, mapID,
+					static_cast<long long>(SpareBackoff::Delay(backoff.Failures())));
+			} else {
+				backoff.Running((*it)->GetIsReady(), now);
+			}
+		}
+		if (!backoff.CanStart(now)) continue;
+		uint32_t withRoom = 0;
+		for (const auto& instance : m_Instances) {
+			// One still starting counts: it has room, and starting another would double up
+			if (instance && instance->GetMapID() == mapID && instance->GetCloneID() == 0 && !instance->GetIsPrivate() &&
+				!instance->GetShutdownComplete() && !instance->GetIsShuttingDown() && !instance->IsFull(false)) withRoom++;
+		}
+		if (withRoom >= limit.spareInstances) continue;
+		LOG("Zone %u has %u instance(s) with room and should have %u; starting one", mapID, withRoom, limit.spareInstances);
+		const auto& started = CreateInstance(mapID, 0);
+		if (started) backoff.Started(started->GetInstanceID());
+	}
+}
+
 void InstanceManager::PruneUnreadyInstances() {
 	for (int i = static_cast<int>(m_Instances.size()) - 1; i >= 0; i--) {
 		if (!m_Instances[i]->GetIsReady()) m_Instances.erase(m_Instances.cbegin() + i);

dMasterServer/InstanceManager.h

@@ -1,7 +1,10 @@
 #pragma once
 #include <algorithm>
+#include "SpareBackoff.h"
+#include <map>
 #include <vector>
 #include "dCommonVars.h"
+#include "IServerOperations.h"
 #include "RakNetTypes.h"
 #include "dZMCommon.h"
 #include "Logger.h"
@@ -25,6 +28,7 @@ public:
 		m_ZoneID = LWOZONEID(mapID, instanceID, cloneID);
 		m_MaxClientsSoftCap = softCap;
 		m_MaxClientsHardCap = hardCap;
+		m_StartedHardCap = hardCap;
 		m_CurrentClientCount = 0;
 		m_IsPrivate = isPrivate;
 		m_Password = password;
@@ -63,6 +67,12 @@ public:
 	int GetSoftCap() const { return m_MaxClientsSoftCap; }
 	int GetCurrentClientCount() const { return m_CurrentClientCount; }
 
+	// New caps for a running instance: the hard cap can't go above what its world server was started with
+	void SetCaps(int softCap, int hardCap) {
+		m_MaxClientsHardCap = std::min(hardCap, m_StartedHardCap);
+		m_MaxClientsSoftCap = std::min(softCap, m_MaxClientsHardCap);
+	}
+
 	void SetAffirmationTimeout(const uint32_t value) { m_AffirmationTimeout = value; }
 	uint32_t GetAffirmationTimeout() const { return m_AffirmationTimeout; }
 
@@ -90,6 +100,7 @@ private:
 	LWOZONEID m_ZoneID{};
 	int m_MaxClientsSoftCap{};
 	int m_MaxClientsHardCap{};
+	int m_StartedHardCap{}; // the -maxclients its world server was started with
 	int m_CurrentClientCount{};
 	std::vector<Player> m_Players{};
 	SystemAddress m_SysAddr{};
@@ -147,6 +158,15 @@ public:
 	// Start a new public instance of a zone even when one with room is running (instance migrations)
 	const InstancePtr& StartNewInstance(LWOMAPID mapID, LWOCLONEID cloneID) { return CreateInstance(mapID, cloneID); }
 
+	/**
+	 * Player caps and spare instances set per zone on the dashboard (zone_limits). Caps apply to public instances
+	 * started from now on, and running ones get the new caps too (a hard cap only as high as they were started with).
+	 */
+	void LoadZoneLimits();
+
+	// Start an instance of each zone that has fewer instances with room than its spare_instances (one per call)
+	void KeepSpareInstances();
+
 private:
 	std::string mExternalIP;
 	std::vector<std::unique_ptr<Instance>> m_Instances;
@@ -158,6 +178,9 @@ private:
 	 */
 	bool m_IsShuttingDown = false;
 
+	std::map<LWOMAPID, IServerOperations::ZoneLimit> m_ZoneLimits;
+	std::map<LWOMAPID, SpareBackoff> m_SpareBackoff;
+
 	//Private functions:
 	int GetSoftCap(LWOMAPID mapID);
 	int GetHardCap(LWOMAPID mapID);

dMasterServer/MasterServer.cpp

@@ -1,8 +1,11 @@
+#include "PlayerAction.h"
+#include "DashboardMessages.h"
 #include <chrono>
 #include <cstdlib>
 #include <ctime>
 #include <iostream>
 #include <map>
+#include <set>
 #include <string>
 #include <thread>
 #include <fstream>
@@ -16,6 +19,7 @@
 #include "CDClientManager.h"
 #include "Database.h"
 #include "MigrationRunner.h"
+#include "ConfigSync.h"
 #include "Diagnostics.h"
 #include "dCommonVars.h"
 #include "dConfig.h"
@@ -73,6 +77,47 @@ void HandlePacket(Packet* packet);
 std::map<uint32_t, std::string> activeSessions;
 SystemAddress authServerMasterPeerSysAddr;
 SystemAddress chatServerMasterPeerSysAddr;
+SystemAddress dashboardServerMasterPeerSysAddr;
+
+namespace {
+	// Dashboard player actions waiting for world servers to answer
+	struct PendingPlayerAction {
+		ePlayerAction action{};
+		std::set<SystemAddress> waitingOn;
+		uint32_t affected{};
+		std::chrono::steady_clock::time_point deadline;
+	};
+	constexpr auto PLAYER_ACTION_TIMEOUT = std::chrono::seconds(3);
+	std::map<uint32_t, PendingPlayerAction> g_PendingPlayerActions;
+
+	void FinishPlayerAction(uint32_t requestId, bool timedOut) {
+		const auto it = g_PendingPlayerActions.find(requestId);
+		if (it == g_PendingPlayerActions.end()) return;
+
+		if (dashboardServerMasterPeerSysAddr != UNASSIGNED_SYSTEM_ADDRESS) {
+			PlayerActionResult result;
+			result.requestId = requestId;
+			result.action = it->second.action;
+			result.affected = it->second.affected;
+			result.timedOut = timedOut;
+
+			CBITSTREAM;
+			BitStreamUtils::WriteHeader(bitStream, ServiceType::MASTER, MessageType::Master::PLAYER_ACTION_RESULT);
+			result.Serialize(bitStream);
+			Game::server->Send(bitStream, dashboardServerMasterPeerSysAddr, false);
+		}
+		g_PendingPlayerActions.erase(it);
+	}
+
+	void CheckPlayerActionTimeouts() {
+		const auto now = std::chrono::steady_clock::now();
+		std::vector<uint32_t> expired;
+		for (const auto& [id, pending] : g_PendingPlayerActions) {
+			if (now >= pending.deadline) expired.push_back(id);
+		}
+		for (const auto id : expired) FinishPlayerAction(id, true);
+	}
+}
 
 int GenerateBCryptPassword(const std::string& password, const int workFactor, char salt[BCRYPT_HASHSIZE], char hash[BCRYPT_HASHSIZE]) {
 	int32_t bcryptState = ::bcrypt_gensalt(workFactor, salt);
@@ -166,6 +211,8 @@ int main(int argc, char** argv) {
 	}
 
 	MigrationRunner::RunMigrations();
+	// Settings edited on the dashboard (server_config table) are layered over the files from here on
+	Game::config->SetDatabaseSync(ConfigSync::Sync);
 	Database::Get()->Commit();
 	const auto resServerPath = BinaryPathFinder::GetBinaryDir() / "resServer";
 	std::filesystem::create_directories(resServerPath);
@@ -313,7 +360,8 @@ int main(int argc, char** argv) {
 
 		//Create account
 		try {
-			Database::Get()->InsertNewAccount(username, std::string(hash, BCRYPT_HASHSIZE));
+			// Accounts created from the command line are server operators
+			Database::Get()->InsertNewAccount(username, std::string(hash, BCRYPT_HASHSIZE), eGameMasterLevel::OPERATOR);
 		} catch (std::exception& e) {
 			LOG("A SQL error occurred!:\n %s", e.what());
 			return EXIT_FAILURE;
@@ -382,6 +430,7 @@ int main(int argc, char** argv) {
 			if (instance && instance->GetIsReady() && !instance->GetShutdownComplete()) Game::server->Send(bitStream, instance->GetSysAddr(), false);
 		}
 	});
+	Game::im->LoadZoneLimits();
 
 	//Depending on the config, start up servers:
 	if (Game::config->GetValue("prestart_servers") != "0") {
@@ -392,6 +441,11 @@ int main(int argc, char** argv) {
 		StartAuthServer();
 	}
 
+	// Start web dashboard if enabled
+	if (Game::config->GetValue("enable_dashboard") == "1") {
+		StartDashboardServer();
+	}
+
 	auto t = std::chrono::high_resolution_clock::now();
 	Packet* packet = nullptr;
 	constexpr uint32_t logFlushTime = 15 * masterFramerate;
@@ -401,6 +455,8 @@ int main(int argc, char** argv) {
 	uint32_t framesSinceLastFlush = 0;
 	uint32_t framesSinceLastSQLPing = 0;
 	uint32_t framesSinceKillUniverseCommand = 0;
+	constexpr uint32_t spareCheckTime = 10 * masterFramerate;
+	uint32_t framesSinceSpareCheck = 0;
 
 	Game::logger->Flush();
 	while (!Game::ShouldShutdown()) {
@@ -415,6 +471,13 @@ int main(int argc, char** argv) {
 		}
 
 		MigrationCoordinator::Update();
+		CheckPlayerActionTimeouts();
+
+		// Spare instances for busy zones (zone_limits), checked every few seconds
+		if (framesSinceSpareCheck >= spareCheckTime) {
+			Game::im->KeepSpareInstances();
+			framesSinceSpareCheck = 0;
+		} else framesSinceSpareCheck++;
 
 		//Push our log every 15s:
 		if (framesSinceLastFlush >= logFlushTime) {
@@ -479,6 +542,7 @@ int main(int argc, char** argv) {
 			}
 
 			if (instance->GetShutdownComplete()) {
+				MigrationCoordinator::OnInstanceGone(*instance);
 				Game::im->RemoveInstance(instance);
 			}
 		}
@@ -497,49 +561,65 @@ int main(int argc, char** argv) {
 
 void HandlePacket(Packet* packet) {
 	if (packet->length < 1) return;
-	if (packet->data[0] == ID_DISCONNECTION_NOTIFICATION) {
-		LOG("A server has disconnected");
+	if (packet->data[0] == ID_DISCONNECTION_NOTIFICATION || packet->data[0] == ID_CONNECTION_LOST) {
+		const bool intentional = packet->data[0] == ID_DISCONNECTION_NOTIFICATION;
+		LOG("A server has %s", intentional ? "disconnected" : "lost the connection");
 
-		//Since this disconnection is intentional, we'll just delete it as
-		//we'll start a new one anyway if needed:
 		const auto& instance =
 			Game::im->GetInstanceBySysAddr(packet->systemAddress);
 		if (instance) {
 			LOG("Actually disconnected from zone %i clone %i instance %i port %i", instance->GetMapID(), instance->GetCloneID(), instance->GetInstanceID(), instance->GetPort());
+
+			if (dashboardServerMasterPeerSysAddr != UNASSIGNED_SYSTEM_ADDRESS &&
+				packet->systemAddress != dashboardServerMasterPeerSysAddr) {
+				CBITSTREAM;
+				BitStreamUtils::WriteHeader(bitStream, ServiceType::MASTER, MessageType::Master::SHUTDOWN_RESPONSE);
+				bitStream.Write(instance->GetMapID());
+				bitStream.Write(instance->GetInstanceID());
+				Game::server->Send(bitStream, dashboardServerMasterPeerSysAddr, false);
+			}
+
 			MigrationCoordinator::OnInstanceGone(*instance);
-			Game::im->RemoveInstance(instance); //Delete the old
+			Game::im->RemoveInstance(instance);
 		}
 
 		if (packet->systemAddress == chatServerMasterPeerSysAddr) {
 			chatServerMasterPeerSysAddr = UNASSIGNED_SYSTEM_ADDRESS;
+
+			if (dashboardServerMasterPeerSysAddr != UNASSIGNED_SYSTEM_ADDRESS) {
+				CBITSTREAM;
+				BitStreamUtils::WriteHeader(bitStream, ServiceType::MASTER, MessageType::Master::SERVER_INFO);
+				bitStream.Write<uint32_t>(0);
+				bitStream.Write<uint32_t>(0);
+				bitStream.Write<uint32_t>(0);
+				bitStream.Write(ServiceType::CHAT);
+				bitStream.Write(LUString("offline"));
+				Game::server->Send(bitStream, dashboardServerMasterPeerSysAddr, false);
+			}
+
 			StartChatServer();
 		}
 
 		if (packet->systemAddress == authServerMasterPeerSysAddr) {
 			authServerMasterPeerSysAddr = UNASSIGNED_SYSTEM_ADDRESS;
+
+			if (dashboardServerMasterPeerSysAddr != UNASSIGNED_SYSTEM_ADDRESS) {
+				CBITSTREAM;
+				BitStreamUtils::WriteHeader(bitStream, ServiceType::MASTER, MessageType::Master::SERVER_INFO);
+				bitStream.Write<uint32_t>(0);
+				bitStream.Write<uint32_t>(0);
+				bitStream.Write<uint32_t>(0);
+				bitStream.Write(ServiceType::AUTH);
+				bitStream.Write(LUString("offline"));
+				Game::server->Send(bitStream, dashboardServerMasterPeerSysAddr, false);
+			}
+
 			StartAuthServer();
 		}
-	}
 
-	if (packet->data[0] == ID_CONNECTION_LOST) {
-		LOG("A server has lost the connection");
-
-		const auto& instance =
-			Game::im->GetInstanceBySysAddr(packet->systemAddress);
-		if (instance) {
-			LWOZONEID zoneID = instance->GetZoneID(); //Get the zoneID so we can recreate a server
-			MigrationCoordinator::OnInstanceGone(*instance);
-			Game::im->RemoveInstance(instance); //Delete the old
-		}
-
-		if (packet->systemAddress == chatServerMasterPeerSysAddr) {
-			chatServerMasterPeerSysAddr = UNASSIGNED_SYSTEM_ADDRESS;
-			StartChatServer();
-		}
-
-		if (packet->systemAddress == authServerMasterPeerSysAddr) {
-			authServerMasterPeerSysAddr = UNASSIGNED_SYSTEM_ADDRESS;
-			StartAuthServer();
+		if (packet->systemAddress == dashboardServerMasterPeerSysAddr) {
+			dashboardServerMasterPeerSysAddr = UNASSIGNED_SYSTEM_ADDRESS;
+			StartDashboardServer();
 		}
 	}
 
@@ -622,9 +702,22 @@ void HandlePacket(Packet* packet) {
 			case ServiceType::AUTH:
 				authServerMasterPeerSysAddr = packet->systemAddress;
 				break;
-			default:
-				// We just ignore any other server type
+			case ServiceType::DASHBOARD:
+				dashboardServerMasterPeerSysAddr = packet->systemAddress;
 				break;
+			default:
+				break;
+			}
+
+			if (theirServerType != ServiceType::DASHBOARD && dashboardServerMasterPeerSysAddr != UNASSIGNED_SYSTEM_ADDRESS) {
+				CBITSTREAM;
+				BitStreamUtils::WriteHeader(bitStream, ServiceType::MASTER, MessageType::Master::SERVER_INFO);
+				bitStream.Write(theirPort);
+				bitStream.Write(theirZoneID);
+				bitStream.Write(theirInstanceID);
+				bitStream.Write(theirServerType);
+				bitStream.Write(theirIP);
+				Game::server->Send(bitStream, dashboardServerMasterPeerSysAddr, false);
 			}
 
 			LOG("Received %s server info, instance: %i port: %i", StringifiedEnum::ToString(theirServerType).data(), theirInstanceID, theirPort);
@@ -692,6 +785,14 @@ void HandlePacket(Packet* packet) {
 			} else {
 				LOG("Instance missing? What?");
 			}
+
+			if (dashboardServerMasterPeerSysAddr != UNASSIGNED_SYSTEM_ADDRESS) {
+				CBITSTREAM;
+				BitStreamUtils::WriteHeader(bitStream, ServiceType::MASTER, MessageType::Master::PLAYER_ADDED);
+				bitStream.Write(theirZoneID);
+				bitStream.Write(theirInstanceID);
+				Game::server->Send(bitStream, dashboardServerMasterPeerSysAddr, false);
+			}
 			break;
 		}
 
@@ -709,6 +810,14 @@ void HandlePacket(Packet* packet) {
 			if (instance) {
 				instance->RemovePlayer(Player());
 			}
+
+			if (dashboardServerMasterPeerSysAddr != UNASSIGNED_SYSTEM_ADDRESS) {
+				CBITSTREAM;
+				BitStreamUtils::WriteHeader(bitStream, ServiceType::MASTER, MessageType::Master::PLAYER_REMOVED);
+				bitStream.Write(theirZoneID);
+				bitStream.Write(theirInstanceID);
+				Game::server->Send(bitStream, dashboardServerMasterPeerSysAddr, false);
+			}
 			break;
 		}
 
@@ -792,6 +901,18 @@ void HandlePacket(Packet* packet) {
 
 			LOG("Ready zone %i", zoneID);
 			Game::im->ReadyInstance(instance);
+
+			if (dashboardServerMasterPeerSysAddr != UNASSIGNED_SYSTEM_ADDRESS) {
+				CBITSTREAM;
+				BitStreamUtils::WriteHeader(bitStream, ServiceType::MASTER, MessageType::Master::WORLD_READY);
+				bitStream.Write(zoneID);
+				bitStream.Write(instanceID);
+				bitStream.Write(instance->GetCloneID());
+				bitStream.Write(LUString(instance->GetIP()));
+				bitStream.Write(instance->GetPort());
+				bitStream.Write<uint8_t>(instance->GetIsPrivate() ? 1 : 0);
+				Game::server->Send(bitStream, dashboardServerMasterPeerSysAddr, false);
+			}
 			break;
 		}
 
@@ -876,6 +997,186 @@ void HandlePacket(Packet* packet) {
 			break;
 		}
 
+		case MessageType::Master::PLAYER_ACTION: {
+			// Only the dashboard may ask worlds to act on players
+			if (packet->systemAddress != dashboardServerMasterPeerSysAddr) {
+				LOG("Ignoring player action from a server that is not the dashboard");
+				break;
+			}
+
+			CINSTREAM_SKIP_HEADER;
+			PlayerActionRequest request;
+			if (!request.Deserialize(inStream)) break;
+
+			PendingPlayerAction pending;
+			pending.action = request.action;
+			pending.deadline = std::chrono::steady_clock::now() + PLAYER_ACTION_TIMEOUT;
+
+			CBITSTREAM;
+			BitStreamUtils::WriteHeader(bitStream, ServiceType::MASTER, MessageType::Master::PLAYER_ACTION);
+			request.Serialize(bitStream);
+			for (const auto& instance : Game::im->GetInstances()) {
+				if (!instance || !instance->GetIsReady() || instance->GetIsShuttingDown()) continue;
+				Game::server->Send(bitStream, instance->GetSysAddr(), false);
+				pending.waitingOn.insert(instance->GetSysAddr());
+			}
+			// The chat server checks web chat with the same filter: it reloads too, but doesn't answer
+			if (request.action == ePlayerAction::RELOAD_CHAT_FILTER && chatServerMasterPeerSysAddr != UNASSIGNED_SYSTEM_ADDRESS) {
+				Game::server->Send(bitStream, chatServerMasterPeerSysAddr, false);
+			}
+
+			LOG("Dashboard player action %i (request %u) sent to %zu world(s)", static_cast<int>(request.action), request.requestId, pending.waitingOn.size());
+			const bool noWorlds = pending.waitingOn.empty();
+			g_PendingPlayerActions[request.requestId] = std::move(pending);
+			if (noWorlds) FinishPlayerAction(request.requestId, false);
+			break;
+		}
+
+		case MessageType::Master::PLAYER_ACTION_RESULT: {
+			CINSTREAM_SKIP_HEADER;
+			PlayerActionResult result;
+			if (!result.Deserialize(inStream)) break;
+
+			const auto it = g_PendingPlayerActions.find(result.requestId);
+			if (it == g_PendingPlayerActions.end()) break;
+			it->second.affected += result.affected;
+			it->second.waitingOn.erase(packet->systemAddress);
+			if (it->second.waitingOn.empty()) FinishPlayerAction(result.requestId, false);
+			break;
+		}
+
+		case MessageType::Master::PLAYER_POSITIONS: {
+			if (dashboardServerMasterPeerSysAddr == UNASSIGNED_SYSTEM_ADDRESS || !Game::im->GetInstanceBySysAddr(packet->systemAddress)) break;
+			RakNet::BitStream forward(packet->data, packet->length, false);
+			Game::server->Send(forward, dashboardServerMasterPeerSysAddr, false);
+			break;
+		}
+
+		case MessageType::Master::ANNOUNCE: {
+			if (packet->systemAddress != dashboardServerMasterPeerSysAddr) {
+				LOG("Ignoring announcement from a server that is not the dashboard");
+				break;
+			}
+			CINSTREAM_SKIP_HEADER;
+			Announcement announcement;
+			if (!announcement.Deserialize(inStream)) break;
+			RakNet::BitStream forward(packet->data, packet->length, false);
+			uint32_t worlds = 0;
+			for (const auto& instance : Game::im->GetInstances()) {
+				if (!instance || !instance->GetIsReady() || instance->GetIsShuttingDown() || !announcement.ShownIn(instance->GetMapID())) continue;
+				Game::server->Send(forward, instance->GetSysAddr(), false);
+				worlds++;
+			}
+			LOG("Dashboard announcement sent to %u world(s)", worlds);
+			break;
+		}
+
+		case MessageType::Master::CONFIG_RELOAD: {
+			if (packet->systemAddress != dashboardServerMasterPeerSysAddr) {
+				LOG("Ignoring config reload from a server that is not the dashboard");
+				break;
+			}
+			LOG("Reloading settings (changed on the dashboard)");
+			Game::config->ReloadConfig();
+			Game::im->LoadZoneLimits();
+			// Everyone else: auth, chat and every world
+			RakNet::BitStream forward(packet->data, packet->length, false);
+			for (const auto& peer : { authServerMasterPeerSysAddr, chatServerMasterPeerSysAddr }) {
+				if (peer != UNASSIGNED_SYSTEM_ADDRESS) Game::server->Send(forward, peer, false);
+			}
+			for (const auto& instance : Game::im->GetInstances()) {
+				if (instance && instance->GetIsReady()) Game::server->Send(forward, instance->GetSysAddr(), false);
+			}
+			break;
+		}
+
+		case MessageType::Master::INSTANCE_SHUTDOWN: {
+			if (packet->systemAddress != dashboardServerMasterPeerSysAddr) {
+				LOG("Ignoring instance shutdown from a server that is not the dashboard");
+				break;
+			}
+			CINSTREAM_SKIP_HEADER;
+			uint32_t zoneId = 0, instanceId = 0;
+			if (!inStream.Read(zoneId) || !inStream.Read(instanceId)) break;
+			const auto& instance = Game::im->FindInstanceWithPrivate(static_cast<LWOMAPID>(zoneId), static_cast<LWOINSTANCEID>(instanceId));
+			if (!instance) {
+				LOG("Dashboard asked to shut down zone %u instance %u, which isn't running", zoneId, instanceId);
+				break;
+			}
+			LOG("Shutting down zone %u instance %u (from the dashboard)", zoneId, instanceId);
+			instance->Shutdown();
+			break;
+		}
+
+		case MessageType::Master::DASHBOARD_SHUTDOWN: {
+			if (packet->systemAddress != dashboardServerMasterPeerSysAddr) {
+				LOG("Ignoring shutdown request from a server that is not the dashboard");
+				break;
+			}
+			LOG("Shutdown requested from the dashboard (scheduled restart)");
+			Game::lastSignal = -1;
+			break;
+		}
+
+		case MessageType::Master::DATA_CHANGED: {
+			// Only world servers report game writes; pass them on unchanged
+			if (dashboardServerMasterPeerSysAddr == UNASSIGNED_SYSTEM_ADDRESS || !Game::im->GetInstanceBySysAddr(packet->systemAddress)) break;
+			RakNet::BitStream forward(packet->data, packet->length, false);
+			Game::server->Send(forward, dashboardServerMasterPeerSysAddr, false);
+			break;
+		}
+
+		case MessageType::Master::MESSAGE_CAPTURE_CONTROL: {
+			// Only the dashboard starts message captures; every world gets it, and the one with the player acts on it
+			if (packet->systemAddress != dashboardServerMasterPeerSysAddr) {
+				LOG("Ignoring a message capture request from a server that is not the dashboard");
+				break;
+			}
+			RakNet::BitStream forward(packet->data, packet->length, false);
+			for (const auto& instance : Game::im->GetInstances()) {
+				if (instance && instance->GetIsReady() && !instance->GetIsShuttingDown()) Game::server->Send(forward, instance->GetSysAddr(), false);
+			}
+			break;
+		}
+
+		case MessageType::Master::MESSAGE_CAPTURE_DATA: {
+			if (dashboardServerMasterPeerSysAddr == UNASSIGNED_SYSTEM_ADDRESS || !Game::im->GetInstanceBySysAddr(packet->systemAddress)) break;
+			RakNet::BitStream forward(packet->data, packet->length, false);
+			Game::server->Send(forward, dashboardServerMasterPeerSysAddr, false);
+			break;
+		}
+
+		case MessageType::Master::REQUEST_SERVER_LIST: {
+			LOG("Dashboard requested server list");
+
+			CBITSTREAM;
+			BitStreamUtils::WriteHeader(bitStream, ServiceType::MASTER, MessageType::Master::SERVER_LIST_RESPONSE);
+
+			bitStream.Write<uint8_t>(authServerMasterPeerSysAddr != UNASSIGNED_SYSTEM_ADDRESS ? 1 : 0);
+			bitStream.Write<uint8_t>(chatServerMasterPeerSysAddr != UNASSIGNED_SYSTEM_ADDRESS ? 1 : 0);
+
+			const auto& instances = Game::im->GetInstances();
+			uint32_t instanceCount = 0;
+			for (const auto& inst : instances) {
+				if (inst && inst->GetIsReady() && !inst->GetIsShuttingDown()) instanceCount++;
+			}
+			bitStream.Write(instanceCount);
+
+			for (const auto& inst : instances) {
+				if (!inst || !inst->GetIsReady() || inst->GetIsShuttingDown()) continue;
+				bitStream.Write(inst->GetMapID());
+				bitStream.Write(inst->GetInstanceID());
+				bitStream.Write(inst->GetCloneID());
+				bitStream.Write<uint32_t>(inst->GetCurrentClientCount());
+				bitStream.Write(LUString(inst->GetIP()));
+				bitStream.Write(inst->GetPort());
+				bitStream.Write<uint8_t>(inst->GetIsPrivate() ? 1 : 0);
+			}
+
+			Game::server->Send(bitStream, packet->systemAddress, false);
+			break;
+		}
+
 		default:
 			LOG("Unknown master packet ID from server: %i", packet->data[3]);
 		}
@@ -914,11 +1215,11 @@ int ShutdownSequence(int32_t signal) {
 		instance->SetIsShuttingDown(true);
 	}
 
-	LOG("Attempting to shutdown instances, max 60 seconds...");
+	LOG("Attempting to shutdown instances, max 10 seconds...");
 
 	auto t = std::chrono::high_resolution_clock::now();
 	uint32_t framesSinceShutdownStart = 0;
-	constexpr uint32_t maxShutdownTime = 60 * mediumFramerate;
+	constexpr uint32_t maxShutdownTime = 10 * mediumFramerate;
 	bool allInstancesShutdown = false;
 	Packet* packet = nullptr;
 	while (true) {
@@ -941,7 +1242,10 @@ int ShutdownSequence(int32_t signal) {
 			}
 		}
 
-		if (allInstancesShutdown && authServerMasterPeerSysAddr == UNASSIGNED_SYSTEM_ADDRESS && chatServerMasterPeerSysAddr == UNASSIGNED_SYSTEM_ADDRESS) {
+		if (allInstancesShutdown && \
+			authServerMasterPeerSysAddr == UNASSIGNED_SYSTEM_ADDRESS && \
+			chatServerMasterPeerSysAddr == UNASSIGNED_SYSTEM_ADDRESS && \
+			dashboardServerMasterPeerSysAddr == UNASSIGNED_SYSTEM_ADDRESS) {
 			LOG("Finished shutting down MasterServer!");
 			break;
 		}
@@ -953,6 +1257,26 @@ int ShutdownSequence(int32_t signal) {
 
 		if (framesSinceShutdownStart == maxShutdownTime) {
 			LOG("Finished shutting down by timeout!");
+			// log what we were waiting on: worlds, chat, auth, dashboard, etc
+			if (authServerMasterPeerSysAddr != UNASSIGNED_SYSTEM_ADDRESS) {
+				LOG("Auth server did not shutdown in time");
+			}
+			if (chatServerMasterPeerSysAddr != UNASSIGNED_SYSTEM_ADDRESS) {
+				LOG("Chat server did not shutdown in time");
+			}
+			if (dashboardServerMasterPeerSysAddr != UNASSIGNED_SYSTEM_ADDRESS) {
+				LOG("Dashboard server did not shutdown in time");
+			}
+			for (const auto& instance : Game::im->GetInstances()) {
+				if (instance == nullptr) {
+					continue;
+				}
+
+				if (!instance->GetShutdownComplete()) {
+					LOG("Instance zone %i clone %i instance %i port %i did not shutdown in time", instance->GetMapID(), instance->GetCloneID(), instance->GetInstanceID(), instance->GetPort());
+				}
+			}
+
 			break;
 		}
 	}

dMasterServer/SpareBackoff.h

@@ -0,0 +1,60 @@
+#pragma once
+
+#include <algorithm>
+#include <cstdint>
+
+/**
+ * When to start another spare instance of a zone (InstanceManager::KeepSpareInstances). A spare that stops before it
+ * has run for STABLE seconds (a crash at start or soon after, a missing file, a bad port) counts as a failure, and each
+ * failure in a row doubles the wait before the next try, up to half an hour, so a broken zone isn't restarted every few
+ * seconds forever. A spare that keeps running for STABLE seconds resets it. Times are seconds on any steady clock,
+ * passed in so it can be tested.
+ */
+class SpareBackoff {
+public:
+	static constexpr int64_t BASE_DELAY = 10;
+	static constexpr int64_t MAX_DELAY = 30 * 60;
+	static constexpr int64_t STABLE = 5 * 60;
+
+	// Whether a new spare may be started now: none of ours is still starting, and any wait has passed
+	bool CanStart(int64_t now) const { return (m_Watched == 0 || m_ReadySince != 0) && now >= m_NextAttempt; }
+
+	void Started(uint32_t instanceId) {
+		m_Watched = instanceId;
+		m_ReadySince = 0;
+	}
+
+	// The instance we started is still there; ready says whether it has finished starting
+	void Running(bool ready, int64_t now) {
+		if (!ready) return;
+		if (m_ReadySince == 0) m_ReadySince = now;
+		if (now - m_ReadySince >= STABLE) {
+			m_Watched = 0;
+			m_ReadySince = 0;
+			m_Failures = 0;
+		}
+	}
+
+	// The instance we started went away before running for STABLE seconds
+	void Lost(int64_t now) {
+		m_Watched = 0;
+		m_ReadySince = 0;
+		m_Failures++;
+		m_NextAttempt = now + Delay(m_Failures);
+	}
+
+	uint32_t Watched() const { return m_Watched; }
+	uint32_t Failures() const { return m_Failures; }
+
+	static int64_t Delay(uint32_t failures) {
+		if (failures == 0) return 0;
+		const auto shift = std::min<uint32_t>(failures - 1, 20);
+		return std::min<int64_t>(BASE_DELAY << shift, MAX_DELAY);
+	}
+
+private:
+	uint32_t m_Watched{};
+	int64_t m_ReadySince{};
+	uint32_t m_Failures{};
+	int64_t m_NextAttempt{};
+};

dMasterServer/Start.cpp

@@ -107,6 +107,42 @@ uint32_t StartAuthServer() {
 	return auth_pid;
 }
 
+uint32_t StartDashboardServer() {
+	if (Game::ShouldShutdown()) {
+		LOG("Currently shutting down.  DashboardServer will not be restarted.");
+		return 0;
+	}
+	auto web_path = BinaryPathFinder::GetBinaryDir() / "DashboardServer";
+#ifdef _WIN32
+	web_path.replace_extension(".exe");
+	auto web_startup = startup;
+	auto web_info = PROCESS_INFORMATION{};
+	if (!CreateProcessW(web_path.wstring().data(), web_path.wstring().data(),
+						nullptr, nullptr, false, 0, nullptr, nullptr,
+						&web_startup, &web_info))
+	{
+		LOG("Failed to launch DashboardServer");
+		return 0;
+	}
+
+	// get pid and close unused handles
+	auto web_pid = web_info.dwProcessId;
+	CloseHandle(web_info.hProcess);
+	CloseHandle(web_info.hThread);
+#else // *nix systems
+	const auto web_pid = fork();
+	if (web_pid < 0) {
+		LOG("Failed to launch DashboardServer");
+		return 0;
+	} else if (web_pid == 0) {
+		// We are the child process
+		execl(web_path.string().c_str(), web_path.string().c_str(), nullptr);
+	}
+#endif
+	LOG("DashboardServer PID is %d", web_pid);
+	return web_pid;
+}
+
 uint32_t StartWorldServer(LWOMAPID mapID, uint16_t port, LWOINSTANCEID lastInstanceID, int maxPlayers, LWOCLONEID cloneID) {
 	auto world_path = BinaryPathFinder::GetBinaryDir() / "WorldServer";
 #ifdef _WIN32

dMasterServer/Start.h

@@ -3,4 +3,5 @@
 
 uint32_t StartAuthServer();
 uint32_t StartChatServer();
+uint32_t StartDashboardServer();
 uint32_t StartWorldServer(LWOMAPID mapID, uint16_t port, LWOINSTANCEID lastInstanceID, int maxPlayers, LWOCLONEID cloneID);