feat: live updates move every server onto a new build without a restart

With new binaries in place, master moves everything onto them while the
server keeps running (the dashboard's Live update, /liveupdate or SIGUSR2 to
master):

- database migrations of the new build first; a failure stops there
- UGC finishes the jobs it is running (queued rows stay pending), auth
  restarts, chat hands its teams to master for the next chat server; master
  starts the new processes and retries ones that don't come back
- once the new chat server is up (CHAT_SERVER_READY) every world connects at
  once and sends its players again (LoginSessionNotify resync, no login logged)
- every world instance is replaced with an instance migration: public worlds
  and private ones (same password) at once, properties after the old instance
  saved and froze the property (MIGRATE_PREPARE: no building, claiming or
  saving there any more), activity zones and character select once their
  players left or after a wait; empty instances just stop, zones in
  prestart_worlds get a new one first
- the dashboard restarts last and picks the status up again

Players land where they stood (position carried in CarriedPlayerState, also on
properties and Moon Base). Draining instances get no new players
(InstanceMigration::AcceptsNewPlayers) and show as "Moving players" in the
world list. The order lives in LiveUpdateMachine.h without master state and is
unit tested; master's glue is LiveUpdateCoordinator. Master itself is not
replaced. Message IDs are appended only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dChatServer/ChatPacketHandler.cpp

@@ -32,6 +32,24 @@ void ChatPacketHandler::SendRouted(const LWOOBJID target, const SystemAddress& w
 	Game::server->Send(bitStream, world, broadcast);
 }
 
+void ChatPacketHandler::LoadFriends(PlayerData& player) {
+	player.friends.clear();
+	player.countOfBestFriends = 0;
+	for (const auto& friendData : Database::Get()->GetFriendsList(player.playerID)) {
+		FriendData fd;
+		fd.isFTP = false;
+		fd.friendID = friendData.friendID;
+		GeneralUtils::SetBit(fd.friendID, eObjectBits::CHARACTER);
+		fd.isBestFriend = friendData.isBestFriend;
+		if (fd.isBestFriend) player.countOfBestFriends += 1;
+		fd.friendName = friendData.friendName;
+		const auto& online = Game::playerContainer.GetPlayerData(fd.friendID);
+		fd.isOnline = static_cast<bool>(online);
+		fd.zoneID = online ? online.zoneID : LWOZONEID();
+		player.friends.push_back(fd);
+	}
+}
+
 void ChatPacketHandler::HandleFriendlistRequest(const ChatPackets::GetFriendsList& request, const SystemAddress& sysAddr) {
 	//Get from the packet which player we want to do something with:
 	const LWOOBJID playerID = request.playerID;

dChatServer/ChatPacketHandler.h

@@ -17,6 +17,9 @@ namespace ChatPacketHandler {
 	// Sends msg to the world server `world`, which passes it on to the client of `target`
 	void SendRouted(const LWOOBJID target, const SystemAddress& world, const LUBitStream& msg, const bool broadcast = false);
 
+	// Reads a player's friends from the database into their data, noting which are online; nothing is sent
+	void LoadFriends(PlayerData& player);
+
 	void HandleFriendlistRequest(const ChatPackets::GetFriendsList& request, const SystemAddress& sysAddr);
 	void HandleFriendRequest(const ChatPackets::AddFriendRequest& request, const SystemAddress& sysAddr);
 	void HandleFriendResponse(const ChatPackets::AddFriendResponse& response, const SystemAddress& sysAddr);

dChatServer/ChatServer.cpp

@@ -26,6 +26,8 @@
 #include "TeamContainer.h"
 #include "PacketDispatcher.h"
 #include "ChatPackets.h"
+#include "master/LiveUpdate.h"
+#include "MasterPackets.h"
 
 #include "Game.h"
 #include "Server.h"
@@ -260,7 +262,7 @@ namespace {
 			handlers.On<GMLevelUpdate>(Chat::GMLEVEL_UPDATE, ChatPacketHandler::HandleGMLevelUpdate);
 			handlers.On<LoginSessionNotify>(Chat::LOGIN_SESSION_NOTIFY, [](const LoginSessionNotify& notify, const SystemAddress& sysAddr) { Game::playerContainer.InsertPlayer(notify, sysAddr); });
 			// we just forward this packet to every connected server
-			handlers.On<Announcement>(Chat::GM_ANNOUNCE, [](const Announcement& announcement, const SystemAddress& sysAddr) {
+			handlers.On<ChatPackets::Announcement>(Chat::GM_ANNOUNCE, [](const ChatPackets::Announcement& announcement, const SystemAddress& sysAddr) {
 				RakNet::BitStream bitStream;
 				announcement.WritePacket(bitStream);
 				Game::server->Send(bitStream, sysAddr, true); // send to everyone except origin
@@ -286,6 +288,18 @@ namespace {
 	const PacketDispatcher<MessageType::Master>& MasterHandlers() {
 		static const auto handlers = [] {
 			PacketDispatcher<MessageType::Master> handlers;
+			// Live update: a new chat server takes over. Hand the teams over (through master) and stop; master starts the
+			// new one once this one is gone, and the worlds send it their players again.
+			handlers.On<LiveUpdateRetire>(MessageType::Master::LIVE_UPDATE_RETIRE, [](const LiveUpdateRetire&, const SystemAddress&) {
+				const auto handoff = TeamContainer::MakeHandoff();
+				LOG("Live update: handing %zu team(s) over to the next chat server and stopping", handoff.teams.size());
+				MasterPackets::SendToMaster(handoff);
+				Game::playerContainer.SetRetiring(true);
+				Game::lastSignal = -1;
+			});
+			handlers.On<ChatHandoff>(MessageType::Master::CHAT_HANDOFF, [](const ChatHandoff& handoff, const SystemAddress&) {
+				TeamContainer::Restore(handoff);
+			});
 			handlers.On<PlayerActionRequest>(MessageType::Master::PLAYER_ACTION, [](const PlayerActionRequest& request, const SystemAddress&) {
 				// Words added or removed on the dashboard: web chat is checked with the same filter as the worlds
 				if (request.action == ePlayerAction::RELOAD_CHAT_FILTER && Game::chatFilter) {

dChatServer/PlayerContainer.cpp

@@ -30,6 +30,31 @@ void PlayerContainer::InsertPlayer(const ChatPackets::LoginSessionNotify& notify
 	// Names longer than ChatPackets::LoginSessionNotify::MAX_NAME_LENGTH were dropped when read
 	const LWOOBJID playerId = notify.playerID;
 
+	// A world sending its players to a new chat server (live update): nobody logged in or changed zones
+	if (notify.resync) {
+		if (m_Players.contains(playerId)) {
+			m_Players[playerId].worldServerSysAddr = sysAddr;
+			m_PlayersToRemove.erase(playerId);
+			return;
+		}
+		auto& data = m_Players[playerId];
+		data = PlayerData();
+		data.playerID = playerId;
+		data.playerName = notify.playerName;
+		data.zoneID = notify.zoneID;
+		data.muteExpire = static_cast<time_t>(notify.muteExpire);
+		data.gmLevel = notify.gmLevel;
+		data.worldServerSysAddr = sysAddr;
+		m_Names[data.playerID] = GeneralUtils::UTF8ToUTF16(data.playerName);
+		m_PlayerCount++;
+		// Their friends list is read again when their client next asks (every zone load); until then the friends
+		// they have are what friend updates go to
+		ChatPacketHandler::LoadFriends(data);
+		m_PlayersToRemove.erase(playerId);
+		LOG("Took over user: %s (%llu), zone: %i", data.playerName.c_str(), data.playerID, data.zoneID.GetMapID());
+		return;
+	}
+
 	auto isLogin = !m_Players.contains(playerId);
 	auto& data = m_Players[playerId];
 	data = PlayerData();
@@ -169,6 +194,11 @@ const PlayerData& PlayerContainer::GetPlayerData(const std::string& playerName)
 
 void PlayerContainer::Shutdown() {
 	m_Players.erase(LWOOBJID_EMPTY);
+	// Handed over to the next chat server (live update): everyone is still online
+	if (m_Retiring) {
+		m_Players.clear();
+		return;
+	}
 	while (!m_Players.empty()) {
 		const auto& [id, playerData] = *m_Players.begin();
 		Database::Get()->UpdateActivityLog(id, eActivityType::PlayerLoggedOut, playerData.zoneID.GetMapID());

dChatServer/PlayerContainer.h

@@ -67,6 +67,9 @@ class PlayerContainer {
 public:
 	void Initialize();
 	void InsertPlayer(const ChatPackets::LoginSessionNotify& notify, const SystemAddress& sysAddr);
+	// Live updates: this chat server replaces one that retired; don't log everyone out when it stops
+	void SetRetiring(bool retiring) { m_Retiring = retiring; }
+	bool IsRetiring() const { return m_Retiring; }
 	void ScheduleRemovePlayer(const ChatPackets::UnexpectedDisconnect& notify, const SystemAddress& sysAddr);
 	void RemovePlayer(const LWOOBJID playerID);
 	void MuteUpdate(const ChatPackets::GMMute& mute, const SystemAddress& sysAddr);
@@ -96,5 +99,6 @@ private:
 	uint32_t m_MaxNumberOfFriends = 50;
 	uint32_t m_PlayerCount = 0;
 	uint32_t m_SimCount = 0;
+	bool m_Retiring = false;
 };
 

dChatServer/TeamContainer.cpp

@@ -13,6 +13,39 @@ namespace {
 	LWOOBJID g_TeamIDCounter = 0;
 }
 
+ChatHandoff TeamContainer::MakeHandoff() {
+	ChatHandoff handoff;
+	for (const auto* team : GetTeams()) {
+		if (!team) continue;
+		auto& carried = handoff.teams.emplace_back();
+		carried.teamId = team->teamID;
+		carried.leaderId = team->leaderID;
+		carried.members = team->memberIDs;
+		carried.lootFlag = team->lootFlag;
+		carried.local = team->local;
+		carried.zoneId = team->zoneId.GetMapID();
+		carried.instanceId = team->zoneId.GetInstanceID();
+		carried.cloneId = team->zoneId.GetCloneID();
+	}
+	return handoff;
+}
+
+void TeamContainer::Restore(const ChatHandoff& handoff) {
+	for (const auto& carried : handoff.teams) {
+		if (carried.members.empty()) continue;
+		auto* team = new TeamData();
+		team->teamID = carried.teamId;
+		team->leaderID = carried.leaderId;
+		team->memberIDs = carried.members;
+		team->lootFlag = carried.lootFlag;
+		team->local = carried.local;
+		team->zoneId = LWOZONEID(carried.zoneId, carried.instanceId, carried.cloneId);
+		GetTeamsMut().push_back(team);
+		g_TeamIDCounter = std::max(g_TeamIDCounter, carried.teamId);
+	}
+	LOG("Took over %zu team(s) from the last chat server", handoff.teams.size());
+}
+
 const TeamContainer::Data& TeamContainer::GetTeamContainer() {
 	return g_TeamContainer;
 }

dChatServer/TeamContainer.h

@@ -10,6 +10,7 @@
 
 #include "dCommonVars.h"
 #include "ChatPackets.h"
+#include "master/LiveUpdate.h"
 
 struct SystemAddress;
 struct PlayerData;
@@ -53,6 +54,13 @@ namespace TeamContainer {
 	void UpdateTeamsOnWorld(TeamData* team, bool deleteTeam);
 
 	const TeamContainer::Data& GetTeamContainer();
+
+	// Live updates: the teams, for the next chat server (CHAT_HANDOFF)
+	ChatHandoff MakeHandoff();
+
+	// Live updates: the teams the last chat server had. Nothing is sent to anyone: the clients and worlds still know
+	// their teams; the players themselves come back as the worlds send them again.
+	void Restore(const ChatHandoff& handoff);
 	std::vector<TeamData*>& GetTeamsMut();
 	const std::vector<TeamData*>& GetTeams();
 };