From 109a93679885a8e098aa3ef0383750f6d34fafd0 Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Mon, 28 Sep 2026 15:01:05 -0500 Subject: [PATCH] feat: live updates move every server onto a new build without a restart With new binaries in place, master moves everything onto them while the server keeps running (the dashboard's Live update, /liveupdate or SIGUSR2 to master): - database migrations of the new build first; a failure stops there - UGC finishes the jobs it is running (queued rows stay pending), auth restarts, chat hands its teams to master for the next chat server; master starts the new processes and retries ones that don't come back - once the new chat server is up (CHAT_SERVER_READY) every world connects at once and sends its players again (LoginSessionNotify resync, no login logged) - every world instance is replaced with an instance migration: public worlds and private ones (same password) at once, properties after the old instance saved and froze the property (MIGRATE_PREPARE: no building, claiming or saving there any more), activity zones and character select once their players left or after a wait; empty instances just stop, zones in prestart_worlds get a new one first - the dashboard restarts last and picks the status up again Players land where they stood (position carried in CarriedPlayerState, also on properties and Moon Base). Draining instances get no new players (InstanceMigration::AcceptsNewPlayers) and show as "Moving players" in the world list. The order lives in LiveUpdateMachine.h without master state and is unit tested; master's glue is LiveUpdateCoordinator. Master itself is not replaced. Message IDs are appended only. Co-Authored-By: Claude Opus 5.5 --- dChatServer/ChatPacketHandler.cpp | 18 + dChatServer/ChatPacketHandler.h | 3 + dChatServer/ChatServer.cpp | 16 +- dChatServer/PlayerContainer.cpp | 30 + dChatServer/PlayerContainer.h | 4 + dChatServer/TeamContainer.cpp | 33 + dChatServer/TeamContainer.h | 8 + dCommon/Permissions.cpp | 1 + dCommon/dEnums/MessageType/Master.h | 14 + dDashboardServer/DashboardServer.cpp | 8 +- dDashboardServer/routes/CMakeLists.txt | 1 + dDashboardServer/routes/LiveUpdateRoutes.cpp | 138 ++++ dDashboardServer/routes/LiveUpdateRoutes.h | 21 + dDashboardServer/routes/LiveWorld.cpp | 4 +- dDashboardServer/routes/ServerState.h | 8 +- dDashboardServer/routes/SettingsCatalog.cpp | 11 + dDashboardServer/templates/index.jinja2 | 86 ++ dGame/UserManager.h | 2 + .../PropertyManagementComponent.cpp | 33 +- .../dComponents/PropertyManagementComponent.h | 18 + dGame/dUtilities/SlashCommandHandler.cpp | 9 + dGame/dUtilities/WorldMigration.cpp | 178 +++- dGame/dUtilities/WorldMigration.h | 13 + dMasterServer/CMakeLists.txt | 1 + dMasterServer/InstanceManager.cpp | 5 +- dMasterServer/InstanceManager.h | 18 + dMasterServer/LiveUpdateCoordinator.cpp | 315 +++++++ dMasterServer/LiveUpdateCoordinator.h | 48 ++ dMasterServer/LiveUpdateMachine.h | 580 +++++++++++++ dMasterServer/MasterServer.cpp | 145 +++- dMasterServer/MigrationCoordinator.cpp | 140 +++- dMasterServer/MigrationCoordinator.h | 19 +- dNet/ChatPackets.cpp | 4 + dNet/ChatPackets.h | 3 + dNet/MasterPackets.cpp | 2 +- dNet/MasterPackets.h | 8 +- dNet/master/InstanceMigration.h | 91 ++- dNet/master/LiveUpdate.h | 299 +++++++ dUgcServer/UgcProcessor.cpp | 31 + dUgcServer/UgcProcessor.h | 10 + dUgcServer/UgcServer.cpp | 12 + dWorldServer/WorldServer.cpp | 51 ++ resources/masterconfig.ini | 21 + tests/dCommonTests/CMakeLists.txt | 1 + tests/dCommonTests/LiveUpdateTests.cpp | 766 ++++++++++++++++++ tests/dCommonTests/MessageIdPinTests.cpp | 26 +- .../dGameTests/dNetTests/ChatPacketsTests.cpp | 19 + 47 files changed, 3213 insertions(+), 59 deletions(-) create mode 100644 dDashboardServer/routes/LiveUpdateRoutes.cpp create mode 100644 dDashboardServer/routes/LiveUpdateRoutes.h create mode 100644 dMasterServer/LiveUpdateCoordinator.cpp create mode 100644 dMasterServer/LiveUpdateCoordinator.h create mode 100644 dMasterServer/LiveUpdateMachine.h create mode 100644 dNet/master/LiveUpdate.h create mode 100644 tests/dCommonTests/LiveUpdateTests.cpp diff --git a/dChatServer/ChatPacketHandler.cpp b/dChatServer/ChatPacketHandler.cpp index a48ff0eaa..f25e4ae6d 100644 --- a/dChatServer/ChatPacketHandler.cpp +++ b/dChatServer/ChatPacketHandler.cpp @@ -32,6 +32,24 @@ void ChatPacketHandler::SendRouted(const LWOOBJID target, const SystemAddress& w Game::server->Send(bitStream, world, broadcast); } +void ChatPacketHandler::LoadFriends(PlayerData& player) { + player.friends.clear(); + player.countOfBestFriends = 0; + for (const auto& friendData : Database::Get()->GetFriendsList(player.playerID)) { + FriendData fd; + fd.isFTP = false; + fd.friendID = friendData.friendID; + GeneralUtils::SetBit(fd.friendID, eObjectBits::CHARACTER); + fd.isBestFriend = friendData.isBestFriend; + if (fd.isBestFriend) player.countOfBestFriends += 1; + fd.friendName = friendData.friendName; + const auto& online = Game::playerContainer.GetPlayerData(fd.friendID); + fd.isOnline = static_cast(online); + fd.zoneID = online ? online.zoneID : LWOZONEID(); + player.friends.push_back(fd); + } +} + void ChatPacketHandler::HandleFriendlistRequest(const ChatPackets::GetFriendsList& request, const SystemAddress& sysAddr) { //Get from the packet which player we want to do something with: const LWOOBJID playerID = request.playerID; diff --git a/dChatServer/ChatPacketHandler.h b/dChatServer/ChatPacketHandler.h index fe48f357c..6187feca2 100644 --- a/dChatServer/ChatPacketHandler.h +++ b/dChatServer/ChatPacketHandler.h @@ -17,6 +17,9 @@ namespace ChatPacketHandler { // Sends msg to the world server `world`, which passes it on to the client of `target` void SendRouted(const LWOOBJID target, const SystemAddress& world, const LUBitStream& msg, const bool broadcast = false); + // Reads a player's friends from the database into their data, noting which are online; nothing is sent + void LoadFriends(PlayerData& player); + void HandleFriendlistRequest(const ChatPackets::GetFriendsList& request, const SystemAddress& sysAddr); void HandleFriendRequest(const ChatPackets::AddFriendRequest& request, const SystemAddress& sysAddr); void HandleFriendResponse(const ChatPackets::AddFriendResponse& response, const SystemAddress& sysAddr); diff --git a/dChatServer/ChatServer.cpp b/dChatServer/ChatServer.cpp index bd0487fb1..904b48c9c 100644 --- a/dChatServer/ChatServer.cpp +++ b/dChatServer/ChatServer.cpp @@ -26,6 +26,8 @@ #include "TeamContainer.h" #include "PacketDispatcher.h" #include "ChatPackets.h" +#include "master/LiveUpdate.h" +#include "MasterPackets.h" #include "Game.h" #include "Server.h" @@ -260,7 +262,7 @@ namespace { handlers.On(Chat::GMLEVEL_UPDATE, ChatPacketHandler::HandleGMLevelUpdate); handlers.On(Chat::LOGIN_SESSION_NOTIFY, [](const LoginSessionNotify& notify, const SystemAddress& sysAddr) { Game::playerContainer.InsertPlayer(notify, sysAddr); }); // we just forward this packet to every connected server - handlers.On(Chat::GM_ANNOUNCE, [](const Announcement& announcement, const SystemAddress& sysAddr) { + handlers.On(Chat::GM_ANNOUNCE, [](const ChatPackets::Announcement& announcement, const SystemAddress& sysAddr) { RakNet::BitStream bitStream; announcement.WritePacket(bitStream); Game::server->Send(bitStream, sysAddr, true); // send to everyone except origin @@ -286,6 +288,18 @@ namespace { const PacketDispatcher& MasterHandlers() { static const auto handlers = [] { PacketDispatcher handlers; + // Live update: a new chat server takes over. Hand the teams over (through master) and stop; master starts the + // new one once this one is gone, and the worlds send it their players again. + handlers.On(MessageType::Master::LIVE_UPDATE_RETIRE, [](const LiveUpdateRetire&, const SystemAddress&) { + const auto handoff = TeamContainer::MakeHandoff(); + LOG("Live update: handing %zu team(s) over to the next chat server and stopping", handoff.teams.size()); + MasterPackets::SendToMaster(handoff); + Game::playerContainer.SetRetiring(true); + Game::lastSignal = -1; + }); + handlers.On(MessageType::Master::CHAT_HANDOFF, [](const ChatHandoff& handoff, const SystemAddress&) { + TeamContainer::Restore(handoff); + }); handlers.On(MessageType::Master::PLAYER_ACTION, [](const PlayerActionRequest& request, const SystemAddress&) { // Words added or removed on the dashboard: web chat is checked with the same filter as the worlds if (request.action == ePlayerAction::RELOAD_CHAT_FILTER && Game::chatFilter) { diff --git a/dChatServer/PlayerContainer.cpp b/dChatServer/PlayerContainer.cpp index 5c9c1cd70..a83ba9b6e 100644 --- a/dChatServer/PlayerContainer.cpp +++ b/dChatServer/PlayerContainer.cpp @@ -30,6 +30,31 @@ void PlayerContainer::InsertPlayer(const ChatPackets::LoginSessionNotify& notify // Names longer than ChatPackets::LoginSessionNotify::MAX_NAME_LENGTH were dropped when read const LWOOBJID playerId = notify.playerID; + // A world sending its players to a new chat server (live update): nobody logged in or changed zones + if (notify.resync) { + if (m_Players.contains(playerId)) { + m_Players[playerId].worldServerSysAddr = sysAddr; + m_PlayersToRemove.erase(playerId); + return; + } + auto& data = m_Players[playerId]; + data = PlayerData(); + data.playerID = playerId; + data.playerName = notify.playerName; + data.zoneID = notify.zoneID; + data.muteExpire = static_cast(notify.muteExpire); + data.gmLevel = notify.gmLevel; + data.worldServerSysAddr = sysAddr; + m_Names[data.playerID] = GeneralUtils::UTF8ToUTF16(data.playerName); + m_PlayerCount++; + // Their friends list is read again when their client next asks (every zone load); until then the friends + // they have are what friend updates go to + ChatPacketHandler::LoadFriends(data); + m_PlayersToRemove.erase(playerId); + LOG("Took over user: %s (%llu), zone: %i", data.playerName.c_str(), data.playerID, data.zoneID.GetMapID()); + return; + } + auto isLogin = !m_Players.contains(playerId); auto& data = m_Players[playerId]; data = PlayerData(); @@ -169,6 +194,11 @@ const PlayerData& PlayerContainer::GetPlayerData(const std::string& playerName) void PlayerContainer::Shutdown() { m_Players.erase(LWOOBJID_EMPTY); + // Handed over to the next chat server (live update): everyone is still online + if (m_Retiring) { + m_Players.clear(); + return; + } while (!m_Players.empty()) { const auto& [id, playerData] = *m_Players.begin(); Database::Get()->UpdateActivityLog(id, eActivityType::PlayerLoggedOut, playerData.zoneID.GetMapID()); diff --git a/dChatServer/PlayerContainer.h b/dChatServer/PlayerContainer.h index 569465aea..d0bd3397f 100644 --- a/dChatServer/PlayerContainer.h +++ b/dChatServer/PlayerContainer.h @@ -67,6 +67,9 @@ class PlayerContainer { public: void Initialize(); void InsertPlayer(const ChatPackets::LoginSessionNotify& notify, const SystemAddress& sysAddr); + // Live updates: this chat server replaces one that retired; don't log everyone out when it stops + void SetRetiring(bool retiring) { m_Retiring = retiring; } + bool IsRetiring() const { return m_Retiring; } void ScheduleRemovePlayer(const ChatPackets::UnexpectedDisconnect& notify, const SystemAddress& sysAddr); void RemovePlayer(const LWOOBJID playerID); void MuteUpdate(const ChatPackets::GMMute& mute, const SystemAddress& sysAddr); @@ -96,5 +99,6 @@ private: uint32_t m_MaxNumberOfFriends = 50; uint32_t m_PlayerCount = 0; uint32_t m_SimCount = 0; + bool m_Retiring = false; }; diff --git a/dChatServer/TeamContainer.cpp b/dChatServer/TeamContainer.cpp index c542e39f4..c3daf6708 100644 --- a/dChatServer/TeamContainer.cpp +++ b/dChatServer/TeamContainer.cpp @@ -13,6 +13,39 @@ namespace { LWOOBJID g_TeamIDCounter = 0; } +ChatHandoff TeamContainer::MakeHandoff() { + ChatHandoff handoff; + for (const auto* team : GetTeams()) { + if (!team) continue; + auto& carried = handoff.teams.emplace_back(); + carried.teamId = team->teamID; + carried.leaderId = team->leaderID; + carried.members = team->memberIDs; + carried.lootFlag = team->lootFlag; + carried.local = team->local; + carried.zoneId = team->zoneId.GetMapID(); + carried.instanceId = team->zoneId.GetInstanceID(); + carried.cloneId = team->zoneId.GetCloneID(); + } + return handoff; +} + +void TeamContainer::Restore(const ChatHandoff& handoff) { + for (const auto& carried : handoff.teams) { + if (carried.members.empty()) continue; + auto* team = new TeamData(); + team->teamID = carried.teamId; + team->leaderID = carried.leaderId; + team->memberIDs = carried.members; + team->lootFlag = carried.lootFlag; + team->local = carried.local; + team->zoneId = LWOZONEID(carried.zoneId, carried.instanceId, carried.cloneId); + GetTeamsMut().push_back(team); + g_TeamIDCounter = std::max(g_TeamIDCounter, carried.teamId); + } + LOG("Took over %zu team(s) from the last chat server", handoff.teams.size()); +} + const TeamContainer::Data& TeamContainer::GetTeamContainer() { return g_TeamContainer; } diff --git a/dChatServer/TeamContainer.h b/dChatServer/TeamContainer.h index a879061a2..d63be9fe2 100644 --- a/dChatServer/TeamContainer.h +++ b/dChatServer/TeamContainer.h @@ -10,6 +10,7 @@ #include "dCommonVars.h" #include "ChatPackets.h" +#include "master/LiveUpdate.h" struct SystemAddress; struct PlayerData; @@ -53,6 +54,13 @@ namespace TeamContainer { void UpdateTeamsOnWorld(TeamData* team, bool deleteTeam); const TeamContainer::Data& GetTeamContainer(); + + // Live updates: the teams, for the next chat server (CHAT_HANDOFF) + ChatHandoff MakeHandoff(); + + // Live updates: the teams the last chat server had. Nothing is sent to anyone: the clients and worlds still know + // their teams; the players themselves come back as the worlds send them again. + void Restore(const ChatHandoff& handoff); std::vector& GetTeamsMut(); const std::vector& GetTeams(); }; diff --git a/dCommon/Permissions.cpp b/dCommon/Permissions.cpp index 1b4ade2ed..91f3daffa 100644 --- a/dCommon/Permissions.cpp +++ b/dCommon/Permissions.cpp @@ -100,6 +100,7 @@ namespace { { "live_events_manage", "Server", "Live events", "Start and end live events in game: treasure hunts, bonus coins/U-score/loot, invasions and celebrations; also as part of scheduled events", 8 }, { "challenges_manage", "Server", "Community challenges", "Create, change, cancel and delete server-wide challenges and their rewards (rewards are mailed to everyone who took part)", 8 }, { "events_manage", "Server", "Scheduled events: features", "Scheduled events that switch game features (event_1..event_8) on and off", 8 }, + { "server_live_update", "Server", "Live updates", "Move every server and world instance onto a new build without a restart (players see a short loading screen), and cancel one", 9 }, { "instances_manage", "Server", "Instance limits", "Change players per instance and spare instances per zone (the master server applies them)", 9 }, { "maintenance", "Server", "Data maintenance", "Repair tools on the Maintenance page", 9 }, { "backups", "Server", "Database backups", "Make, download and delete database backups (downloads also need your password)", 9 }, diff --git a/dCommon/dEnums/MessageType/Master.h b/dCommon/dEnums/MessageType/Master.h index cc803ab07..6ef2b1c17 100644 --- a/dCommon/dEnums/MessageType/Master.h +++ b/dCommon/dEnums/MessageType/Master.h @@ -70,5 +70,19 @@ namespace MessageType { // UGC server -> master -> every world: player models whose mesh (model.nif) was just made or made again (see // UgcModelsMade.h), so worlds showing them tell their clients (NotifyClientUGCModelReady) UGC_MODELS_MADE, + + // Live updates (see LiveUpdate.h, docs/LiveUpdate.md) + // Master -> world: a property (clone) instance is about to be replaced; freeze and save it, answer MIGRATE_STATUS + MIGRATE_PREPARE, + // Dashboard or world (a GM's /liveupdate) -> master: start, cancel or ask about a live update + LIVE_UPDATE_REQUEST, + // Master -> dashboard and worlds: how a live update is going + LIVE_UPDATE_STATUS, + // Master -> chat or UGC server: a new build takes over; finish up and exit + LIVE_UPDATE_RETIRE, + // Retiring chat server -> master -> the next chat server: the teams to carry over + CHAT_HANDOFF, + // Master -> worlds during a live update: a new chat server is up; connect and send it who is online + CHAT_SERVER_READY, }; } diff --git a/dDashboardServer/DashboardServer.cpp b/dDashboardServer/DashboardServer.cpp index d234af947..8d4740a55 100644 --- a/dDashboardServer/DashboardServer.cpp +++ b/dDashboardServer/DashboardServer.cpp @@ -59,6 +59,7 @@ #include "ModerationTools.h" #include "ModeratorHelper.h" #include "LiveWorld.h" +#include "LiveUpdateRoutes.h" #include "Scenery.h" #include "Workers.h" #include "ReportRoutes.h" @@ -222,7 +223,9 @@ namespace { info.zoneName = GetZoneDisplayName(info.mapID); AddPropertyDetails(info); using eState = MasterPackets::ServerListResponse::eState; - if (instance.state == eState::READY) { + if (instance.state == eState::READY || instance.state == eState::DRAINING) { + // Draining: still up with its players, who are being moved to a new instance + if (instance.state == eState::DRAINING) info.state = "draining"; ServerState::g_WorldInstances.push_back(info); } else { info.state = instance.state == eState::STARTING ? "starting" : "stopping"; @@ -340,6 +343,7 @@ namespace { handlers.On(Master::PLAYER_ACTION_RESULT, [](const PlayerActionResult& result, const SystemAddress&) { PlayerActions::HandleResult(result); }); handlers.On(Master::SHUTDOWN_RESPONSE, OnWorldShutDown); handlers.On(Master::SERVER_TRAFFIC, [](const ServerTraffic& report, const SystemAddress&) { Traffic::Ingest(report); }); + handlers.On(Master::LIVE_UPDATE_STATUS, [](const LiveUpdateStatus& status, const SystemAddress&) { LiveUpdateRoutes::HandleStatus(status); }); return handlers; }(); return handlers; @@ -540,6 +544,7 @@ int main(int argc, char** argv) { RegisterModerationToolRoutes(); ModeratorHelper::RegisterRoutes(); LiveWorld::RegisterRoutes(); + LiveUpdateRoutes::RegisterRoutes(); Inspector::RegisterRoutes(); RegisterCDClientBrowserRoutes(); RegisterSettingsRoutes(); @@ -606,6 +611,7 @@ int main(int argc, char** argv) { Scheduler::Update(); ServerRoutes::Update(); LiveWorld::Update(); + LiveUpdateRoutes::Update(); Inspector::Update(); Announcements::Update(); EventsCalendar::Update(); diff --git a/dDashboardServer/routes/CMakeLists.txt b/dDashboardServer/routes/CMakeLists.txt index 6d2b8b8cf..a11e08ae7 100644 --- a/dDashboardServer/routes/CMakeLists.txt +++ b/dDashboardServer/routes/CMakeLists.txt @@ -16,6 +16,7 @@ set(DASHBOARDROUTES_SOURCES "WebhookRoutes.cpp" "EconomyJobs.cpp" "LiveWorld.cpp" + "LiveUpdateRoutes.cpp" "SettingsRoutes.cpp" "Background.cpp" "Scheduler.cpp" diff --git a/dDashboardServer/routes/LiveUpdateRoutes.cpp b/dDashboardServer/routes/LiveUpdateRoutes.cpp new file mode 100644 index 000000000..a2c049a0b --- /dev/null +++ b/dDashboardServer/routes/LiveUpdateRoutes.cpp @@ -0,0 +1,138 @@ +#include "LiveUpdateRoutes.h" + +#include +#include + +#include "Alerts.h" +#include "DashboardRoutes.h" +#include "Game.h" +#include "Logger.h" +#include "MasterPackets.h" +#include "Permissions.h" +#include "RouteUtils.h" +#include "Web.h" +#include "dServer.h" +#include "eHTTPMethod.h" +#include "master/LiveUpdate.h" + +using namespace RouteUtils; + +namespace { + constexpr const char* TOPIC = "live_update"; + constexpr const char* PERMISSION = "server_live_update"; + + LiveUpdateStatus g_Status; + bool g_Asked = false; // asked master for the status since connecting + bool g_WasConnected = false; + LiveUpdate::ePhase g_AlertedPhase = LiveUpdate::ePhase::IDLE; + uint32_t g_AlertedUpdate = 0; + + nlohmann::json ToJson(const LiveUpdateStatus& status) { + using namespace LiveUpdate; + const auto& zones = ZoneNames(); + nlohmann::json units = nlohmann::json::array(); + size_t done = 0, failed = 0; + for (const auto& unit : status.units) { + if (IsFinished(unit.state)) done++; + if (unit.state == eUnitState::FAILED) failed++; + nlohmann::json row{ {"kind", KindName(unit.kind)}, {"state", StateName(unit.state)}, {"message", unit.message} }; + if (unit.kind == eUnitKind::WORLD) { + const auto zone = std::to_string(unit.zoneId); + row["zone"] = unit.zoneId; + row["zoneName"] = unit.zoneId == 0 ? "Character Select" : zones.contains(zone) ? zones[zone].get() : "Zone " + zone; + row["instance"] = unit.instanceId; + row["clone"] = unit.cloneId; + row["replacement"] = unit.replacement; + row["players"] = unit.players; + row["moved"] = unit.moved; + row["isPrivate"] = unit.isPrivate != 0; + } + units.push_back(std::move(row)); + } + return { + {"updateId", status.updateId}, {"phase", PhaseName(status.phase)}, {"running", !IsFinished(status.phase) && status.phase != ePhase::IDLE}, + {"startedAt", status.startedAt}, {"finishedAt", status.finishedAt}, {"by", status.by}, {"message", status.message}, + {"done", done}, {"failed", failed}, {"total", status.units.size()}, {"units", units} + }; + } + + bool SendRequest(LiveUpdate::eAction action, int32_t warnSeconds, const std::string& by) { + if (!Game::server || !Game::server->GetIsConnectedToMaster()) return false; + LiveUpdateRequest request; + request.action = action; + request.warnSeconds = warnSeconds; + request.requestedBy = by.substr(0, LiveUpdateRequest::MAX_BY); + MasterPackets::SendToMaster(request); + return true; + } +} + +namespace LiveUpdateRoutes { + void HandleStatus(const LiveUpdateStatus& status) { + g_Status = status; + auto json = ToJson(status); + Game::web.SendWSMessage(TOPIC, json); + // Webhooks hear about the start and the end + using LiveUpdate::ePhase; + if (status.updateId != 0 && (status.updateId != g_AlertedUpdate || status.phase != g_AlertedPhase)) { + const bool started = status.phase == ePhase::RUNNING && status.updateId != g_AlertedUpdate; + const bool finished = LiveUpdate::IsFinished(status.phase); + if (started) Alerts::Emit("server", "Live update started", "The servers are being moved onto the new build.", { { "By", status.by } }, "/#liveUpdate"); + // Not for an old one a restarted dashboard hears about + const bool recent = status.finishedAt == 0 || std::time(nullptr) - status.finishedAt < 600; + if (finished && g_AlertedPhase != status.phase && recent) { + Alerts::Emit("server", std::string("Live update ") + LiveUpdate::PhaseName(status.phase), status.message, { { "By", status.by } }, "/#liveUpdate"); + } + g_AlertedUpdate = status.updateId; + g_AlertedPhase = status.phase; + } + } + + void Update() { + const bool connected = Game::server && Game::server->GetIsConnectedToMaster(); + if (connected && !g_WasConnected) g_Asked = false; + g_WasConnected = connected; + if (!connected || g_Asked) return; + g_Asked = SendRequest(LiveUpdate::eAction::STATUS, LiveUpdateRequest::DEFAULT_WARN, "dashboard"); + } + + nlohmann::json StatusJson() { + return ToJson(g_Status); + } + + void RegisterRoutes() { + Game::web.RegisterWSSubscription(TOPIC, std::function([] { return Permissions::Level(PERMISSION); }), PERMISSION); + + Route(eHTTPMethod::GET, "/api/server/live_update", Perm(PERMISSION), + "The last live update (or the running one): phase, who started it, and every server and world instance with its state (also pushed on the live_update socket topic)", + [](HTTPReply& reply, const HTTPContext&) { + JsonSuccess(reply, { {"liveUpdate", StatusJson()} }); + }); + + Route(eHTTPMethod::POST, "/api/server/live_update", Perm(PERMISSION), + "Start a live update onto the binaries on disk now: UGC, auth and chat restart, every world instance is replaced and its players moved, the dashboard restarts last. Body: {warnSeconds (0-300, optional)}", + [](HTTPReply& reply, const HTTPContext& context) { + const auto body = ParseBody(context); + int32_t warn = LiveUpdateRequest::DEFAULT_WARN; + if (body && body->contains("warnSeconds") && !(*body)["warnSeconds"].is_null()) { + warn = body->value("warnSeconds", -1); + if (warn < 0 || warn > InstanceMigrationRequest::MAX_WARN_SECONDS) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Warn players 0 to 300 seconds before they are moved"); + } + if (!SendRequest(LiveUpdate::eAction::START, warn, context.authenticatedUser)) { + return JsonError(reply, eHTTPStatusCode::SERVICE_UNAVAILABLE, "Not connected to the master server"); + } + Audit(context, "live_update", warn >= 0 ? "Started a live update (players warned " + std::to_string(warn) + " s before they are moved)" : "Started a live update"); + JsonSuccess(reply, { {"message", "Asked master to start a live update"} }); + }); + + Route(eHTTPMethod::POST, "/api/server/live_update/cancel", Perm(PERMISSION), + "Cancel the running live update: nothing new is started, what is under way finishes", + [](HTTPReply& reply, const HTTPContext& context) { + if (!SendRequest(LiveUpdate::eAction::CANCEL, LiveUpdateRequest::DEFAULT_WARN, context.authenticatedUser)) { + return JsonError(reply, eHTTPStatusCode::SERVICE_UNAVAILABLE, "Not connected to the master server"); + } + Audit(context, "live_update_cancel", "Cancelled the live update"); + JsonSuccess(reply, { {"message", "Asked master to cancel the live update"} }); + }); + } +} diff --git a/dDashboardServer/routes/LiveUpdateRoutes.h b/dDashboardServer/routes/LiveUpdateRoutes.h new file mode 100644 index 000000000..55fd9cffd --- /dev/null +++ b/dDashboardServer/routes/LiveUpdateRoutes.h @@ -0,0 +1,21 @@ +#pragma once + +#include "json.hpp" + +struct LiveUpdateStatus; + +/** + * The dashboard's "Live update" (docs/LiveUpdate.md): starts one on master and shows how it goes. The status comes + * from master (LIVE_UPDATE_STATUS) and is pushed to browsers on the live_update socket topic. + */ +namespace LiveUpdateRoutes { + void RegisterRoutes(); + + // Master sent a status + void HandleStatus(const LiveUpdateStatus& status); + + // Main loop: ask master for the status once connected (a restarted dashboard picks a running update up again) + void Update(); + + nlohmann::json StatusJson(); +} diff --git a/dDashboardServer/routes/LiveWorld.cpp b/dDashboardServer/routes/LiveWorld.cpp index ad3c5d24a..33ea6811b 100644 --- a/dDashboardServer/routes/LiveWorld.cpp +++ b/dDashboardServer/routes/LiveWorld.cpp @@ -245,14 +245,14 @@ namespace LiveWorld { JsonSuccess(reply, { {"message", "Shutting it down"} }); }); - Route(eHTTPMethod::GET, "/api/worlds", Perm("players_view"), "Every world with its instance and clone IDs, address, players and, for property instances, the property and owner; worlds still starting or shutting down have state: starting|stopping", + Route(eHTTPMethod::GET, "/api/worlds", Perm("players_view"), "Every world with its instance and clone IDs, address, players and, for property instances, the property and owner; worlds still starting or shutting down have state: starting|stopping, and those whose players are being moved to a new instance state: draining", [](HTTPReply& reply, const HTTPContext&) { nlohmann::json worlds = nlohmann::json::array(); std::lock_guard lock(ServerState::g_StatusMutex); for (const auto& w : ServerState::g_WorldInstances) { worlds.push_back({ {"mapID", w.mapID}, {"zoneName", w.zoneName}, {"instanceID", w.instanceID}, {"cloneID", w.cloneID}, {"players", w.players}, {"isPrivate", w.isPrivate}, {"ip", w.ip}, {"port", w.port}, {"propertyId", w.propertyId}, {"propertyName", w.propertyName}, - {"ownerId", w.ownerId}, {"ownerName", w.ownerName} }); + {"ownerId", w.ownerId}, {"ownerName", w.ownerName}, {"state", w.state} }); } // Launched but not connected yet, or shutting down for (const auto& w : ServerState::g_PendingWorlds) { diff --git a/dDashboardServer/routes/ServerState.h b/dDashboardServer/routes/ServerState.h index da187c2b0..3f4a662c4 100644 --- a/dDashboardServer/routes/ServerState.h +++ b/dDashboardServer/routes/ServerState.h @@ -39,7 +39,8 @@ struct WorldInstanceInfo { std::string propertyName{}; std::string ownerId{}; std::string ownerName{}; - // "starting" (launched, not connected yet) or "stopping"; worlds that are up are in g_WorldInstances instead + // "starting" (launched, not connected yet) or "stopping" (in g_PendingWorlds), or "draining" for a world that is up + // but whose players are being moved to a new instance (in g_WorldInstances); empty for one that is just up std::string state{}; }; @@ -77,7 +78,8 @@ namespace ServerState { {"cloneID", world.cloneID}, {"players", world.players}, {"isPrivate", world.isPrivate}, - {"zoneName", world.zoneName} + {"zoneName", world.zoneName}, + {"state", world.state} }); } @@ -103,7 +105,7 @@ namespace ServerState { const auto mapID = world.value("mapID", 0u); if (world.value("cloneID", 0u) == 0) { worlds.push_back({ {"mapID", mapID}, {"instanceID", world.value("instanceID", 0u)}, {"players", world.value("players", 0u)}, - {"zoneName", world.value("zoneName", "")} }); + {"zoneName", world.value("zoneName", "")}, {"state", world.value("state", "")} }); continue; } const auto it = propertyRows.find(mapID); diff --git a/dDashboardServer/routes/SettingsCatalog.cpp b/dDashboardServer/routes/SettingsCatalog.cpp index 84d495e29..6ee4abf0f 100644 --- a/dDashboardServer/routes/SettingsCatalog.cpp +++ b/dDashboardServer/routes/SettingsCatalog.cpp @@ -137,6 +137,17 @@ namespace { c.Add(Bool(MASTER, "enable_dashboard", "Start the web dashboard", "", false, true)); c.Add(Bool(SHARED, "skip_account_creation", "Skip the first-account prompt", "For non-interactive setups: master doesn't ask for an account when there are none.", false, true)); + c.AddSection("Live updates", "Moving every server and world instance onto a new build without a restart (the dashboard's Live update, /liveupdate or SIGUSR2 to master). Read when one starts."); + c.Add(Unit(Int(MASTER, "live_update_warn_seconds", "Warning before moving players", "The game's Mythran maintenance warning is shown this long before players are moved. The dashboard can pick another for one update.", "10", 0, 300), "seconds")); + c.Add(Int(MASTER, "live_update_parallel_worlds", "Worlds at once", "World instances replaced at the same time.", "4", 1, 64)); + c.Add(Unit(Int(MASTER, "live_update_player_wait", "Wait for busy players", "Players who are dead or building are moved once they are done, or after this long.", "30", 0, 600), "seconds")); + c.Add(Unit(Int(MASTER, "live_update_property_build_wait", "Wait for property builders", "A property is saved for its new instance once nobody builds there, or after this long (they leave build mode then).", "60", 0, 600), "seconds")); + c.Add(Unit(Int(MASTER, "live_update_char_select_wait", "Wait at character select", "Players picking a character get this long to go in by themselves; then they are moved to the new character select.", "60", 0, 3600), "seconds")); + c.Add(Unit(Int(MASTER, "live_update_activity_wait", "Wait for activities", "Races, minigames and other activity zones get this long to finish; then their players are moved (the activity is lost).", "1800", 0, 86400), "seconds")); + c.Add(Unit(Int(MASTER, "live_update_ugc_drain_timeout", "Wait for the UGC server", "The UGC server finishes the models it is making; after this long it is stopped (they are made again).", "300", 0, 3600), "seconds")); + c.Add(Unit(Int(MASTER, "live_update_service_timeout", "Server restart timeout", "How long auth, chat, the UGC server or the dashboard may take to stop or come back before master starts it again (3 tries).", "30", 10, 600), "seconds")); + c.Add(Bool(MASTER, "live_update_run_migrations", "Run database migrations", "Run the new build's database migrations first. The running servers must cope with the new schema until they are replaced.", true)); + c.AddSection("Game client"); c.Add(Format(Text(SHARED, "client_location", "Client folder", "The folder with res/, or the one with client/ and versions/.", "", true), eFormat::PATH)); c.Add(Int(SHARED, "client_net_version", "Network version", "Clients reporting another version are refused. 171022 for the retail client, 171023 for Darkflame Universe clients.", "171022", 0, std::nullopt, true)); diff --git a/dDashboardServer/templates/index.jinja2 b/dDashboardServer/templates/index.jinja2 index ad2f31e12..2517bb571 100644 --- a/dDashboardServer/templates/index.jinja2 +++ b/dDashboardServer/templates/index.jinja2 @@ -48,6 +48,44 @@ {% include "world_instances.jinja2" %} +{% if can.server_live_update %} +
+
+
+
+
Live update
+ +
+
+

Moves everything onto the server binaries on disk now, without a restart: the UGC server finishes its jobs + and restarts, auth restarts (logins pause for a few seconds), chat hands its teams over and restarts, every world instance is + replaced by a new one and its players are moved there (the game's Mythran maintenance warning, then a short loading screen), and + the dashboard restarts last. Properties are saved before their new instance loads them; races and minigames finish first. + Master keeps running: updating master itself takes a normal restart.

+
+
+ Warn players + +
+ + +
+

+
+ + + +
WhatInstancePlayersState
+
+
+
+
+
+{% endif %} + {% if can.server_announce or can.server_restart %}
{% if can.server_announce %} @@ -122,12 +160,60 @@ $(function () { if (window.Live) Live.onStatus(function (s) { showRestart(s.restart); }); } + // Live update: start or cancel it, and show every server and world as it goes (pushed on the live_update topic) + if (document.getElementById('liveUpdate')) { + var stateColors = { pending: 'secondary', preparing: 'info', starting: 'info', ready: 'primary', waiting: 'warning', draining: 'warning', + stopping: 'info', stopped: 'success', failed: 'danger', skipped: 'secondary' }; + var stateNames = { pending: 'Waiting its turn', preparing: 'Saving property', starting: 'Starting new one', ready: 'New one ready', waiting: 'Waiting for players', + draining: 'Moving players', stopping: 'Stopping old one', stopped: 'Done', failed: 'Failed', skipped: 'Skipped' }; + var phaseColors = { idle: 'secondary', running: 'warning', cancelling: 'warning', done: 'success', failed: 'danger', cancelled: 'secondary' }; + var kindNames = { database: 'Database migrations', ugc: 'UGC server', auth: 'Auth server', chat: 'Chat server', dashboard: 'Dashboard' }; + var showLiveUpdate = function (u) { + if (!u) return; + var running = !!u.running; + document.getElementById('liveUpdateStart').classList.toggle('d-none', running); + document.getElementById('liveUpdateWarn').closest('.input-group').classList.toggle('d-none', running); + document.getElementById('liveUpdateCancel').classList.toggle('d-none', !running || u.phase === 'cancelling'); + document.getElementById('liveUpdatePhase').innerHTML = u.updateId ? fmt.badge(u.phase, phaseColors[u.phase] || 'secondary') : ''; + var summary = ''; + if (u.updateId) { + summary = 'Started ' + esc(fmt.unix(u.startedAt)) + (u.by ? ' by ' + esc(u.by) : '') + (u.finishedAt ? ', finished ' + esc(fmt.unix(u.finishedAt)) : '') + + '. ' + esc(u.done) + ' of ' + esc(u.total) + ' done' + (u.failed ? ', ' + esc(u.failed) + ' failed' : '') + '.'; + } + if (u.message) summary += (summary ? ' ' : '') + '' + esc(u.message) + ''; + document.getElementById('liveUpdateSummary').innerHTML = summary; + var rows = (u.units || []).map(function (unit) { + var what = unit.kind === 'world' ? fmt.zone(unit.zone, unit.zoneName) + (unit.clone ? ' property clone ' + esc(unit.clone) + '' : '') + + (unit.isPrivate ? ' ' + fmt.badge('Private', 'warning') : '') : esc(kindNames[unit.kind] || unit.kind); + var instance = unit.kind === 'world' ? esc(unit.instance) + (unit.replacement ? ' → ' + esc(unit.replacement) : '') : ''; + var players = unit.kind === 'world' ? esc(unit.players) + (unit.moved ? ' (' + esc(unit.moved) + ' moved)' : '') : ''; + return '' + what + '' + instance + '' + players + '' + fmt.badge(stateNames[unit.state] || unit.state, stateColors[unit.state] || 'secondary') + + '' + esc(unit.message || '') + ''; + }); + document.getElementById('liveUpdateRows').innerHTML = rows.join(''); + document.getElementById('liveUpdateTableWrap').classList.toggle('d-none', rows.length === 0); + }; + document.getElementById('liveUpdateStart').addEventListener('click', function () { + if (!confirm('Start a live update onto the binaries on disk now? Everyone online is moved to new instances (a short loading screen), and the dashboard restarts at the end.')) return; + var warn = document.getElementById('liveUpdateWarn').value; + api.action('/api/server/live_update', warn === '' ? {} : { warnSeconds: parseInt(warn, 10) }, 'Live update requested').catch(function () {}); + }); + document.getElementById('liveUpdateCancel').addEventListener('click', function () { + if (!confirm('Cancel the live update? Nothing new is started; what is under way finishes.')) return; + api.action('/api/server/live_update/cancel', {}, 'Cancelling').catch(function () {}); + }); + var loadLiveUpdate = function () { api.get('/api/server/live_update').then(function (d) { if (d.success) showLiveUpdate(d.liveUpdate); }).catch(function () {}); }; + loadLiveUpdate(); + if (window.Live) Live.onTopic('live_update', showLiveUpdate); + } + // Staff get the detailed list (address, which property a clone is); everyone else the basic one from the status var detailedWorlds = DASH.can('players_view'); // Worlds master launched that haven't connected yet, and those shutting down function stateBadge(row) { if (row.state === 'starting') return ' ' + fmt.badge('Starting', 'info'); if (row.state === 'stopping') return ' ' + fmt.badge('Shutting down', 'secondary'); + if (row.state === 'draining') return ' ' + fmt.badge('Moving players', 'warning'); return ''; } function withPending(data) { diff --git a/dGame/UserManager.h b/dGame/UserManager.h index e5bfa7be3..e6603a95e 100644 --- a/dGame/UserManager.h +++ b/dGame/UserManager.h @@ -47,6 +47,8 @@ public: void SaveAllActiveCharacters(); size_t GetUserCount() const { return m_Users.size(); } + // Everyone connected, by address (character selection moves them one by one in a live update) + const std::map& GetUsers() const { return m_Users; } // Access cached config values bool GetMuteAutoRejectNames() const { return m_MuteAutoRejectNames; } diff --git a/dGame/dComponents/PropertyManagementComponent.cpp b/dGame/dComponents/PropertyManagementComponent.cpp index 4befcd900..8088eb9fc 100644 --- a/dGame/dComponents/PropertyManagementComponent.cpp +++ b/dGame/dComponents/PropertyManagementComponent.cpp @@ -117,6 +117,8 @@ bool PropertyManagementComponent::IsBestFriend(const LWOOBJID player) const { } bool PropertyManagementComponent::CanBuild(const Entity& player) const { + // Handed off to a new instance: building happens there + if (frozen) return false; const bool bestFriendsBuild = PropertyBuilders::BestFriendsBuild(); PropertyBuilders::Player state{ .id = player.GetObjectID(), .isBuilding = builders.contains(player.GetObjectID()) }; // Only a best friend needs the lookup @@ -267,7 +269,7 @@ void PropertyManagementComponent::UpdatePropertyDetails(std::string name, std::s } bool PropertyManagementComponent::Claim(const LWOOBJID playerId) { - if (owner != LWOOBJID_EMPTY) { + if (owner != LWOOBJID_EMPTY || frozen) { return false; } @@ -510,6 +512,8 @@ void PropertyManagementComponent::SendModelsOnProperty() const { } void PropertyManagementComponent::UpdateModelPosition(Entity& builder, const LWOOBJID id, const NiPoint3 position, NiQuaternion rotation) { + // Handed off to a new instance (live update): nothing changes here any more + if (frozen) return; LOG("Placing model <%f, %f, %f>", position.x, position.y, position.z); auto* entity = &builder; @@ -560,6 +564,7 @@ void PropertyManagementComponent::UpdateModelPosition(Entity& builder, const LWO } void PropertyManagementComponent::DeleteModel(Entity& builder, const LWOOBJID id, const int deleteReason) { + if (frozen) return; LOG("Delete model: (%llu) (%i)", id, deleteReason); auto* model = Game::entityManager->GetEntity(id); @@ -761,6 +766,9 @@ void PropertyManagementComponent::Load() { } void PropertyManagementComponent::Save() { + // Handed off to a new instance (live update): what it saves is what counts + if (frozen) return; + // From the property's own ids: it saves whoever is here (the owner may not be) if (propertyId == LWOOBJID_EMPTY || owner == LWOOBJID_EMPTY) { return; @@ -833,6 +841,29 @@ void PropertyManagementComponent::Save() { DashboardNotify::Changed("properties", propertyId); } +void PropertyManagementComponent::FreezeForHandOff() { + if (frozen) return; + // Everyone still building stops (their models are kept as they are now), and nobody can start again + std::vector building(builders.begin(), builders.end()); + for (const auto id : building) { + auto* builder = Game::entityManager->GetEntity(id); + if (builder) EndBuilding(*builder); + else builders.erase(id); + } + Save(); + frozen = true; + // Takes the build rights the clients were given away (CanBuild is false for everyone now) + UpdateBuildRights(); + LOG("Property %llu (clone %u) saved and handed off to a new instance", propertyId, clone_Id); +} + +void PropertyManagementComponent::Unfreeze() { + if (!frozen) return; + frozen = false; + UpdateBuildRights(); + LOG("Property %llu (clone %u) is no longer handed off", propertyId, clone_Id); +} + void PropertyManagementComponent::AddModel(LWOOBJID modelId, LWOOBJID spawnerId) { models[modelId] = spawnerId; } diff --git a/dGame/dComponents/PropertyManagementComponent.h b/dGame/dComponents/PropertyManagementComponent.h index 217419506..9377d094c 100644 --- a/dGame/dComponents/PropertyManagementComponent.h +++ b/dGame/dComponents/PropertyManagementComponent.h @@ -203,6 +203,21 @@ public: void OnChatMessageReceived(const std::string& sMessage) const; + + /** + * Live updates (docs/LiveUpdate.md): a new instance of this property is about to load it from the database. Takes + * everyone out of build mode, saves, and from then on nobody can build or claim it here and it is never saved again, + * so the new instance's saves can't be overwritten by this one. + */ + void FreezeForHandOff(); + + // The hand-off was called off before anyone went to the new instance: building and saving work again + void Unfreeze(); + + bool IsFrozen() const noexcept { return frozen; } + + // Players in build mode here now + size_t GetBuilderCount() const noexcept { return builders.size(); } private: // Sends the property's data to one player; a player who can build is told they own it, which is the client's only // check before editing (the owner's name stays the owner's) @@ -219,6 +234,9 @@ private: */ static PropertyManagementComponent* instance; + // Handed off to a new instance for a live update (FreezeForHandOff) + bool frozen = false; + /** * The ID of the owner of this property */ diff --git a/dGame/dUtilities/SlashCommandHandler.cpp b/dGame/dUtilities/SlashCommandHandler.cpp index 0efd0f85e..c451a6db6 100644 --- a/dGame/dUtilities/SlashCommandHandler.cpp +++ b/dGame/dUtilities/SlashCommandHandler.cpp @@ -1714,6 +1714,15 @@ void SlashCommandHandler::Startup() { .requiredLevel = eGameMasterLevel::DEVELOPER }); + RegisterCommand({ + .help = "Moves the whole server onto the binaries on disk now", + .info = "A live update: with a new build in place, master restarts the UGC, auth and chat servers, replaces every world instance with a new one and moves its players there (with the game's Mythran maintenance warning and a short loading screen), then restarts the dashboard. Master itself keeps running. Usage: /liveupdate [start [warn seconds, 0-300] | cancel | status]", + .aliases = {"liveupdate"}, + .handle = WorldMigration::LiveUpdateCommand, + .requiredLevel = eGameMasterLevel::OPERATOR, + .dashboardPermission = "server_live_update" + }); + RegisterCommand({ .help = "[claim] Community challenges and live events", .info = "Shows the server-wide community challenges running now, how far along they are and what you added, and the live events in this world. Also gives you the coins waiting from challenges you helped complete", diff --git a/dGame/dUtilities/WorldMigration.cpp b/dGame/dUtilities/WorldMigration.cpp index b9c80af08..fb855c0b5 100644 --- a/dGame/dUtilities/WorldMigration.cpp +++ b/dGame/dUtilities/WorldMigration.cpp @@ -20,6 +20,9 @@ #include "MessageType/Game.h" #include "MessageType/Master.h" #include "PetComponent.h" +#include "PropertyManagementComponent.h" +#include "ControllablePhysicsComponent.h" +#include "UserManager.h" #include "PlayerManager.h" #include "ServiceType.h" #include "TradingManager.h" @@ -30,8 +33,6 @@ using namespace InstanceMigration; namespace { - // Players who are dead or building wait this long to finish before they go anyway - constexpr float MAX_WAIT_SECONDS = 15.0f; // A client that got its transfer and is still connected after this is dropped (it is saved already) constexpr float LEAVE_TIMEOUT_SECONDS = 20.0f; // Nobody left to move for this long: done (players still loading in finish first) @@ -60,6 +61,17 @@ namespace { }; std::optional g_Active; + + // A property being saved before its new instance loads it (MIGRATE_PREPARE) + struct Preparing { + uint32_t migrationId{}; + float waitLeft{}; + bool told{}; + }; + std::optional g_Preparing; + // The migration this property was frozen for (0: not frozen) + uint32_t g_FrozenFor = 0; + std::string g_LiveUpdateTold; // the last live update summary the GM who asked was told std::map g_Leaving; std::map> g_Carried; std::function g_Cleanup; @@ -93,14 +105,38 @@ namespace { announcement.SendToClient(player->GetSystemAddress()); } + // Character selection has users but no player entities + bool IsCharacterSelect() { + return Game::server->GetZoneID() == 0; + } + uint16_t Remaining() { uint16_t remaining = 0; + if (IsCharacterSelect()) { + for (const auto& [sysAddr, user] : UserManager::Instance()->GetUsers()) { + if (user && !g_Leaving.contains(sysAddr)) remaining++; + } + return remaining; + } for (auto* player : PlayerManager::GetAllPlayers()) { if (player && !g_Leaving.contains(player->GetSystemAddress())) remaining++; } return remaining; } + // Someone at character selection: nothing to save, they just connect to the new one, which sends them their + // characters again + void MoveUser(const SystemAddress& sysAddr) { + const auto& order = g_Active->order; + g_Leaving[sysAddr] = { LWOOBJID_EMPTY, 0.0f }; + ClientPackets::TransferToWorld transfer; + transfer.serverIP = LUString(order.targetIp); + transfer.serverPort = order.targetPort; + transfer.mythranShift = order.mythranShift; + transfer.Send(sysAddr); + LOG("Migration %u: sent %s from character selection to instance %u", order.migrationId, sysAddr.ToString(), order.targetInstance); + } + // Save, lock and send one player over. Returns false when they couldn't be sent. bool MovePlayer(Entity* player) { auto* character = player->GetCharacter(); @@ -120,6 +156,19 @@ namespace { carried.characterId = playerId; carried.seamless = order.seamless; if (auto* pet = PetComponent::GetActivePet(playerId)) carried.petItemId = pet->GetItemId(); + // Where they stand: the saved character doesn't keep it on properties (or Moon Base) + if (auto* physics = player->GetComponent()) { + const auto& position = physics->GetPosition(); + const auto& rotation = physics->GetRotation(); + carried.hasPosition = true; + carried.x = position.x; + carried.y = position.y; + carried.z = position.z; + carried.rotW = rotation.w; + carried.rotX = rotation.x; + carried.rotY = rotation.y; + carried.rotZ = rotation.z; + } // Same zone, so they spawn where they stood (Entity.cpp only uses a spawn point when the zone changes) character->SetZoneID(order.targetZone); @@ -153,6 +202,19 @@ namespace { void MovePlayers(float deltaTime) { auto& active = *g_Active; active.budget = std::min(active.budget + active.order.playersPerSecond * deltaTime, static_cast(active.order.playersPerSecond)); + if (IsCharacterSelect()) { + std::vector users; + for (const auto& [sysAddr, user] : UserManager::Instance()->GetUsers()) { + if (user && !g_Leaving.contains(sysAddr)) users.push_back(sysAddr); + } + for (const auto& sysAddr : users) { + if (active.budget < 1.0f) break; + active.budget -= 1.0f; + MoveUser(sysAddr); + active.moved++; + } + return; + } // Copy: moving a player can change the list (a cancelled trade, a pet) const auto players = PlayerManager::GetAllPlayers(); for (auto* player : players) { @@ -161,7 +223,7 @@ namespace { auto* character = player->GetCharacter(); const bool inTrade = TradingManager::Instance()->GetPlayerTrade(player->GetObjectID()) != nullptr; auto& waited = active.waiting[player->GetObjectID()]; - const auto decision = DecidePlayer(inTrade, character && character->GetBuildMode(), player->GetIsDead(), waited, MAX_WAIT_SECONDS); + const auto decision = DecidePlayer(inTrade, character && character->GetBuildMode(), player->GetIsDead(), waited, active.order.maxWaitSeconds); if (decision == ePlayerDecision::WAIT) { waited += deltaTime; continue; @@ -195,6 +257,15 @@ void WorldMigration::SetCleanupHandler(std::function void WorldMigration::HandleOrder(const MigratePlayersOrder& order) { if (order.targetPort == 0) { + if (g_Preparing && g_Preparing->migrationId == order.migrationId) { + LOG("Migration %u cancelled while the property was being saved", order.migrationId); + g_Preparing.reset(); + } + // A property nobody went to the new instance of is ours again; once someone did, it stays with them + if (g_FrozenFor == order.migrationId && (!g_Active || g_Active->moved == 0)) { + if (auto* property = PropertyManagementComponent::Instance()) property->Unfreeze(); + g_FrozenFor = 0; + } if (!g_Active || g_Active->order.migrationId != order.migrationId) return; LOG("Migration %u cancelled; %u player(s) were moved", order.migrationId, g_Active->moved); SendStatus(eState::FAILED, Remaining(), "Cancelled"); @@ -219,6 +290,27 @@ void WorldMigration::HandleOrder(const MigratePlayersOrder& order) { } } +void WorldMigration::HandlePrepare(const MigratePrepare& prepare) { + if (g_Active || g_Preparing) { + LOG("Migration %u refused: another migration is running here", prepare.migrationId); + SendStatus(prepare.migrationId, 0, eState::FAILED, 0, 0, Remaining(), "Another migration is running on this instance"); + return; + } + g_Preparing = Preparing{ prepare.migrationId, static_cast(prepare.maxWaitSeconds), false }; + LOG("Migration %u: saving this instance's property for its replacement (waiting up to %u s for builders)", prepare.migrationId, prepare.maxWaitSeconds); +} + +void WorldMigration::ApplyCarriedPosition(Entity* player) { + if (!player) return; + const auto it = g_Carried.find(player->GetObjectID()); + if (it == g_Carried.end() || !it->second.first.hasPosition) return; + const auto& state = it->second.first; + auto* physics = player->GetComponent(); + if (!physics) return; + physics->SetPosition(NiPoint3(state.x, state.y, state.z)); + physics->SetRotation(NiQuaternion(state.rotW, state.rotX, state.rotY, state.rotZ)); +} + void WorldMigration::StoreCarriedState(const CarriedPlayerState& state) { if (state.targetZone != Game::server->GetZoneID() || state.targetInstance != Game::server->GetInstanceID()) return; g_Carried[state.characterId] = { state, 0.0f }; @@ -231,6 +323,28 @@ void WorldMigration::Update(float deltaTime) { else ++it; } DropStuckClients(deltaTime); + + if (g_Preparing) { + auto* property = PropertyManagementComponent::Instance(); + if (property && property->GetBuilderCount() > 0 && g_Preparing->waitLeft > 0.0f) { + if (!g_Preparing->told) { + g_Preparing->told = true; + const auto seconds = std::to_string(static_cast(g_Preparing->waitLeft)); + for (auto* player : PlayerManager::GetAllPlayers()) { + if (player) ChatPackets::SendSystemMessage(player->GetSystemAddress(), + GeneralUtils::ASCIIToUTF16("The server is being updated: building on this property ends in " + seconds + " seconds.")); + } + } + g_Preparing->waitLeft -= deltaTime; + } else { + const auto migrationId = g_Preparing->migrationId; + g_Preparing.reset(); + if (property) property->FreezeForHandOff(); + g_FrozenFor = migrationId; + SendStatus(migrationId, 0, eState::PREPARED, 0, 0, Remaining(), property ? "Property saved" : "Nothing to save"); + } + } + if (!g_Active) return; auto& active = *g_Active; @@ -401,3 +515,61 @@ void WorldMigration::MergeInstanceCommand(Entity* entity, const SystemAddress& s (target ? u"instance " + GeneralUtils::ASCIIToUTF16(std::to_string(target)) : std::u16string(u"the best fit")) + u"."); RequestMigration(entity, eKind::MERGE, target, warnSeconds, seamless); } + +namespace { + std::string LiveUpdateSummary(const LiveUpdateStatus& status) { + using namespace LiveUpdate; + size_t done = 0, failed = 0; + for (const auto& unit : status.units) { + if (IsFinished(unit.state)) done++; + if (unit.state == eUnitState::FAILED) failed++; + } + std::string text = "Live update " + std::to_string(status.updateId) + ": " + PhaseName(status.phase); + if (!status.units.empty()) text += ", " + std::to_string(done) + " of " + std::to_string(status.units.size()) + " done"; + if (failed) text += ", " + std::to_string(failed) + " failed"; + if (!status.message.empty()) text += ". " + status.message; + return text; + } +} + +void WorldMigration::LiveUpdateCommand(Entity* entity, const SystemAddress& sysAddr, const std::string args) { + const auto words = GeneralUtils::SplitString(args, ' '); + const std::string verb = words.empty() || words[0].empty() ? "status" : words[0]; + LiveUpdateRequest request; + if (verb == "start") { + request.action = LiveUpdate::eAction::START; + if (words.size() > 1) { + const auto warn = GeneralUtils::TryParse(words[1]); + if (!warn || *warn < 0 || *warn > InstanceMigrationRequest::MAX_WARN_SECONDS) { + ChatPackets::SendSystemMessage(sysAddr, u"Usage: /liveupdate start [warn seconds, 0-300]"); + return; + } + request.warnSeconds = *warn; + } + } else if (verb == "cancel") { + request.action = LiveUpdate::eAction::CANCEL; + } else if (verb == "status") { + request.action = LiveUpdate::eAction::STATUS; + } else { + ChatPackets::SendSystemMessage(sysAddr, u"Usage: /liveupdate [start [warn seconds] | cancel | status]"); + return; + } + if (entity) { + request.requesterId = entity->GetObjectID(); + if (auto* character = entity->GetCharacter()) request.requestedBy = character->GetName(); + } + g_LiveUpdateTold.clear(); + LOG("Asking master to %s a live update (requested by %s)", verb.c_str(), request.requestedBy.c_str()); + MasterPackets::SendToMaster(request); + if (request.action == LiveUpdate::eAction::START) ChatPackets::SendSystemMessage(sysAddr, u"Asking master to move everything onto the binaries on disk now."); +} + +void WorldMigration::HandleLiveUpdateStatus(const LiveUpdateStatus& status) { + if (status.requesterId == LWOOBJID_EMPTY) return; + auto* requester = PlayerManager::GetPlayer(status.requesterId); + if (!requester) return; + const auto text = LiveUpdateSummary(status); + if (text == g_LiveUpdateTold) return; + g_LiveUpdateTold = text; + ChatPackets::SendSystemMessage(requester->GetSystemAddress(), GeneralUtils::ASCIIToUTF16(text)); +} diff --git a/dGame/dUtilities/WorldMigration.h b/dGame/dUtilities/WorldMigration.h index dbecc8328..d5d7e85f2 100644 --- a/dGame/dUtilities/WorldMigration.h +++ b/dGame/dUtilities/WorldMigration.h @@ -4,6 +4,7 @@ #include #include "master/InstanceMigration.h" +#include "master/LiveUpdate.h" #include "RakNetTypes.h" #include "dCommonVars.h" @@ -28,6 +29,12 @@ namespace WorldMigration { // MIGRATE_PLAYER_STATE from master: a player on their way here void StoreCarriedState(const CarriedPlayerState& state); + // MIGRATE_PREPARE from master (live update of a property): wait for builders, save the property and freeze it + void HandlePrepare(const MigratePrepare& prepare); + + // The player entity of someone moved here was just made: put them where they stood (properties don't save it) + void ApplyCarriedPosition(Entity* player); + // Every world frame void Update(float deltaTime); @@ -59,6 +66,12 @@ namespace WorldMigration { // /mergeinstance [target instance, 0 for the best fit] [warn seconds] [seamless] void MergeInstanceCommand(Entity* entity, const SystemAddress& sysAddr, const std::string args); + + // /liveupdate [start [warn seconds] | cancel | status]: a live update of the whole server (docs/LiveUpdate.md) + void LiveUpdateCommand(Entity* entity, const SystemAddress& sysAddr, const std::string args); + + // LIVE_UPDATE_STATUS from master: tell the GM who asked, if they are here + void HandleLiveUpdateStatus(const LiveUpdateStatus& status); } #endif //!__WORLDMIGRATION__H__ diff --git a/dMasterServer/CMakeLists.txt b/dMasterServer/CMakeLists.txt index 02e9ec12f..fcad65779 100644 --- a/dMasterServer/CMakeLists.txt +++ b/dMasterServer/CMakeLists.txt @@ -1,6 +1,7 @@ set(DMASTERSERVER_SOURCES "InstanceManager.cpp" "MigrationCoordinator.cpp" + "LiveUpdateCoordinator.cpp" "Start.cpp" ) diff --git a/dMasterServer/InstanceManager.cpp b/dMasterServer/InstanceManager.cpp index 62e405dfa..87b5511a6 100644 --- a/dMasterServer/InstanceManager.cpp +++ b/dMasterServer/InstanceManager.cpp @@ -240,7 +240,7 @@ const InstancePtr& InstanceManager::GetInstanceBySysAddr(const SystemAddress& sy const InstancePtr& InstanceManager::FindInstance(LWOMAPID mapID, bool isFriendTransfer, LWOCLONEID cloneId) { for (const auto& i : m_Instances) { - if (i && i->GetMapID() == mapID && i->GetCloneID() == cloneId && !i->IsFull(isFriendTransfer) && !i->GetIsPrivate() && !i->GetShutdownComplete() && !i->GetIsShuttingDown() && !i->GetIsDraining()) { + if (i && InstanceMigration::AcceptsNewPlayers(i->View(), mapID, cloneId, isFriendTransfer)) { return i; } } @@ -304,7 +304,8 @@ const InstancePtr& InstanceManager::FindPrivateInstance(const std::string& passw for (const auto& instance : m_Instances) { if (!instance) continue; - if (!instance->GetIsPrivate()) { + // A private instance being replaced (live update) takes nobody new: its replacement has the same password + if (!instance->GetIsPrivate() || instance->GetIsDraining() || instance->GetIsShuttingDown() || instance->GetShutdownComplete()) { continue; } diff --git a/dMasterServer/InstanceManager.h b/dMasterServer/InstanceManager.h index 4b3d2445e..fe94c8422 100644 --- a/dMasterServer/InstanceManager.h +++ b/dMasterServer/InstanceManager.h @@ -11,6 +11,7 @@ #include "Stamps.h" #include "dZMCommon.h" #include "Logger.h" +#include "master/InstanceMigration.h" struct Player { LWOOBJID id; @@ -98,6 +99,23 @@ public: bool IsFull(bool isFriendTransfer) const; + // What instance migrations and live updates plan with + InstanceMigration::InstanceView View() const { + InstanceMigration::InstanceView view; + view.zoneId = GetMapID(); + view.instanceId = GetInstanceID(); + view.cloneId = GetCloneID(); + view.players = GetCurrentClientCount(); + view.softCap = GetSoftCap(); + view.hardCap = GetHardCap(); + view.reserved = GetReserved(); + view.ready = GetIsReady(); + view.isPrivate = GetIsPrivate(); + view.shuttingDown = GetIsShuttingDown() || GetShutdownComplete(); + view.draining = GetIsDraining(); + return view; + } + private: std::string m_IP{}; uint32_t m_Port{}; diff --git a/dMasterServer/LiveUpdateCoordinator.cpp b/dMasterServer/LiveUpdateCoordinator.cpp new file mode 100644 index 000000000..0d5fddc5e --- /dev/null +++ b/dMasterServer/LiveUpdateCoordinator.cpp @@ -0,0 +1,315 @@ +#include "LiveUpdateCoordinator.h" + +#include +#include +#include +#include + +#include "BinaryPathFinder.h" +#include "CDActivitiesTable.h" +#include "CDClientManager.h" +#include "Database.h" +#include "Game.h" +#include "GeneralUtils.h" +#include "InstanceManager.h" +#include "Logger.h" +#include "MigrationCoordinator.h" +#include "MigrationRunner.h" +#include "dConfig.h" + +using namespace LiveUpdate; + +namespace { + // How often the machine runs (master runs at 30 frames a second; nothing here needs that) + constexpr auto TICK_INTERVAL = std::chrono::milliseconds(250); + // Live update migrations get IDs of their own, far from the worlds' (time-based) ones + constexpr uint32_t MIGRATION_ID_BASE = 0xC0000000; + + LiveUpdateCoordinator::Hooks g_Hooks; + Machine g_Machine; + LiveUpdateStatus g_Status; // the last one sent + uint32_t g_UpdateId = 0; + uint32_t g_NextMigrationId = 0; + Clock::time_point g_NextTick{}; + std::map> g_Logged; // unit index -> what was logged last + ePhase g_LoggedPhase = ePhase::IDLE; + + int64_t Now() { + return static_cast(std::time(nullptr)); + } + + template + T Setting(const std::string& key, T fallback) { + return GeneralUtils::TryParse(Game::config->GetValue(key)).value_or(fallback); + } + + std::set KeepZones() { + std::set zones; + for (auto part : GeneralUtils::SplitString(Game::config->GetValue("prestart_worlds", "0,1000"), ',')) { + std::erase_if(part, [](const char c) { return std::isspace(static_cast(c)); }); + if (const auto zone = GeneralUtils::TryParse(part)) zones.insert(*zone); + } + zones.insert(0); + return zones; + } + + Settings ReadSettings(int32_t warnSeconds) { + Settings settings; + settings.warnSeconds = warnSeconds >= 0 ? static_cast(warnSeconds) : + std::min(Setting("live_update_warn_seconds", 10), InstanceMigrationRequest::MAX_WARN_SECONDS); + settings.parallelWorlds = std::max(Setting("live_update_parallel_worlds", 4), 1); + settings.serviceTimeout = std::chrono::seconds(std::max(Setting("live_update_service_timeout", 30), 10)); + settings.ugcDrainTimeout = std::chrono::seconds(Setting("live_update_ugc_drain_timeout", 300)); + settings.charSelectWait = std::chrono::seconds(Setting("live_update_char_select_wait", 60)); + settings.activityWait = std::chrono::seconds(Setting("live_update_activity_wait", 1800)); + settings.playerWaitSeconds = std::min(Setting("live_update_player_wait", 30), MigratePlayersOrder::MAX_MAX_WAIT_SECONDS); + settings.propertyBuildWaitSeconds = std::min(Setting("live_update_property_build_wait", 60), MigratePrepare::MAX_WAIT_SECONDS); + settings.runMigrations = Game::config->GetValue("live_update_run_migrations") != "0"; + settings.keepZones = KeepZones(); + return settings; + } + + // Races, minigames and other activities run in their own zones with state that only lives in that world + bool IsActivityZone(uint32_t zone) { + static std::map cache; + if (const auto it = cache.find(zone); it != cache.end()) return it->second; + auto* activities = CDClientManager::GetTable(); + const bool activity = activities && !activities->Query([zone](const CDActivities& row) { return row.instanceMapID == zone; }).empty(); + return cache[zone] = activity; + } + + Observed Observe() { + Observed observed; + observed.ugc = g_Hooks.service(eService::UGC); + observed.auth = g_Hooks.service(eService::AUTH); + observed.chat = g_Hooks.service(eService::CHAT); + observed.dashboard = g_Hooks.service(eService::DASHBOARD); + for (const auto& instance : Game::im->GetInstances()) { + if (!instance) continue; + observed.worlds.push_back({ instance->View(), IsActivityZone(instance->GetMapID()) }); + } + return observed; + } + + class Actions final : public IActions { + public: + bool RunMigrations(std::string& error) override { + LOG("Live update: running the new build's database migrations"); + try { + MigrationRunner::RunMigrations(); + MigrationRunner::RunSQLiteMigrations(); + Database::Get()->Commit(); + } catch (const std::exception& ex) { + error = std::string("The database migrations failed: ") + ex.what(); + LOG("Live update: %s", error.c_str()); + return false; + } + return true; + } + + void RetireService(eService service) override { + LOG("Live update: retiring the %s", ServiceName(service)); + g_Hooks.retire(service); + } + + void StopService(eService service) override { + LOG("Live update: stopping the %s", ServiceName(service)); + g_Hooks.stop(service); + } + + void StartService(eService service) override { + LOG("Live update: starting the %s", ServiceName(service)); + g_Hooks.start(service); + } + + void ChatReady() override { + LOG("Live update: the new chat server is up; telling the worlds"); + g_Hooks.chatReady(); + } + + std::optional StartWorld(uint32_t zone) override { + const auto& started = Game::im->StartNewInstance(static_cast(zone), 0); + if (!started) return std::nullopt; + LOG("Live update: started instance %u of zone %u", started->GetInstanceID(), zone); + return started->GetInstanceID(); + } + + void SetDraining(uint32_t zone, uint32_t instance, bool draining) override { + const auto& found = Game::im->FindInstanceWithPrivate(static_cast(zone), static_cast(instance)); + if (!found) return; + LOG("Live update: zone %u instance %u %s", zone, instance, draining ? "takes no new players" : "takes players again"); + found->SetIsDraining(draining); + } + + std::optional Migrate(const Move& move, std::string& refusal) override { + InstanceMigrationRequest request; + request.requestId = MIGRATION_ID_BASE | (++g_NextMigrationId & 0x3FFFFFFF); + request.kind = move.kind; + request.zoneId = move.zone; + request.sourceInstance = move.instance; + request.targetInstance = move.target; + request.warnSeconds = move.warnSeconds; + request.shutdownSource = true; + request.requestedBy = "live update"; + MigrationCoordinator::Options options; + options.liveUpdate = true; + options.prepare = move.prepare; + options.prepareWaitSeconds = move.prepareWaitSeconds; + options.playerWaitSeconds = move.playerWaitSeconds; + const auto result = MigrationCoordinator::Start(request, options); + if (result != InstanceMigration::eRefusal::NONE) { + refusal = InstanceMigration::Describe(result); + return std::nullopt; + } + return request.requestId; + } + + void StopWorld(uint32_t zone, uint32_t instance) override { + const auto& found = Game::im->FindInstanceWithPrivate(static_cast(zone), static_cast(instance)); + if (found) found->Shutdown(); + } + }; + + Actions g_Actions; + + void LogChanges() { + const auto& units = g_Machine.Units(); + for (size_t i = 0; i < units.size(); i++) { + const auto& unit = units[i]; + auto& logged = g_Logged[i]; + if (logged.first == unit.state && logged.second == unit.message) continue; + logged = { unit.state, unit.message }; + if (unit.kind == eUnitKind::WORLD) { + LOG("Live update %u: zone %u clone %u instance %u%s: %s%s%s", g_UpdateId, unit.zone, unit.clone, unit.instance, + unit.replacement ? (" -> " + std::to_string(unit.replacement)).c_str() : "", StateName(unit.state), + unit.message.empty() ? "" : ": ", unit.message.c_str()); + } else { + LOG("Live update %u: %s: %s%s%s", g_UpdateId, KindName(unit.kind), StateName(unit.state), unit.message.empty() ? "" : ": ", unit.message.c_str()); + } + } + if (g_LoggedPhase != g_Machine.Phase()) { + g_LoggedPhase = g_Machine.Phase(); + LOG("Live update %u: %s%s%s", g_UpdateId, PhaseName(g_LoggedPhase), g_Machine.Message().empty() ? "" : ": ", g_Machine.Message().c_str()); + } + } + + void Publish(bool phaseOrFinishChanged) { + g_Machine.FillStatus(g_Status); + g_Status.updateId = g_UpdateId; + if (IsFinished(g_Status.phase) && g_Status.finishedAt == 0) g_Status.finishedAt = Now(); + LogChanges(); + if (g_Hooks.publish) g_Hooks.publish(g_Status, phaseOrFinishChanged); + } + + size_t FinishedUnits() { + return static_cast(std::count_if(g_Machine.Units().begin(), g_Machine.Units().end(), [](const Unit& unit) { return IsFinished(unit.state); })); + } + + // The binaries the update starts; a build still being written would fail to start + bool CheckBinaries(std::string& error) { + const auto dir = BinaryPathFinder::GetBinaryDir(); + std::vector names{ "WorldServer", "AuthServer", "ChatServer" }; + if (Game::config->GetValue("enable_dashboard") == "1") names.push_back("DashboardServer"); + if (Game::config->GetValue("enable_ugc_server") == "1") names.push_back("UgcServer"); + for (auto name : names) { +#ifdef _WIN32 + name += ".exe"; +#endif + std::error_code ec; + const auto path = dir / name; + if (!std::filesystem::is_regular_file(path, ec) || std::filesystem::file_size(path, ec) == 0) { + error = name + " is missing from " + dir.string() + " (is a build still running?)"; + return false; + } + } + return true; + } +} + +void LiveUpdateCoordinator::Initialize(Hooks hooks) { + g_Hooks = std::move(hooks); +} + +bool LiveUpdateCoordinator::Start(const std::string& by, LWOOBJID requesterId, int32_t warnSeconds, std::string& error) { + if (g_Machine.IsRunning()) { + error = "A live update is already running"; + return false; + } + if (!CheckBinaries(error)) return false; + const auto settings = ReadSettings(warnSeconds); + if (!g_Machine.Start(settings, Observe(), Clock::now(), error)) return false; + g_UpdateId++; + g_Logged.clear(); + g_LoggedPhase = ePhase::IDLE; + g_Status = LiveUpdateStatus{}; + g_Status.startedAt = Now(); + g_Status.by = by; + g_Status.requesterId = requesterId; + g_NextTick = {}; + LOG("Live update %u started by %s: %zu world instance(s), warn %u s, %u at a time", g_UpdateId, by.c_str(), + g_Machine.Units().size() - 5, settings.warnSeconds, settings.parallelWorlds); + Publish(true); + return true; +} + +void LiveUpdateCoordinator::HandleRequest(const LiveUpdateRequest& request) { + switch (request.action) { + case eAction::START: { + std::string error; + const auto by = request.requestedBy.empty() ? std::string("unknown") : request.requestedBy; + if (!Start(by, request.requesterId, request.warnSeconds, error)) { + LOG("Live update requested by %s refused: %s", by.c_str(), error.c_str()); + // The status tells whoever asked why (the running one's, or the last one's with the reason) + auto status = Status(); + status.requesterId = request.requesterId; + status.message = error; + if (g_Hooks.publish) g_Hooks.publish(status, true); + } + return; + } + case eAction::CANCEL: + if (g_Machine.Cancel()) { + LOG("Live update %u cancelled by %s", g_UpdateId, request.requestedBy.c_str()); + Publish(true); + } + return; + case eAction::STATUS: { + auto status = Status(); + if (request.requesterId != LWOOBJID_EMPTY) status.requesterId = request.requesterId; + if (g_Hooks.publish) g_Hooks.publish(status, request.requesterId != LWOOBJID_EMPTY); + return; + } + } +} + +void LiveUpdateCoordinator::OnMigrationStatus(const MigrationStatus& status) { + if (!g_Machine.IsRunning()) return; + g_Machine.OnMigration(status); +} + +void LiveUpdateCoordinator::Update() { + if (!g_Machine.IsRunning()) return; + const auto now = Clock::now(); + if (now < g_NextTick) return; + g_NextTick = now + TICK_INTERVAL; + const auto phaseBefore = g_Machine.Phase(); + const auto finishedBefore = FinishedUnits(); + if (g_Machine.Tick(Observe(), now, g_Actions)) Publish(phaseBefore != g_Machine.Phase() || finishedBefore != FinishedUnits()); +} + +void LiveUpdateCoordinator::Abort(const std::string& why) { + if (!g_Machine.IsRunning()) return; + g_Machine.Abort(why); + Publish(true); +} + +bool LiveUpdateCoordinator::IsRunning() { + return g_Machine.IsRunning(); +} + +LiveUpdateStatus LiveUpdateCoordinator::Status() { + auto status = g_Status; + g_Machine.FillStatus(status); + status.updateId = g_UpdateId; + return status; +} diff --git a/dMasterServer/LiveUpdateCoordinator.h b/dMasterServer/LiveUpdateCoordinator.h new file mode 100644 index 000000000..2a22ee4bb --- /dev/null +++ b/dMasterServer/LiveUpdateCoordinator.h @@ -0,0 +1,48 @@ +#ifndef __LIVEUPDATECOORDINATOR__H__ +#define __LIVEUPDATECOORDINATOR__H__ + +#include +#include + +#include "LiveUpdateMachine.h" +#include "master/LiveUpdate.h" + +/** + * Master's side of live updates (docs/LiveUpdate.md): runs LiveUpdate::Machine against the instance manager, the + * instance migrations and the other servers. Started from the dashboard (LIVE_UPDATE_REQUEST), a GM's /liveupdate or + * SIGUSR2 to master. Main thread only. + */ +namespace LiveUpdateCoordinator { + // What MasterServer.cpp provides: the other servers, and where statuses go + struct Hooks { + std::function service; + std::function retire; // LIVE_UPDATE_RETIRE (chat, UGC) or SHUTDOWN (auth, dashboard) + std::function stop; // SHUTDOWN + std::function start; // start its process + std::function chatReady; // CHAT_SERVER_READY to every world + std::function publish; + }; + + void Initialize(Hooks hooks); + + // Start one; false (and why) when it can't + bool Start(const std::string& by, LWOOBJID requesterId, int32_t warnSeconds, std::string& error); + + // LIVE_UPDATE_REQUEST (the sender is checked by master: the dashboard, or a world for a GM) + void HandleRequest(const LiveUpdateRequest& request); + + // Every instance migration's progress (MigrationCoordinator's observer) + void OnMigrationStatus(const MigrationStatus& status); + + // Every master frame + void Update(); + + // Master is shutting down + void Abort(const std::string& why); + + bool IsRunning(); + + LiveUpdateStatus Status(); +} + +#endif //!__LIVEUPDATECOORDINATOR__H__ diff --git a/dMasterServer/LiveUpdateMachine.h b/dMasterServer/LiveUpdateMachine.h new file mode 100644 index 000000000..77b4f439e --- /dev/null +++ b/dMasterServer/LiveUpdateMachine.h @@ -0,0 +1,580 @@ +#ifndef __LIVEUPDATEMACHINE__H__ +#define __LIVEUPDATEMACHINE__H__ + +#include +#include +#include +#include +#include +#include +#include + +#include "master/InstanceMigration.h" +#include "master/LiveUpdate.h" + +/** + * The order of a live update (docs/LiveUpdate.md), without master's state: it sees the servers and instances master + * knows (Observed), asks for things to be done through IActions, and hears how instance migrations go (OnMigration). + * Master's LiveUpdateCoordinator runs it every frame. Header only, so it is unit tested without a master server. + * + * Order: database migrations; then the UGC server, auth and chat together; once chat is back, the world instances + * (a few at a time); the dashboard last, so it shows the progress until then. + */ +namespace LiveUpdate { + using Clock = std::chrono::steady_clock; + + enum class eService : uint8_t { UGC, AUTH, CHAT, DASHBOARD }; + + inline const char* ServiceName(eService service) { + switch (service) { + case eService::UGC: return "UGC server"; + case eService::AUTH: return "auth server"; + case eService::CHAT: return "chat server"; + case eService::DASHBOARD: return "dashboard"; + } + return "server"; + } + + struct Settings { + uint16_t warnSeconds{ 10 }; // players are warned this long before they are moved + uint32_t parallelWorlds{ 4 }; // world instances being replaced at once + std::chrono::seconds serviceTimeout{ 60 }; // a server to stop, or its new process to connect + std::chrono::seconds ugcDrainTimeout{ 300 }; // the UGC server finishing the jobs it is running + std::chrono::seconds charSelectWait{ 60 }; // players at character selection to pick a character + std::chrono::seconds activityWait{ 1800 }; // players in an activity zone (races, minigames) to finish + std::chrono::seconds worldStartTimeout{ 180 }; // a replacement instance to be ready + std::chrono::seconds worldTimeout{ 900 }; // backstop for one world's whole replacement + std::chrono::seconds chatSettle{ 3 }; // worlds to reconnect to the new chat server + uint16_t playerWaitSeconds{ 30 }; // dead or building players before they are moved anyway + uint16_t propertyBuildWaitSeconds{ 60 }; // builders on a property before it is saved anyway + bool runMigrations{ true }; + std::set keepZones{ 0, 1000 }; // zones that always have an instance (prestart_worlds) + uint32_t maxServiceStarts{ 3 }; // tries to start a server that didn't come back + }; + + struct ServiceView { + bool enabled{ true }; + bool online{}; + uint32_t connects{}; // counts every time one connected to master + }; + + struct WorldView { + InstanceMigration::InstanceView view; + bool activityZone{}; + }; + + struct Observed { + ServiceView ugc, auth, chat, dashboard; + std::vector worlds; + + const WorldView* Find(uint32_t zone, uint32_t instance) const { + for (const auto& world : worlds) { + if (world.view.zoneId == zone && world.view.instanceId == instance) return &world; + } + return nullptr; + } + + const ServiceView& Service(eService service) const { + switch (service) { + case eService::UGC: return ugc; + case eService::AUTH: return auth; + case eService::CHAT: return chat; + case eService::DASHBOARD: return dashboard; + } + return auth; + } + }; + + // What the machine has master do + class IActions { + public: + virtual ~IActions() = default; + // The new build's database migrations; false (and why) when they failed + virtual bool RunMigrations(std::string& error) = 0; + // Chat and UGC: LIVE_UPDATE_RETIRE; auth and the dashboard: SHUTDOWN. Master starts the new one when it disconnects. + virtual void RetireService(eService service) = 0; + // SHUTDOWN right away (a UGC server that took too long to finish its jobs) + virtual void StopService(eService service) = 0; + virtual void StartService(eService service) = 0; + // CHAT_SERVER_READY to every world + virtual void ChatReady() = 0; + // Start a new public instance of zone (clone 0); its instance ID, or none when master refused + virtual std::optional StartWorld(uint32_t zone) = 0; + virtual void SetDraining(uint32_t zone, uint32_t instance, bool draining) = 0; + + struct Move { + uint32_t zone{}; + uint32_t instance{}; + InstanceMigration::eKind kind{ InstanceMigration::eKind::REPLACE }; + uint32_t target{}; // merges only + uint16_t warnSeconds{}; + uint16_t playerWaitSeconds{}; + bool prepare{}; // properties: freeze and save before the new instance loads it + uint16_t prepareWaitSeconds{}; + }; + // An instance migration (MigrationCoordinator); its ID, or none (and why) when refused + virtual std::optional Migrate(const Move& move, std::string& refusal) = 0; + virtual void StopWorld(uint32_t zone, uint32_t instance) = 0; + }; + + // What happens to one world instance, decided when its turn comes + enum class eWorldPlan : uint8_t { + STOP, // nobody there: shut it down (a new one starts when someone goes there) + REPLACE_EMPTY, // nobody there, but the zone should always have one: start the new one, then stop this one + MOVE, // replace it and move its players (properties save and freeze first) + WAIT_THEN_MOVE, // activity zones and character selection: nobody new goes there; its players leave by + // themselves or, after a while, are moved + }; + + inline const char* PlanName(eWorldPlan plan) { + switch (plan) { + case eWorldPlan::STOP: return "stop"; + case eWorldPlan::REPLACE_EMPTY: return "replace_empty"; + case eWorldPlan::MOVE: return "move"; + case eWorldPlan::WAIT_THEN_MOVE: return "wait_then_move"; + } + return "unknown"; + } + + // zoneReplaced: this update already started a new instance of the zone (keeping one is then taken care of) + inline eWorldPlan PlanWorld(const InstanceMigration::InstanceView& view, bool activityZone, bool keepZone, bool zoneReplaced) { + if (view.players <= 0) { + if ((keepZone || view.zoneId == 0) && !view.isPrivate && view.cloneId == 0 && !zoneReplaced) return eWorldPlan::REPLACE_EMPTY; + return eWorldPlan::STOP; + } + if (view.zoneId == 0 || activityZone) return eWorldPlan::WAIT_THEN_MOVE; + return eWorldPlan::MOVE; + } + + struct Unit { + eUnitKind kind{}; + eUnitState state{ eUnitState::PENDING }; + eWorldPlan plan{ eWorldPlan::STOP }; + uint32_t zone{}; + uint32_t instance{}; + uint32_t clone{}; + bool isPrivate{}; + uint32_t replacement{}; + uint32_t players{}; + uint16_t moved{}; + uint32_t migrationId{}; + bool drained{}; // we set it draining (so we undo that if it fails) + uint32_t connectsAtStart{}; + uint32_t starts{}; + Clock::time_point deadline{}; + std::string message; + }; + + class Machine { + public: + /** + * Plans an update of what runs now. False (and why) when one is already running. Instances that are shutting + * down are left out; instances started later are the new build's already. + */ + bool Start(const Settings& settings, const Observed& observed, Clock::time_point now, std::string& error) { + if (m_Phase == ePhase::RUNNING || m_Phase == ePhase::CANCELLING) { + error = "A live update is already running"; + return false; + } + m_Settings = settings; + m_Units.clear(); + m_ReplacedZones.clear(); + m_Phase = ePhase::RUNNING; + m_Message.clear(); + m_Dirty = true; + m_Units.push_back(ServiceUnit(eUnitKind::DATABASE)); + m_Units.push_back(ServiceUnit(eUnitKind::UGC)); + m_Units.push_back(ServiceUnit(eUnitKind::AUTH)); + m_Units.push_back(ServiceUnit(eUnitKind::CHAT)); + // Character selection first (logins go to the new one soonest), then the busiest + std::vector worlds; + for (const auto& world : observed.worlds) { + if (!world.view.shuttingDown) worlds.push_back(world); + } + std::stable_sort(worlds.begin(), worlds.end(), [](const WorldView& a, const WorldView& b) { + if ((a.view.zoneId == 0) != (b.view.zoneId == 0)) return a.view.zoneId == 0; + return a.view.players > b.view.players; + }); + for (const auto& world : worlds) { + Unit unit; + unit.kind = eUnitKind::WORLD; + unit.zone = world.view.zoneId; + unit.instance = world.view.instanceId; + unit.clone = world.view.cloneId; + unit.isPrivate = world.view.isPrivate; + unit.players = static_cast(std::max(world.view.players, 0)); + m_Units.push_back(unit); + } + m_Units.push_back(ServiceUnit(eUnitKind::DASHBOARD)); + (void)now; + return true; + } + + // Start nothing new; what is under way finishes. False when nothing is running. + bool Cancel() { + if (m_Phase != ePhase::RUNNING) return false; + m_Phase = ePhase::CANCELLING; + for (auto& unit : m_Units) { + if (unit.state == eUnitState::PENDING) SetState(unit, eUnitState::SKIPPED, "Cancelled"); + } + m_Message = "Cancelled; waiting for what is under way"; + m_Dirty = true; + return true; + } + + // Master is shutting down: forget the rest + void Abort(const std::string& why) { + if (!IsRunning()) return; + for (auto& unit : m_Units) { + if (!IsFinished(unit.state)) SetState(unit, eUnitState::SKIPPED, why); + } + Finish(ePhase::CANCELLED, why); + } + + /** + * Moves things along. Master calls it every frame (with what it knows now); it returns whether anything changed + * (the status is sent again then). + */ + bool Tick(const Observed& observed, Clock::time_point now, IActions& actions) { + if (!IsRunning()) return TakeDirty(); + + auto& database = m_Units[0]; + if (database.state == eUnitState::PENDING) { + if (!m_Settings.runMigrations) { + SetState(database, eUnitState::SKIPPED, "Switched off (live_update_run_migrations)"); + } else { + std::string error; + if (actions.RunMigrations(error)) { + SetState(database, eUnitState::STOPPED, "Up to date"); + } else { + SetState(database, eUnitState::FAILED, error.empty() ? "The migrations failed" : error); + for (auto& unit : m_Units) { + if (unit.state == eUnitState::PENDING) SetState(unit, eUnitState::SKIPPED, "Not started: the database migrations failed"); + } + Finish(ePhase::FAILED, "The database migrations failed; nothing else was touched"); + return TakeDirty(); + } + } + } + + for (auto& unit : m_Units) { + switch (unit.kind) { + case eUnitKind::UGC: TickService(unit, eService::UGC, observed, now, actions); break; + case eUnitKind::AUTH: TickService(unit, eService::AUTH, observed, now, actions); break; + case eUnitKind::CHAT: TickService(unit, eService::CHAT, observed, now, actions); break; + default: break; + } + } + + // Worlds once chat is back (their players are sent to the new chat server first) + if (IsFinished(Find(eUnitKind::CHAT).state)) { + for (auto& unit : m_Units) { + if (unit.kind != eUnitKind::WORLD || IsFinished(unit.state)) continue; + if (unit.state == eUnitState::PENDING) { + if (m_Phase != ePhase::RUNNING || Busy() >= std::max(m_Settings.parallelWorlds, 1)) continue; + StartWorld(unit, observed, now, actions); + } else { + TickWorld(unit, observed, now, actions); + } + } + } + + // The dashboard last: it shows the progress until then + auto& dashboard = Find(eUnitKind::DASHBOARD); + const bool othersDone = std::all_of(m_Units.begin(), m_Units.end(), [](const Unit& unit) { + return unit.kind == eUnitKind::DASHBOARD || IsFinished(unit.state); + }); + if (othersDone && (dashboard.state != eUnitState::PENDING || m_Phase == ePhase::RUNNING)) { + TickService(dashboard, eService::DASHBOARD, observed, now, actions); + } + + if (std::all_of(m_Units.begin(), m_Units.end(), [](const Unit& unit) { return IsFinished(unit.state); })) { + const auto failed = std::count_if(m_Units.begin(), m_Units.end(), [](const Unit& unit) { return unit.state == eUnitState::FAILED; }); + if (m_Phase == ePhase::CANCELLING) Finish(ePhase::CANCELLED, "Cancelled"); + else Finish(ePhase::DONE, failed == 0 ? "Everything runs on the new build" : + std::to_string(failed) + " part(s) failed and still run on the old build"); + } + return TakeDirty(); + } + + // How an instance migration this update started is going + void OnMigration(const MigrationStatus& status) { + for (auto& unit : m_Units) { + if (unit.kind != eUnitKind::WORLD || unit.migrationId == 0 || unit.migrationId != status.migrationId || IsFinished(unit.state)) continue; + using InstanceMigration::eState; + if (status.targetInstance != 0) unit.replacement = status.targetInstance; + unit.moved = status.moved; + switch (status.state) { + case eState::PREPARING: SetState(unit, eUnitState::PREPARING, "Saving the property"); break; + case eState::PREPARED: SetState(unit, eUnitState::STARTING, "Property saved"); break; + case eState::STARTING_TARGET: SetState(unit, eUnitState::STARTING, "Starting instance " + std::to_string(unit.replacement)); break; + case eState::WARNING: SetState(unit, eUnitState::READY, "Instance " + std::to_string(unit.replacement) + " is ready; players were warned"); break; + case eState::MOVING: SetState(unit, eUnitState::DRAINING, std::to_string(status.moved) + " moved, " + std::to_string(status.remaining) + " left"); break; + case eState::DONE: + unit.migrationId = 0; + SetState(unit, eUnitState::STOPPING, std::to_string(status.moved) + " player(s) moved" + (status.message.empty() ? "" : "; " + status.message)); + unit.deadline = Clock::time_point::max(); // TickWorld sets the stop deadline + break; + case eState::FAILED: + unit.migrationId = 0; + SetState(unit, eUnitState::FAILED, status.message.empty() ? "The move failed" : status.message); + break; + } + m_Dirty = true; + } + } + + bool IsRunning() const { return m_Phase == ePhase::RUNNING || m_Phase == ePhase::CANCELLING; } + ePhase Phase() const { return m_Phase; } + const std::vector& Units() const { return m_Units; } + const std::string& Message() const { return m_Message; } + const Settings& GetSettings() const { return m_Settings; } + + // Units taking a slot of parallelWorlds: worlds being prepared, started or emptied + uint32_t Busy() const { + uint32_t busy = 0; + for (const auto& unit : m_Units) { + if (unit.kind != eUnitKind::WORLD) continue; + if (unit.state == eUnitState::PREPARING || unit.state == eUnitState::STARTING || unit.state == eUnitState::READY || + unit.state == eUnitState::DRAINING) busy++; + } + return busy; + } + + void FillStatus(LiveUpdateStatus& status) const { + status.phase = m_Phase; + status.message = m_Message; + status.units.clear(); + for (const auto& unit : m_Units) { + LiveUpdateStatus::Unit row; + row.kind = unit.kind; + row.state = unit.state; + row.zoneId = unit.zone; + row.instanceId = unit.instance; + row.cloneId = unit.clone; + row.replacement = unit.replacement; + row.players = unit.players; + row.moved = unit.moved; + row.isPrivate = unit.isPrivate ? 1 : 0; + row.message = unit.message; + status.units.push_back(std::move(row)); + } + } + + private: + static Unit ServiceUnit(eUnitKind kind) { + Unit unit; + unit.kind = kind; + return unit; + } + + Unit& Find(eUnitKind kind) { + for (auto& unit : m_Units) { + if (unit.kind == kind) return unit; + } + return m_Units.front(); + } + + void SetState(Unit& unit, eUnitState state, const std::string& message) { + if (unit.state != state || unit.message != message) m_Dirty = true; + unit.state = state; + unit.message = message; + } + + void Finish(ePhase phase, const std::string& message) { + m_Phase = phase; + m_Message = message; + m_Dirty = true; + } + + bool TakeDirty() { + const bool dirty = m_Dirty; + m_Dirty = false; + return dirty; + } + + // A server: asked to stop (the UGC server first finishes what it is making); master starts the new one when the + // old one disconnects; done once a new one connected + void TickService(Unit& unit, eService service, const Observed& observed, Clock::time_point now, IActions& actions) { + const auto& view = observed.Service(service); + const bool replaced = view.connects > unit.connectsAtStart && view.online; + switch (unit.state) { + case eUnitState::PENDING: + if (m_Phase != ePhase::RUNNING) return SetState(unit, eUnitState::SKIPPED, "Cancelled"); + if (!view.enabled) return SetState(unit, eUnitState::SKIPPED, "Not enabled"); + if (!view.online) return SetState(unit, eUnitState::SKIPPED, "Not running (master starts it from the new build when it comes back)"); + unit.connectsAtStart = view.connects; + actions.RetireService(service); + if (service == eService::UGC) { + unit.deadline = now + m_Settings.ugcDrainTimeout; + return SetState(unit, eUnitState::DRAINING, "Finishing the jobs it is running"); + } + unit.deadline = now + m_Settings.serviceTimeout; + return SetState(unit, eUnitState::STOPPING, service == eService::CHAT ? "Handing its teams over" : "Stopping"); + case eUnitState::DRAINING: + case eUnitState::STOPPING: + if (replaced) return Replaced(unit, service, now, actions); + if (!view.online) { + unit.deadline = now + m_Settings.serviceTimeout; + return SetState(unit, eUnitState::STARTING, "Starting the new " + std::string(ServiceName(service))); + } + if (now < unit.deadline) return; + if (unit.state == eUnitState::DRAINING) { + actions.StopService(service); + unit.deadline = now + m_Settings.serviceTimeout; + return SetState(unit, eUnitState::STOPPING, "Took too long; stopped (the jobs it was making are made again)"); + } + return SetState(unit, eUnitState::FAILED, "It didn't stop; the old one keeps running"); + case eUnitState::STARTING: + if (replaced) return Replaced(unit, service, now, actions); + if (now < unit.deadline) return; + if (unit.starts >= m_Settings.maxServiceStarts) { + return SetState(unit, eUnitState::FAILED, "The new " + std::string(ServiceName(service)) + " didn't come up"); + } + unit.starts++; + actions.StartService(service); + unit.deadline = now + m_Settings.serviceTimeout; + return SetState(unit, eUnitState::STARTING, "Starting the new " + std::string(ServiceName(service)) + " again (try " + std::to_string(unit.starts) + ")"); + case eUnitState::READY: + // Chat: worlds had a moment to reconnect and send their players + if (now >= unit.deadline) SetState(unit, eUnitState::STOPPED, "Replaced; worlds sent it their players"); + return; + default: + return; + } + } + + void Replaced(Unit& unit, eService service, Clock::time_point now, IActions& actions) { + if (service != eService::CHAT) return SetState(unit, eUnitState::STOPPED, "Replaced"); + actions.ChatReady(); + unit.deadline = now + m_Settings.chatSettle; + SetState(unit, eUnitState::READY, "New chat server is up; worlds are reconnecting"); + } + + IActions::Move MoveFor(const Unit& unit, InstanceMigration::eKind kind, uint32_t target) const { + IActions::Move move; + move.zone = unit.zone; + move.instance = unit.instance; + move.kind = kind; + move.target = target; + move.warnSeconds = m_Settings.warnSeconds; + move.playerWaitSeconds = m_Settings.playerWaitSeconds; + move.prepare = unit.clone != 0; + move.prepareWaitSeconds = m_Settings.propertyBuildWaitSeconds; + return move; + } + + void StartMigration(Unit& unit, InstanceMigration::eKind kind, uint32_t target, Clock::time_point now, IActions& actions) { + std::string refusal; + const auto id = actions.Migrate(MoveFor(unit, kind, target), refusal); + if (!id) { + if (unit.drained) actions.SetDraining(unit.zone, unit.instance, false); + return SetState(unit, eUnitState::FAILED, refusal.empty() ? "Master refused to move its players" : refusal); + } + unit.migrationId = *id; + unit.deadline = now + m_Settings.worldTimeout; + if (unit.clone != 0) SetState(unit, eUnitState::PREPARING, "Saving the property"); + else SetState(unit, eUnitState::STARTING, "Starting its replacement"); + } + + void StopOld(Unit& unit, Clock::time_point now, IActions& actions, const std::string& message) { + if (!unit.drained) { + actions.SetDraining(unit.zone, unit.instance, true); + unit.drained = true; + } + actions.StopWorld(unit.zone, unit.instance); + unit.deadline = now + m_Settings.serviceTimeout; + SetState(unit, eUnitState::STOPPING, message); + } + + void StartWorld(Unit& unit, const Observed& observed, Clock::time_point now, IActions& actions) { + const auto* world = observed.Find(unit.zone, unit.instance); + if (!world || world->view.shuttingDown) return SetState(unit, eUnitState::STOPPED, "It stopped by itself"); + unit.players = static_cast(std::max(world->view.players, 0)); + unit.plan = PlanWorld(world->view, world->activityZone, m_Settings.keepZones.contains(unit.zone), m_ReplacedZones.contains(unit.zone)); + switch (unit.plan) { + case eWorldPlan::STOP: + return StopOld(unit, now, actions, "Nobody there; stopping it"); + case eWorldPlan::REPLACE_EMPTY: { + const auto started = actions.StartWorld(unit.zone); + if (!started) return SetState(unit, eUnitState::FAILED, "Master refused to start a new instance"); + unit.replacement = *started; + m_ReplacedZones.insert(unit.zone); + unit.deadline = now + m_Settings.worldStartTimeout; + return SetState(unit, eUnitState::STARTING, "Starting instance " + std::to_string(unit.replacement)); + } + case eWorldPlan::MOVE: + return StartMigration(unit, InstanceMigration::eKind::REPLACE, 0, now, actions); + case eWorldPlan::WAIT_THEN_MOVE: { + // Character selection: logins go to a new one at once + if (unit.zone == 0 && !m_ReplacedZones.contains(0)) { + if (const auto started = actions.StartWorld(0)) { + unit.replacement = *started; + m_ReplacedZones.insert(0); + } + } + actions.SetDraining(unit.zone, unit.instance, true); + unit.drained = true; + unit.deadline = now + (unit.zone == 0 ? m_Settings.charSelectWait : m_Settings.activityWait); + return SetState(unit, eUnitState::WAITING, unit.zone == 0 ? "Waiting for players to pick a character" : "Waiting for the activity to end"); + } + } + } + + void TickWorld(Unit& unit, const Observed& observed, Clock::time_point now, IActions& actions) { + const auto* world = observed.Find(unit.zone, unit.instance); + switch (unit.state) { + case eUnitState::STOPPING: + if (!world) return SetState(unit, eUnitState::STOPPED, unit.message.empty() ? "Stopped" : unit.message); + if (unit.deadline == Clock::time_point::max()) unit.deadline = now + m_Settings.worldStartTimeout; + if (now >= unit.deadline) SetState(unit, eUnitState::FAILED, "It didn't stop"); + return; + case eUnitState::WAITING: { + if (!world) return SetState(unit, eUnitState::STOPPED, "Everyone left; it stopped"); + if (world->view.players <= 0) return StopOld(unit, now, actions, "Everyone left; stopping it"); + if (now < unit.deadline) return; + // Whoever is still there is moved: from character selection into the new one, from an activity into a + // fresh instance (the activity is lost) + const auto* replacement = unit.replacement ? observed.Find(unit.zone, unit.replacement) : nullptr; + if (replacement && replacement->view.ready) return StartMigration(unit, InstanceMigration::eKind::MERGE, unit.replacement, now, actions); + return StartMigration(unit, InstanceMigration::eKind::REPLACE, 0, now, actions); + } + case eUnitState::STARTING: + case eUnitState::READY: + if (unit.plan == eWorldPlan::REPLACE_EMPTY) { + const auto* replacement = observed.Find(unit.zone, unit.replacement); + if (!world) return SetState(unit, eUnitState::STOPPED, "It stopped by itself"); + if (!replacement) return SetState(unit, eUnitState::FAILED, "Instance " + std::to_string(unit.replacement) + " stopped while starting"); + if (!replacement->view.ready) { + if (now >= unit.deadline) SetState(unit, eUnitState::FAILED, "Instance " + std::to_string(unit.replacement) + " didn't start in time"); + return; + } + // Someone may have arrived before it was ready: they are moved over + if (world->view.players > 0) { + actions.SetDraining(unit.zone, unit.instance, true); + unit.drained = true; + return StartMigration(unit, InstanceMigration::eKind::MERGE, unit.replacement, now, actions); + } + return StopOld(unit, now, actions, "Instance " + std::to_string(unit.replacement) + " took over; stopping it"); + } + [[fallthrough]]; + case eUnitState::PREPARING: + case eUnitState::DRAINING: + // The migration reports how it goes (OnMigration); this is only a backstop + if (now >= unit.deadline) SetState(unit, eUnitState::FAILED, "Timed out"); + return; + default: + return; + } + } + + Settings m_Settings; + ePhase m_Phase{ ePhase::IDLE }; + std::vector m_Units; + std::set m_ReplacedZones; + std::string m_Message; + bool m_Dirty{}; + }; +} + +#endif //!__LIVEUPDATEMACHINE__H__ diff --git a/dMasterServer/MasterServer.cpp b/dMasterServer/MasterServer.cpp index b7b4e1829..1ab85e51f 100644 --- a/dMasterServer/MasterServer.cpp +++ b/dMasterServer/MasterServer.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -39,6 +40,7 @@ #include "Game.h" #include "InstanceManager.h" #include "MigrationCoordinator.h" +#include "LiveUpdateCoordinator.h" #include "MasterPackets.h" #include "FdbToSqlite.h" #include "BitStreamUtils.h" @@ -51,6 +53,7 @@ #include "master/DataChanged.h" #include "master/MessageCapture.h" #include "master/InstanceMigration.h" +#include "master/LiveUpdate.h" #include "master/ServerTraffic.h" #include "master/UgcModelsMade.h" @@ -88,6 +91,18 @@ SystemAddress ugcServerMasterPeerSysAddr; // The UGC server's process id from its last start (0: not started), for the dashboard uint32_t ugcServerPid = 0; +namespace { + // How many times each server connected (a live update waits for the new process to connect) + uint32_t g_AuthConnects = 0; + uint32_t g_ChatConnects = 0; + uint32_t g_UgcConnects = 0; + uint32_t g_DashboardConnects = 0; + // The teams a chat server retired by a live update handed over, for the next chat server that connects + std::optional g_ChatHandoff; + // SIGUSR2 starts a live update (docs/LiveUpdate.md) + volatile std::sig_atomic_t g_LiveUpdateSignal = 0; +} + namespace { // Every server for the dashboard: auth, chat, the UGC server and the worlds, including those launched but not // connected yet (starting) and those shutting down @@ -110,7 +125,8 @@ namespace { entry.ip = LUString(inst->GetIP()); entry.port = inst->GetPort(); entry.isPrivate = inst->GetIsPrivate() ? 1 : 0; - entry.state = inst->GetIsShuttingDown() ? eState::STOPPING : inst->GetIsReady() ? eState::READY : eState::STARTING; + entry.state = inst->GetIsShuttingDown() ? eState::STOPPING : !inst->GetIsReady() ? eState::STARTING : + inst->GetIsDraining() ? eState::DRAINING : eState::READY; } return response; } @@ -157,6 +173,8 @@ namespace { } } +void InitializeLiveUpdates(); + int GenerateBCryptPassword(const std::string& password, const int workFactor, char salt[BCRYPT_HASHSIZE], char hash[BCRYPT_HASHSIZE]) { int32_t bcryptState = ::bcrypt_gensalt(workFactor, salt); assert(bcryptState == 0); @@ -182,6 +200,10 @@ int main(int argc, char** argv) { std::atexit([]() { ShutdownSequence(); }); std::signal(SIGINT, Game::OnSignal); std::signal(SIGTERM, Game::OnSignal); +#ifdef SIGUSR2 + // kill -USR2 : live update onto the binaries on disk now (docs/LiveUpdate.md) + std::signal(SIGUSR2, [](int) { g_LiveUpdateSignal = 1; }); +#endif Game::config = new dConfig("masterconfig.ini"); @@ -472,6 +494,8 @@ int main(int argc, char** argv) { }); Game::im->LoadZoneLimits(); + InitializeLiveUpdates(); + //Depending on the config, start up servers: if (Game::config->GetValue("prestart_servers") != "0") { StartChatServer(); @@ -525,6 +549,14 @@ int main(int argc, char** argv) { } MigrationCoordinator::Update(); + if (g_LiveUpdateSignal) { + g_LiveUpdateSignal = 0; + std::string error; + if (!LiveUpdateCoordinator::Start("signal (SIGUSR2)", LWOOBJID_EMPTY, LiveUpdateRequest::DEFAULT_WARN, error)) { + LOG("Live update from SIGUSR2 refused: %s", error.c_str()); + } + } + LiveUpdateCoordinator::Update(); CheckPlayerActionTimeouts(); // Spare instances for busy zones (zone_limits), checked every few seconds @@ -673,15 +705,25 @@ namespace { break; case ServiceType::CHAT: chatServerMasterPeerSysAddr = sysAddr; + g_ChatConnects++; + // A live update retired the last one: this one gets its teams + if (g_ChatHandoff) { + LOG("Handing %zu team(s) over to the new chat server", g_ChatHandoff->teams.size()); + MasterPackets::SendTo(sysAddr, *g_ChatHandoff); + g_ChatHandoff.reset(); + } break; case ServiceType::AUTH: authServerMasterPeerSysAddr = sysAddr; + g_AuthConnects++; break; case ServiceType::DASHBOARD: dashboardServerMasterPeerSysAddr = sysAddr; + g_DashboardConnects++; break; case ServiceType::UGC: ugcServerMasterPeerSysAddr = sysAddr; + g_UgcConnects++; break; default: break; @@ -864,6 +906,30 @@ namespace { MigrationCoordinator::Start(request); } + // The dashboard, or a world for a GM's /liveupdate (the world checked the GM level) + void OnLiveUpdateRequest(const LiveUpdateRequest& request, const SystemAddress& sysAddr) { + const bool fromDashboard = sysAddr == dashboardServerMasterPeerSysAddr && sysAddr != UNASSIGNED_SYSTEM_ADDRESS; + if (!fromDashboard && !Game::im->GetInstanceBySysAddr(sysAddr)) { + LOG("Ignoring a live update request from a server that is neither the dashboard nor a world"); + return; + } + if (shutdownSequenceStarted && request.action == LiveUpdate::eAction::START) { + LOG("Shutdown sequence has been started. Not starting a live update."); + return; + } + LiveUpdateCoordinator::HandleRequest(request); + } + + // A chat server retired by a live update hands its teams over + void OnChatHandoff(const ChatHandoff& handoff, const SystemAddress& sysAddr) { + if (sysAddr != chatServerMasterPeerSysAddr) { + LOG("Ignoring a chat handoff from a server that is not the chat server"); + return; + } + LOG("The chat server handed over %zu team(s)", handoff.teams.size()); + g_ChatHandoff = handoff; + } + void OnPlayerAction(const PlayerActionRequest& request, const SystemAddress& sysAddr) { // Only the dashboard may ask worlds to act on players if (sysAddr != dashboardServerMasterPeerSysAddr) { @@ -1028,6 +1094,8 @@ namespace { handlers.On(Master::REQUEST_SERVER_LIST, OnRequestServerList); handlers.On(Master::SERVER_TRAFFIC, OnServerTraffic); handlers.On(Master::UGC_MODELS_MADE, OnUgcModelsMade); + handlers.On(Master::LIVE_UPDATE_REQUEST, OnLiveUpdateRequest); + handlers.On(Master::CHAT_HANDOFF, OnChatHandoff); return handlers; }(); return handlers; @@ -1125,6 +1193,7 @@ int ShutdownSequence(int32_t signal) { FinalizeShutdown(EXIT_FAILURE); } + LiveUpdateCoordinator::Abort("Master is shutting down"); Game::im->SetIsShuttingDown(true); shutdownSequenceStarted = true; Game::lastSignal = -1; @@ -1231,3 +1300,77 @@ int32_t FinalizeShutdown(int32_t signal) { if (signal != EXIT_SUCCESS) exit(signal); return signal; } + +// Live updates (docs/LiveUpdate.md): what the coordinator needs from master +void InitializeLiveUpdates() { + using LiveUpdate::eService; + const auto peer = [](eService service) -> const SystemAddress& { + switch (service) { + case eService::UGC: return ugcServerMasterPeerSysAddr; + case eService::AUTH: return authServerMasterPeerSysAddr; + case eService::CHAT: return chatServerMasterPeerSysAddr; + case eService::DASHBOARD: return dashboardServerMasterPeerSysAddr; + } + return authServerMasterPeerSysAddr; + }; + + LiveUpdateCoordinator::Hooks hooks; + hooks.service = [peer](eService service) { + LiveUpdate::ServiceView view; + view.online = peer(service) != UNASSIGNED_SYSTEM_ADDRESS; + switch (service) { + case eService::UGC: + view.enabled = Game::config->GetValue("enable_ugc_server") == "1"; + view.connects = g_UgcConnects; + break; + case eService::AUTH: + view.connects = g_AuthConnects; + break; + case eService::CHAT: + view.connects = g_ChatConnects; + break; + case eService::DASHBOARD: + view.enabled = Game::config->GetValue("enable_dashboard") == "1"; + view.connects = g_DashboardConnects; + break; + } + return view; + }; + // Chat hands its teams over and the UGC server finishes its jobs first; auth and the dashboard just stop. Either + // way master starts the new one when the old one disconnects (HandlePacket). + hooks.retire = [peer](eService service) { + const auto& address = peer(service); + if (address == UNASSIGNED_SYSTEM_ADDRESS) return; + if (service == eService::CHAT || service == eService::UGC) MasterPackets::SendTo(address, LiveUpdateRetire()); + else MasterPackets::SendTo(address, MasterPackets::Shutdown()); + }; + hooks.stop = [peer](eService service) { + const auto& address = peer(service); + if (address != UNASSIGNED_SYSTEM_ADDRESS) MasterPackets::SendTo(address, MasterPackets::Shutdown()); + }; + hooks.start = [](eService service) { + switch (service) { + case eService::UGC: ugcServerPid = StartUgcServer(); break; + case eService::AUTH: StartAuthServer(); break; + case eService::CHAT: StartChatServer(); break; + case eService::DASHBOARD: StartDashboardServer(); break; + } + }; + hooks.chatReady = [] { + for (const auto& instance : Game::im->GetInstances()) { + if (instance && instance->GetIsReady() && !instance->GetIsShuttingDown()) MasterPackets::SendTo(instance->GetSysAddr(), ChatServerReady()); + } + }; + hooks.publish = [](const LiveUpdateStatus& status, bool toWorlds) { + if (dashboardServerMasterPeerSysAddr != UNASSIGNED_SYSTEM_ADDRESS) MasterPackets::SendTo(dashboardServerMasterPeerSysAddr, status); + // The worlds' states (draining) changed with it + PushServerListToDashboard(); + // The GM who asked in game hears how it goes, wherever they are by then + if (!toWorlds || status.requesterId == LWOOBJID_EMPTY) return; + for (const auto& instance : Game::im->GetInstances()) { + if (instance && instance->GetIsReady() && !instance->GetIsShuttingDown()) MasterPackets::SendTo(instance->GetSysAddr(), status); + } + }; + LiveUpdateCoordinator::Initialize(std::move(hooks)); + MigrationCoordinator::SetObserver(LiveUpdateCoordinator::OnMigrationStatus); +} diff --git a/dMasterServer/MigrationCoordinator.cpp b/dMasterServer/MigrationCoordinator.cpp index f82e2ae49..d60291607 100644 --- a/dMasterServer/MigrationCoordinator.cpp +++ b/dMasterServer/MigrationCoordinator.cpp @@ -36,6 +36,10 @@ namespace { bool shutdownSource{}; bool startedTarget{}; // we started the target for this migration bool seamless{}; + bool liveUpdate{}; + bool prepare{}; + uint16_t prepareWaitSeconds{}; + uint16_t playerWaitSeconds{ MigratePlayersOrder::DEFAULT_MAX_WAIT_SECONDS }; LWOOBJID requester{}; int reserved{}; uint16_t moved{}; @@ -46,25 +50,14 @@ namespace { std::map g_Active; std::function g_Reporter; + std::function g_Observer; const InstancePtr& FindInstance(uint32_t zone, uint32_t instance) { return Game::im->FindInstanceWithPrivate(static_cast(zone), static_cast(instance)); } InstanceView View(const Instance& instance) { - InstanceView view; - view.zoneId = instance.GetMapID(); - view.instanceId = instance.GetInstanceID(); - view.cloneId = instance.GetCloneID(); - view.players = instance.GetCurrentClientCount(); - view.softCap = instance.GetSoftCap(); - view.hardCap = instance.GetHardCap(); - view.reserved = instance.GetReserved(); - view.ready = instance.GetIsReady(); - view.isPrivate = instance.GetIsPrivate(); - view.shuttingDown = instance.GetIsShuttingDown() || instance.GetShutdownComplete(); - view.draining = instance.GetIsDraining(); - return view; + return instance.View(); } // Races, minigames and other activities run in their own zones with state that only lives in that world @@ -84,7 +77,7 @@ namespace { void Report(const Migration& migration, eState state, const std::string& message, uint16_t remaining = 0) { LOG("Migration %u (%s zone %u instance %u -> %u): %s%s%s", migration.id, KindName(migration.kind), migration.zone, migration.source, migration.target, StateName(state), message.empty() ? "" : ": ", message.c_str()); - if (!g_Reporter) return; + if (!g_Reporter && !g_Observer) return; MigrationStatus status; status.migrationId = migration.id; status.state = state; @@ -97,7 +90,9 @@ namespace { status.requesterId = migration.requester; status.remaining = remaining; status.message = message; - g_Reporter(status); + if (g_Observer) g_Observer(status); + // In game only the GM who asked hears about it + if (g_Reporter && status.requesterId != LWOOBJID_EMPTY) g_Reporter(status); } void ReleaseSeats(Migration& migration) { @@ -126,6 +121,10 @@ namespace { if (success && migration.shutdownSource && source) { // Draining stays set: nothing is sent to it while it shuts down source->Shutdown(); + } else if (source && migration.prepare && migration.moved > 0) { + // A property some players already went to: the new instance owns it now, and the old one stays frozen + // (it is shut down once empty); nobody new goes to the old one + LOG("Migration %u: the property's old instance %u keeps draining (players already went to %u)", migration.id, migration.source, migration.target); } else if (source) { source->SetIsDraining(false); } @@ -150,7 +149,9 @@ namespace { order.playersPerSecond = PLAYERS_PER_SECOND; // Without a loading screen the "dimensional shift" notice would be the only sign; leave it out then order.seamless = migration.seamless; - order.mythranShift = !migration.seamless; + // Character selection has no maintenance notice to show + order.mythranShift = !migration.seamless && source.GetMapID() != 0; + order.maxWaitSeconds = migration.playerWaitSeconds; MasterPackets::SendTo(source.GetSysAddr(), order); migration.state = migration.warnSeconds > 0 ? eState::WARNING : eState::MOVING; @@ -164,7 +165,42 @@ void MigrationCoordinator::SetReporter(std::function observer) { + g_Observer = std::move(observer); +} + +namespace { + // The new instance (REPLACE): a fresh world server from the binary on disk now, so an update takes effect. A private + // instance's replacement gets the same password; the source is draining already, so the password finds the new one. + Instance* StartTarget(Instance& source) { + if (source.GetIsPrivate()) { + source.SetIsDraining(true); + const auto& started = Game::im->CreatePrivateInstance(source.GetMapID(), source.GetCloneID(), source.GetPassword()); + return started ? started.get() : nullptr; + } + const auto& started = Game::im->StartNewInstance(source.GetMapID(), source.GetCloneID()); + return started ? started.get() : nullptr; + } + + // Seats held, the source draining, and the order sent once the target is ready + void Begin(Migration& migration, Instance& source, Instance& target) { + migration.target = target.GetInstanceID(); + // Seats for everyone there now; a few may still arrive while draining, the hard cap is the real limit + migration.reserved = source.GetCurrentClientCount(); + target.SetReserved(target.GetReserved() + migration.reserved); + source.SetIsDraining(true); + if (target.GetIsReady()) { + SendOrder(migration, source, target); + } else { + migration.state = eState::STARTING_TARGET; + migration.deadline = Clock::now() + TARGET_START_TIMEOUT; + Report(migration, eState::STARTING_TARGET, "Starting instance " + std::to_string(migration.target), + static_cast(source.GetCurrentClientCount())); + } + } +} + +eRefusal MigrationCoordinator::Start(const InstanceMigrationRequest& request, const Options& options) { Migration migration; migration.id = request.requestId; migration.kind = request.kind; @@ -175,6 +211,10 @@ eRefusal MigrationCoordinator::Start(const InstanceMigrationRequest& request) { migration.seamless = request.seamless; migration.requester = request.requesterId; migration.by = request.requestedBy; + migration.liveUpdate = options.liveUpdate; + migration.prepare = options.prepare; + migration.prepareWaitSeconds = std::min(options.prepareWaitSeconds, MigratePrepare::MAX_WAIT_SECONDS); + migration.playerWaitSeconds = std::min(options.playerWaitSeconds, MigratePlayersOrder::MAX_MAX_WAIT_SECONDS); const auto refuse = [&migration](eRefusal refusal) { Report(migration, eState::FAILED, Describe(refusal)); @@ -186,7 +226,8 @@ eRefusal MigrationCoordinator::Start(const InstanceMigrationRequest& request) { Instance* source = FindInstance(request.zoneId, request.sourceInstance).get(); if (!source) return refuse(eRefusal::NOT_RUNNING); auto sourceView = View(*source); - if (const auto refusal = CheckSource(sourceView, IsActivityZone(request.zoneId)); refusal != eRefusal::NONE) return refuse(refusal); + const auto sourceRefusal = options.liveUpdate ? CheckLiveUpdateSource(sourceView) : CheckSource(sourceView, IsActivityZone(request.zoneId)); + if (sourceRefusal != eRefusal::NONE) return refuse(sourceRefusal); if (IsTakingPart(request.zoneId, request.sourceInstance)) return refuse(eRefusal::ALREADY_MIGRATING); migration.clone = source->GetCloneID(); @@ -208,32 +249,36 @@ eRefusal MigrationCoordinator::Start(const InstanceMigrationRequest& request) { if (IsTakingPart(request.zoneId, targetId)) return refuse(eRefusal::ALREADY_MIGRATING); if (const auto refusal = CheckMergeTarget(View(*found), sourceView); refusal != eRefusal::NONE) return refuse(refusal); target = found.get(); + } else if (migration.prepare) { + // A property: the new instance loads it from the database when it starts, so the source saves and freezes it + // first (HandleStatus starts the target once it is PREPARED). Visitors still go to the source meanwhile. + LOG("Migration %u requested by %s: preparing zone %u clone %u instance %u (%i player(s))", migration.id, migration.by.c_str(), + migration.zone, migration.clone, migration.source, source->GetCurrentClientCount()); + auto& active = g_Active[migration.id] = migration; + active.state = eState::PREPARING; + active.deadline = Clock::now() + std::chrono::seconds(active.prepareWaitSeconds) + TARGET_START_TIMEOUT; + MigratePrepare prepare; + prepare.migrationId = active.id; + prepare.maxWaitSeconds = active.prepareWaitSeconds; + MasterPackets::SendTo(source->GetSysAddr(), prepare); + Report(active, eState::PREPARING, "Saving the property", static_cast(source->GetCurrentClientCount())); + return eRefusal::NONE; } else { // A fresh world server, started from the binary on disk now: this is how a live update takes over - const auto& started = Game::im->StartNewInstance(source->GetMapID(), source->GetCloneID()); - if (!started) return refuse(eRefusal::MASTER_SHUTTING_DOWN); - target = started.get(); + target = StartTarget(*source); + if (!target) { + source->SetIsDraining(false); + return refuse(eRefusal::MASTER_SHUTTING_DOWN); + } migration.startedTarget = true; + // Starting it grew the instance list, which moves the InstancePtrs but not the Instances: source is still good } - migration.target = target->GetInstanceID(); - // Seats for everyone there now; a few may still arrive while draining, the hard cap is the real limit - migration.reserved = source->GetCurrentClientCount(); - target->SetReserved(target->GetReserved() + migration.reserved); - source->SetIsDraining(true); - LOG("Migration %u requested by %s: %s zone %u instance %u (%i player(s)) -> instance %u", migration.id, migration.by.c_str(), - KindName(migration.kind), migration.zone, migration.source, source->GetCurrentClientCount(), migration.target); + KindName(migration.kind), migration.zone, migration.source, source->GetCurrentClientCount(), target->GetInstanceID()); auto& active = g_Active[migration.id] = migration; - if (target->GetIsReady()) { - SendOrder(active, *source, *target); - } else { - active.state = eState::STARTING_TARGET; - active.deadline = Clock::now() + TARGET_START_TIMEOUT; - Report(active, eState::STARTING_TARGET, "Starting instance " + std::to_string(active.target), - static_cast(source->GetCurrentClientCount())); - } + Begin(active, *source, *target); return eRefusal::NONE; } @@ -248,6 +293,20 @@ void MigrationCoordinator::HandleStatus(const SystemAddress& from, const Migrati migration.failed = status.failed; if (status.state == eState::DONE) return Finish(it, true, status.message); if (status.state == eState::FAILED) return Finish(it, false, status.message); + if (status.state == eState::PREPARED) { + if (migration.state != eState::PREPARING) return; + // Saved and frozen: now the new instance may load the property + Instance* sourceInstance = source.get(); + Instance* target = StartTarget(*sourceInstance); + if (!target) { + SendCancel(migration); + return Finish(it, false, "Master refused to start the new instance"); + } + migration.startedTarget = true; + LOG("Migration %u: property of zone %u clone %u saved; instance %u -> %u", migration.id, migration.zone, migration.clone, migration.source, target->GetInstanceID()); + Report(migration, eState::PREPARED, status.message, static_cast(sourceInstance->GetCurrentClientCount())); + return Begin(migration, *sourceInstance, *target); + } migration.state = status.state; Report(migration, status.state, status.message, status.remaining); } @@ -289,7 +348,14 @@ void MigrationCoordinator::Update() { for (auto it = g_Active.begin(); it != g_Active.end();) { auto& migration = it->second; auto current = it++; - if (migration.state == eState::STARTING_TARGET) { + if (migration.state == eState::PREPARING) { + if (!FindInstance(migration.zone, migration.source)) { + Finish(current, false, "The instance being emptied stopped"); + } else if (now > migration.deadline) { + SendCancel(migration); + Finish(current, false, "The property wasn't saved in time"); + } + } else if (migration.state == eState::STARTING_TARGET) { const auto& source = FindInstance(migration.zone, migration.source); const auto& target = FindInstance(migration.zone, migration.target); if (!source || !target) { diff --git a/dMasterServer/MigrationCoordinator.h b/dMasterServer/MigrationCoordinator.h index ab5f244a1..094ed824c 100644 --- a/dMasterServer/MigrationCoordinator.h +++ b/dMasterServer/MigrationCoordinator.h @@ -17,9 +17,24 @@ namespace MigrationCoordinator { // Where statuses go (every world, so the GM who asked hears about it wherever they are); set once at startup void SetReporter(std::function reporter); - // A migration was asked for (a GM command, or later a dashboard); anything but NONE means it was refused (and + // Also told every status, including those nobody asked for in game (live updates follow their migrations here) + void SetObserver(std::function observer); + + // What a live update (LiveUpdateCoordinator) changes about a migration + struct Options { + // Moves what CheckSource refuses (see CheckLiveUpdateSource): character selection, private instances, + // properties and activity zones + bool liveUpdate{}; + // Properties: the source saves and freezes the property (MIGRATE_PREPARE) before the new instance is started + bool prepare{}; + uint16_t prepareWaitSeconds{ 60 }; + // Players who are dead or building wait this long before they are moved anyway + uint16_t playerWaitSeconds{ MigratePlayersOrder::DEFAULT_MAX_WAIT_SECONDS }; + }; + + // A migration was asked for (a GM command, or a live update); anything but NONE means it was refused (and // reported as FAILED) - InstanceMigration::eRefusal Start(const InstanceMigrationRequest& request); + InstanceMigration::eRefusal Start(const InstanceMigrationRequest& request, const Options& options = {}); // A world reported progress on a migration it is running void HandleStatus(const SystemAddress& from, const MigrationStatus& status); diff --git a/dNet/ChatPackets.cpp b/dNet/ChatPackets.cpp index 7906354e8..d7ffd0b78 100644 --- a/dNet/ChatPackets.cpp +++ b/dNet/ChatPackets.cpp @@ -52,6 +52,8 @@ namespace ChatPackets { WriteZone(bitStream, zoneID); bitStream.Write(muteExpire); bitStream.Write(gmLevel); + // Only on a resync, so every other one is byte for byte what it always was + if (resync) bitStream.Write(1); } bool LoginSessionNotify::Deserialize(RakNet::BitStream& bitStream) { @@ -60,6 +62,8 @@ namespace ChatPackets { VALIDATE_READ(ReadZone(bitStream, zoneID)); VALIDATE_READ(bitStream.Read(muteExpire)); VALIDATE_READ(bitStream.Read(gmLevel)); + uint8_t resyncValue{}; + resync = bitStream.GetNumberOfUnreadBits() >= 8 && bitStream.Read(resyncValue) && resyncValue != 0; return true; } diff --git a/dNet/ChatPackets.h b/dNet/ChatPackets.h index 41fd15420..4b2016d7c 100644 --- a/dNet/ChatPackets.h +++ b/dNet/ChatPackets.h @@ -44,6 +44,9 @@ namespace ChatPackets { LWOZONEID zoneID{}; int64_t muteExpire{}; // time_t eGameMasterLevel gmLevel{}; + // Sent again to a new chat server after a live update restarted chat (CHAT_SERVER_READY): the player didn't + // log in or change zones. Written last, only when set, and read only when there. + bool resync{}; LoginSessionNotify() : LUBitStream(ServiceType::CHAT, MessageType::Chat::LOGIN_SESSION_NOTIFY) {} void Serialize(RakNet::BitStream& bitStream) const override; diff --git a/dNet/MasterPackets.cpp b/dNet/MasterPackets.cpp index 2dc9710ce..22c03603a 100644 --- a/dNet/MasterPackets.cpp +++ b/dNet/MasterPackets.cpp @@ -269,7 +269,7 @@ namespace MasterPackets { for (auto& instance : instances) { uint8_t state{}; VALIDATE_READ(bitStream.Read(state)); - if (state > static_cast(eState::STOPPING)) return false; + if (state > static_cast(eState::DRAINING)) return false; instance.state = static_cast(state); } return true; diff --git a/dNet/MasterPackets.h b/dNet/MasterPackets.h index 61ae4ea73..b81a5cf16 100644 --- a/dNet/MasterPackets.h +++ b/dNet/MasterPackets.h @@ -13,6 +13,7 @@ #include "master/DashboardMessages.h" #include "master/DataChanged.h" #include "master/InstanceMigration.h" +#include "master/LiveUpdate.h" #include "master/MessageCapture.h" #include "master/PlayerAction.h" #include "master/UgcModelsMade.h" @@ -24,7 +25,7 @@ class dServer; * * The dashboard's messages and instance migration are MASTER packets too; they live in per-topic headers under * dNet/master/ (PlayerAction.h, DataChanged.h, DashboardMessages.h, MessageCapture.h, InstanceMigration.h, - * UgcModelsMade.h), which + * LiveUpdate.h, UgcModelsMade.h), which * this header includes, so including MasterPackets.h gives every MASTER struct. */ namespace MasterPackets { @@ -251,8 +252,9 @@ namespace MasterPackets { // More than this many worlds is a broken packet static constexpr uint32_t MAX_INSTANCES = 10000; - // Where a world is: connected, launched but not connected yet, or shutting down - enum class eState : uint8_t { READY = 0, STARTING = 1, STOPPING = 2 }; + // Where a world is: connected, launched but not connected yet, shutting down, or connected but having its + // players moved to a new instance (a live update or instance migration; nobody new is sent there) + enum class eState : uint8_t { READY = 0, STARTING = 1, STOPPING = 2, DRAINING = 3 }; struct Instance { LWOMAPID mapID{}; diff --git a/dNet/master/InstanceMigration.h b/dNet/master/InstanceMigration.h index f7dc3b2a5..c635bf520 100644 --- a/dNet/master/InstanceMigration.h +++ b/dNet/master/InstanceMigration.h @@ -55,6 +55,10 @@ namespace InstanceMigration { MOVING, DONE, FAILED, + // Live updates of property (clone) instances: the source freezes the property and saves it before the new + // instance loads it (MIGRATE_PREPARE) + PREPARING, + PREPARED, }; inline const char* KindName(eKind kind) { return kind == eKind::MERGE ? "merge" : "replace"; } @@ -66,6 +70,8 @@ namespace InstanceMigration { case eState::MOVING: return "moving"; case eState::DONE: return "done"; case eState::FAILED: return "failed"; + case eState::PREPARING: return "preparing"; + case eState::PREPARED: return "prepared"; } return "unknown"; } @@ -137,6 +143,29 @@ namespace InstanceMigration { return eRefusal::NONE; } + /** + * A live update replaces every instance, so it moves what CheckSource refuses too: character selection, private + * instances, properties (after MIGRATE_PREPARE) and activity zones (once their players had time to finish). Only + * an instance that isn't running properly is left alone. A draining one is fine: the live update drains activity + * zones and character selection itself before moving whoever is left. + */ + inline eRefusal CheckLiveUpdateSource(const InstanceView& source) { + if (source.shuttingDown) return eRefusal::SHUTTING_DOWN; + if (!source.ready) return eRefusal::NOT_READY; + return eRefusal::NONE; + } + + /** + * Whether master may send a new player to this instance of zone/clone (InstanceManager::FindInstance): not + * private, not shutting down, not draining (being emptied for a live update or migration), and with room: under + * the soft cap, or under the hard cap for players following a friend. Seats held for players being moved in count. + */ + inline bool AcceptsNewPlayers(const InstanceView& instance, uint32_t zone, uint32_t clone, bool friendTransfer) { + if (instance.zoneId != zone || instance.cloneId != clone) return false; + if (instance.isPrivate || instance.shuttingDown || instance.draining) return false; + return instance.Load() < (friendTransfer ? instance.hardCap : instance.softCap); + } + // Whether everyone in source fits into target. Merges may fill up to the hard cap: the soft cap only keeps // room for friends joining, and a merge is those players staying together anyway. inline eRefusal CheckMergeTarget(const InstanceView& target, const InstanceView& source) { @@ -297,6 +326,11 @@ struct MigratePlayersOrder : public LUBitStream { * (LwoClientGhostManager::OnReceiveConstruction). Not tested with the real client yet. */ bool seamless{}; + // How long players who are dead or building may take before they are moved anyway. Written last and read only + // when there (older senders didn't write it) + uint16_t maxWaitSeconds{ DEFAULT_MAX_WAIT_SECONDS }; + static constexpr uint16_t DEFAULT_MAX_WAIT_SECONDS = 15; + static constexpr uint16_t MAX_MAX_WAIT_SECONDS = 600; void Serialize(RakNet::BitStream& stream) const override { stream.Write(migrationId); @@ -309,6 +343,7 @@ struct MigratePlayersOrder : public LUBitStream { stream.Write(playersPerSecond); stream.Write(mythranShift ? 1 : 0); stream.Write(seamless ? 1 : 0); + stream.Write(std::min(maxWaitSeconds, MAX_MAX_WAIT_SECONDS)); } bool Deserialize(RakNet::BitStream& stream) override { @@ -319,6 +354,9 @@ struct MigratePlayersOrder : public LUBitStream { mythranShift = shift != 0; seamless = seamlessValue != 0; if (playersPerSecond == 0) playersPerSecond = 1; + maxWaitSeconds = DEFAULT_MAX_WAIT_SECONDS; + uint16_t wait{}; + if (stream.GetNumberOfUnreadBits() >= 16 && stream.Read(wait)) maxWaitSeconds = std::min(wait, MAX_MAX_WAIT_SECONDS); return true; } }; @@ -365,7 +403,7 @@ struct MigrationStatus : public LUBitStream { if (!stream.Read(migrationId) || !stream.Read(stateValue) || !stream.Read(kindValue) || !stream.Read(zoneId) || !stream.Read(sourceInstance) || !stream.Read(targetInstance) || !stream.Read(moved) || !stream.Read(remaining) || !stream.Read(failed) || !stream.Read(requesterId)) return false; - if (stateValue > static_cast(InstanceMigration::eState::FAILED) || kindValue > static_cast(InstanceMigration::eKind::MERGE)) return false; + if (stateValue > static_cast(InstanceMigration::eState::PREPARED) || kindValue > static_cast(InstanceMigration::eKind::MERGE)) return false; state = static_cast(stateValue); kind = static_cast(kindValue); return InstanceMigration::ReadText(stream, message, MAX_MESSAGE); @@ -385,6 +423,14 @@ struct CarriedPlayerState : public LUBitStream { LWOOBJID characterId{}; LWOOBJID petItemId{}; // the pet that was out (its item), summoned again; 0 for none bool seamless{}; // the client kept its scene: skip LOAD_STATIC_ZONE and load the player at once + /** + * Where the player stood. The saved character only keeps it where the location is saved (not on properties + * unless save_property_location is on, and Moon Base always starts at the spawn point), so the target puts the + * player here when it creates them. Written last and read only when there (older senders didn't write it). + */ + bool hasPosition{}; + float x{}, y{}, z{}; + float rotW{ 1.0f }, rotX{}, rotY{}, rotZ{}; void Serialize(RakNet::BitStream& stream) const override { stream.Write(targetZone); @@ -392,12 +438,55 @@ struct CarriedPlayerState : public LUBitStream { stream.Write(characterId); stream.Write(petItemId); stream.Write(seamless ? 1 : 0); + stream.Write(hasPosition ? 1 : 0); + if (hasPosition) { + stream.Write(x); + stream.Write(y); + stream.Write(z); + stream.Write(rotW); + stream.Write(rotX); + stream.Write(rotY); + stream.Write(rotZ); + } } bool Deserialize(RakNet::BitStream& stream) override { uint8_t seamlessValue{}; if (!stream.Read(targetZone) || !stream.Read(targetInstance) || !stream.Read(characterId) || !stream.Read(petItemId) || !stream.Read(seamlessValue)) return false; seamless = seamlessValue != 0; + hasPosition = false; + uint8_t positionValue{}; + if (stream.GetNumberOfUnreadBits() >= 8 && stream.Read(positionValue) && positionValue != 0) { + if (!stream.Read(x) || !stream.Read(y) || !stream.Read(z) || !stream.Read(rotW) || !stream.Read(rotX) || !stream.Read(rotY) || !stream.Read(rotZ)) return false; + hasPosition = true; + } + return true; + } +}; + +/** + * MIGRATE_PREPARE (master -> source world, live updates of property instances): the new instance loads the property + * from the database when it starts, so before it is started the source waits (up to maxWaitSeconds) for everyone to + * leave build mode, saves the property and freezes it: nobody can build or claim it there any more and it is never + * saved again, so the new instance's saves can't be overwritten. The source answers MIGRATE_STATUS PREPARED (or + * FAILED). A MIGRATE_PLAYERS with target port 0 for the same migration unfreezes it again when nobody was moved yet. + */ +struct MigratePrepare : public LUBitStream { + MigratePrepare() : LUBitStream(ServiceType::MASTER, MessageType::Master::MIGRATE_PREPARE) {} + + static constexpr uint16_t MAX_WAIT_SECONDS = 600; + + uint32_t migrationId{}; + uint16_t maxWaitSeconds{ 60 }; + + void Serialize(RakNet::BitStream& stream) const override { + stream.Write(migrationId); + stream.Write(std::min(maxWaitSeconds, MAX_WAIT_SECONDS)); + } + + bool Deserialize(RakNet::BitStream& stream) override { + if (!stream.Read(migrationId) || !stream.Read(maxWaitSeconds)) return false; + maxWaitSeconds = std::min(maxWaitSeconds, MAX_WAIT_SECONDS); return true; } }; diff --git a/dNet/master/LiveUpdate.h b/dNet/master/LiveUpdate.h new file mode 100644 index 000000000..4e15a95b4 --- /dev/null +++ b/dNet/master/LiveUpdate.h @@ -0,0 +1,299 @@ +#ifndef __LIVEUPDATE__H__ +#define __LIVEUPDATE__H__ + +#include +#include +#include +#include + +#include "BitStream.h" +#include "BitStreamUtils.h" +#include "MessageType/Master.h" +#include "dCommonVars.h" +#include "master/InstanceMigration.h" + +/** + * Live updates (docs/LiveUpdate.md): with a new build of the server binaries in place, master moves everything onto + * processes started from it without taking the server down. The UGC server finishes its jobs and restarts, auth + * restarts, chat hands its teams over and restarts, every world instance is replaced by a new one its players are + * moved to (instance migrations, InstanceMigration.h), and the dashboard restarts last. Master itself keeps running: + * updating master takes a normal restart. + * + * Messages (appended to MessageType::Master): + * LIVE_UPDATE_REQUEST dashboard, or a world for a GM's /liveupdate -> master LiveUpdateRequest + * LIVE_UPDATE_STATUS master -> dashboard (and the world of the GM who asked) LiveUpdateStatus + * LIVE_UPDATE_RETIRE master -> chat or UGC server LiveUpdateRetire + * CHAT_HANDOFF retiring chat -> master -> the next chat server ChatHandoff + * CHAT_SERVER_READY master -> every world, once the new chat server is up ChatServerReady + * MIGRATE_PREPARE master -> a property instance being replaced MigratePrepare (InstanceMigration.h) + */ +namespace LiveUpdate { + enum class eAction : uint8_t { + START = 0, + CANCEL, // start nothing new; what is under way finishes + STATUS, // just send the status (the dashboard asks when it connects) + }; + + // What one row of the update is about + enum class eUnitKind : uint8_t { + DATABASE = 0, // the new build's database migrations + UGC, + AUTH, + CHAT, + WORLD, // one world instance + DASHBOARD, + }; + + /** + * Where one row is. A world goes PENDING -> (PREPARING, a property saving itself) -> STARTING (its replacement is + * starting) -> READY (the replacement took its first players or is waiting for them) -> DRAINING (players are + * being moved, or it waits for an activity to end; nobody new is sent to it) -> STOPPING -> STOPPED. A server: + * PENDING -> STOPPING (asked to exit) -> STARTING (the new one is launching) -> STOPPED (the new one is up). + */ + enum class eUnitState : uint8_t { + PENDING = 0, + PREPARING, + STARTING, + READY, + WAITING, // a world waiting for its players to leave by themselves (activities, character selection) + DRAINING, + STOPPING, + STOPPED, // done: replaced (or simply stopped when nobody was there) + FAILED, // left as it was (a world keeps running on the old build) + SKIPPED, // not running, not enabled, or the update was cancelled before it got there + }; + + enum class ePhase : uint8_t { + IDLE = 0, + RUNNING, + CANCELLING, // cancelled; waiting for what is under way + DONE, + FAILED, // stopped early (e.g. the database migrations failed); nothing after that was touched + CANCELLED, + }; + + inline const char* KindName(eUnitKind kind) { + switch (kind) { + case eUnitKind::DATABASE: return "database"; + case eUnitKind::UGC: return "ugc"; + case eUnitKind::AUTH: return "auth"; + case eUnitKind::CHAT: return "chat"; + case eUnitKind::WORLD: return "world"; + case eUnitKind::DASHBOARD: return "dashboard"; + } + return "unknown"; + } + + inline const char* StateName(eUnitState state) { + switch (state) { + case eUnitState::PENDING: return "pending"; + case eUnitState::PREPARING: return "preparing"; + case eUnitState::STARTING: return "starting"; + case eUnitState::READY: return "ready"; + case eUnitState::WAITING: return "waiting"; + case eUnitState::DRAINING: return "draining"; + case eUnitState::STOPPING: return "stopping"; + case eUnitState::STOPPED: return "stopped"; + case eUnitState::FAILED: return "failed"; + case eUnitState::SKIPPED: return "skipped"; + } + return "unknown"; + } + + inline const char* PhaseName(ePhase phase) { + switch (phase) { + case ePhase::IDLE: return "idle"; + case ePhase::RUNNING: return "running"; + case ePhase::CANCELLING: return "cancelling"; + case ePhase::DONE: return "done"; + case ePhase::FAILED: return "failed"; + case ePhase::CANCELLED: return "cancelled"; + } + return "unknown"; + } + + inline bool IsFinished(eUnitState state) { + return state == eUnitState::STOPPED || state == eUnitState::FAILED || state == eUnitState::SKIPPED; + } + + inline bool IsFinished(ePhase phase) { + return phase == ePhase::DONE || phase == ePhase::FAILED || phase == ePhase::CANCELLED; + } +} + +// LIVE_UPDATE_REQUEST +struct LiveUpdateRequest : public LUBitStream { + LiveUpdateRequest() : LUBitStream(ServiceType::MASTER, MessageType::Master::LIVE_UPDATE_REQUEST) {} + + static constexpr uint16_t MAX_BY = 64; + static constexpr int32_t DEFAULT_WARN = -1; + + LiveUpdate::eAction action{ LiveUpdate::eAction::STATUS }; + int32_t warnSeconds{ DEFAULT_WARN }; // seconds players are warned before they are moved; -1: live_update_warn_seconds + LWOOBJID requesterId{}; // the character who asked (told how it goes in chat); 0 for the dashboard + std::string requestedBy; + + void Serialize(RakNet::BitStream& stream) const override { + stream.Write(static_cast(action)); + stream.Write(warnSeconds); + stream.Write(requesterId); + InstanceMigration::WriteText(stream, requestedBy, MAX_BY); + } + + bool Deserialize(RakNet::BitStream& stream) override { + uint8_t actionValue{}; + if (!stream.Read(actionValue) || actionValue > static_cast(LiveUpdate::eAction::STATUS)) return false; + action = static_cast(actionValue); + if (!stream.Read(warnSeconds) || !stream.Read(requesterId)) return false; + if (warnSeconds < DEFAULT_WARN || warnSeconds > InstanceMigrationRequest::MAX_WARN_SECONDS) return false; + return InstanceMigration::ReadText(stream, requestedBy, MAX_BY); + } +}; + +// LIVE_UPDATE_STATUS: the whole update, sent again whenever something changes +struct LiveUpdateStatus : public LUBitStream { + LiveUpdateStatus() : LUBitStream(ServiceType::MASTER, MessageType::Master::LIVE_UPDATE_STATUS) {} + + static constexpr uint16_t MAX_TEXT = 300; + static constexpr uint32_t MAX_UNITS = 4096; + + struct Unit { + LiveUpdate::eUnitKind kind{}; + LiveUpdate::eUnitState state{}; + uint32_t zoneId{}; + uint32_t instanceId{}; + uint32_t cloneId{}; + uint32_t replacement{}; // the instance that took over (worlds) + uint32_t players{}; // players there when the row was made + uint16_t moved{}; + uint8_t isPrivate{}; + std::string message; + }; + + uint32_t updateId{}; // 0: there never was one since master started + LiveUpdate::ePhase phase{ LiveUpdate::ePhase::IDLE }; + int64_t startedAt{}; // unix time + int64_t finishedAt{}; // unix time, 0 while running + LWOOBJID requesterId{}; + std::string by; + std::string message; + std::vector units; + + void Serialize(RakNet::BitStream& stream) const override { + stream.Write(updateId); + stream.Write(static_cast(phase)); + stream.Write(startedAt); + stream.Write(finishedAt); + stream.Write(requesterId); + InstanceMigration::WriteText(stream, by, MAX_TEXT); + InstanceMigration::WriteText(stream, message, MAX_TEXT); + const auto count = static_cast(std::min(units.size(), MAX_UNITS)); + stream.Write(count); + for (uint32_t i = 0; i < count; i++) { + const auto& unit = units[i]; + stream.Write(static_cast(unit.kind)); + stream.Write(static_cast(unit.state)); + stream.Write(unit.zoneId); + stream.Write(unit.instanceId); + stream.Write(unit.cloneId); + stream.Write(unit.replacement); + stream.Write(unit.players); + stream.Write(unit.moved); + stream.Write(unit.isPrivate); + InstanceMigration::WriteText(stream, unit.message, MAX_TEXT); + } + } + + bool Deserialize(RakNet::BitStream& stream) override { + uint8_t phaseValue{}; + if (!stream.Read(updateId) || !stream.Read(phaseValue) || phaseValue > static_cast(LiveUpdate::ePhase::CANCELLED)) return false; + phase = static_cast(phaseValue); + if (!stream.Read(startedAt) || !stream.Read(finishedAt) || !stream.Read(requesterId)) return false; + if (!InstanceMigration::ReadText(stream, by, MAX_TEXT) || !InstanceMigration::ReadText(stream, message, MAX_TEXT)) return false; + uint32_t count{}; + if (!stream.Read(count) || count > MAX_UNITS) return false; + units.resize(count); + for (auto& unit : units) { + uint8_t kind{}, state{}; + if (!stream.Read(kind) || !stream.Read(state)) return false; + if (kind > static_cast(LiveUpdate::eUnitKind::DASHBOARD) || state > static_cast(LiveUpdate::eUnitState::SKIPPED)) return false; + unit.kind = static_cast(kind); + unit.state = static_cast(state); + if (!stream.Read(unit.zoneId) || !stream.Read(unit.instanceId) || !stream.Read(unit.cloneId) || !stream.Read(unit.replacement) || + !stream.Read(unit.players) || !stream.Read(unit.moved) || !stream.Read(unit.isPrivate)) return false; + if (!InstanceMigration::ReadText(stream, unit.message, MAX_TEXT)) return false; + } + return true; + } +}; + +// LIVE_UPDATE_RETIRE: no payload. Chat hands its teams over first; the UGC server finishes the jobs it is running. +struct LiveUpdateRetire : public LUBitStream { + LiveUpdateRetire() : LUBitStream(ServiceType::MASTER, MessageType::Master::LIVE_UPDATE_RETIRE) {} +}; + +// CHAT_SERVER_READY: no payload +struct ChatServerReady : public LUBitStream { + ChatServerReady() : LUBitStream(ServiceType::MASTER, MessageType::Master::CHAT_SERVER_READY) {} +}; + +/** + * CHAT_HANDOFF: the teams a retiring chat server had. Who is online is not carried: every world sends its players to + * the new chat server again (CHAT_SERVER_READY), and friends lists are read from the database. + */ +struct ChatHandoff : public LUBitStream { + ChatHandoff() : LUBitStream(ServiceType::MASTER, MessageType::Master::CHAT_HANDOFF) {} + + static constexpr uint32_t MAX_TEAMS = 20000; + static constexpr uint8_t MAX_MEMBERS = 16; + + struct Team { + LWOOBJID teamId{}; + LWOOBJID leaderId{}; + std::vector members; + uint8_t lootFlag{}; + bool local{}; + uint32_t zoneId{}; + uint32_t instanceId{}; + uint32_t cloneId{}; + }; + + std::vector teams; + + void Serialize(RakNet::BitStream& stream) const override { + const auto count = static_cast(std::min(teams.size(), MAX_TEAMS)); + stream.Write(count); + for (uint32_t i = 0; i < count; i++) { + const auto& team = teams[i]; + stream.Write(team.teamId); + stream.Write(team.leaderId); + const auto members = static_cast(std::min(team.members.size(), MAX_MEMBERS)); + stream.Write(members); + for (uint8_t m = 0; m < members; m++) stream.Write(team.members[m]); + stream.Write(team.lootFlag); + stream.Write(team.local ? 1 : 0); + stream.Write(team.zoneId); + stream.Write(team.instanceId); + stream.Write(team.cloneId); + } + } + + bool Deserialize(RakNet::BitStream& stream) override { + uint32_t count{}; + if (!stream.Read(count) || count > MAX_TEAMS) return false; + teams.resize(count); + for (auto& team : teams) { + uint8_t members{}, local{}; + if (!stream.Read(team.teamId) || !stream.Read(team.leaderId) || !stream.Read(members) || members > MAX_MEMBERS) return false; + team.members.resize(members); + for (auto& member : team.members) { + if (!stream.Read(member)) return false; + } + if (!stream.Read(team.lootFlag) || !stream.Read(local) || !stream.Read(team.zoneId) || !stream.Read(team.instanceId) || !stream.Read(team.cloneId)) return false; + team.local = local != 0; + } + return true; + } +}; + +#endif //!__LIVEUPDATE__H__ diff --git a/dUgcServer/UgcProcessor.cpp b/dUgcServer/UgcProcessor.cpp index b1433cc6c..3048feba4 100644 --- a/dUgcServer/UgcProcessor.cpp +++ b/dUgcServer/UgcProcessor.cpp @@ -150,6 +150,35 @@ void UgcProcessor::Stop() { m_Threads.clear(); } +void UgcProcessor::Drain() { + if (m_Draining) return; + m_Draining = true; + std::deque dropped; + { + std::lock_guard lock(m_Mutex); + dropped.swap(m_Jobs); + } + // What was waiting for them is forgotten too; the database rows stay pending + for (const auto& job : dropped) { + if (job.kind == Kind::MODEL) { + m_InFlight.erase({ job.kind, job.id }); + continue; + } + m_ComboJobs.erase(job.id); + if (const auto rows = m_ComboRows.find(job.id); rows != m_ComboRows.end()) { + for (const auto& [row, attempts] : rows->second) m_InFlight.erase({ Kind::MODULAR, row }); + m_ComboRows.erase(rows); + } + } + LOG("Draining: %zu queued job(s) left for the next UGC server, waiting for the running ones", dropped.size()); +} + +bool UgcProcessor::Drained() const { + if (!m_Draining) return false; + std::lock_guard lock(m_Mutex); + return m_Active == 0 && m_Jobs.empty() && m_Done.empty(); +} + void UgcProcessor::Worker() { int appliedNice = 0; while (true) { @@ -546,6 +575,8 @@ void UgcProcessor::Backfill() { void UgcProcessor::Update() { Collect(); + // Live update: only the running jobs' outcomes are recorded + if (m_Draining) return; Backfill(); const auto now = std::chrono::steady_clock::now(); if (now - m_CpuSampled >= std::chrono::seconds(2)) SampleUsage(); diff --git a/dUgcServer/UgcProcessor.h b/dUgcServer/UgcProcessor.h index fd51a97ed..ac8a143a3 100644 --- a/dUgcServer/UgcProcessor.h +++ b/dUgcServer/UgcProcessor.h @@ -61,6 +61,15 @@ public: // Waits for the jobs that are running; queued ones are dropped (they stay pending in the database) void Stop(); + /** + * Main thread, live updates: take no new work. Queued jobs are dropped (they stay pending in the database, so the + * next UGC server makes them); the running ones finish and their outcome is recorded by Update as usual. + */ + void Drain(); + // Main thread: draining and nothing running or waiting to be recorded any more + bool Drained() const; + bool IsDraining() const { return m_Draining; } + // Main thread: poll, dispatch, record results, keep the storage under its cap void Update(); @@ -219,6 +228,7 @@ private: double m_CpuSeconds{}; double m_CpuPercent{}; bool m_Stopping{}; + bool m_Draining{}; // main thread: live update, no new work (Drain) std::vector m_Threads; // Main thread only diff --git a/dUgcServer/UgcServer.cpp b/dUgcServer/UgcServer.cpp index 087b579a3..2c84ef808 100644 --- a/dUgcServer/UgcServer.cpp +++ b/dUgcServer/UgcServer.cpp @@ -697,10 +697,22 @@ int main(int argc, char** argv) { Packet* packet = g_Server->ReceiveFromMaster(); while (packet) { + // Live update (docs/LiveUpdate.md): finish the running jobs, then stop; master starts the new build's server + RakNet::BitStream inStream(packet->data, packet->length, false); + LUBitStream header; + if (header.ReadHeader(inStream) && header.connectionType == ServiceType::MASTER && + static_cast(header.internalPacketID) == MessageType::Master::LIVE_UPDATE_RETIRE && !processor.IsDraining()) { + LOG("Live update: finishing the running jobs, then stopping"); + processor.Drain(); + } g_Server->DeallocateMasterPacket(packet); packet = g_Server->ReceiveFromMaster(); } processor.Update(); + if (processor.Drained()) { + LOG("Live update: the running jobs are done; stopping"); + Game::lastSignal = -1; + } // Worlds showing a model whose mesh changed tell their clients (docs/UgcServer.md, "Models without 3D services") if (auto changed = processor.TakeChangedMeshes(); !changed.empty()) { for (size_t start = 0; start < changed.size(); start += UgcModelsMade::MAX_MODELS) { diff --git a/dWorldServer/WorldServer.cpp b/dWorldServer/WorldServer.cpp index 47ce9a847..ca1dcfd4c 100644 --- a/dWorldServer/WorldServer.cpp +++ b/dWorldServer/WorldServer.cpp @@ -140,8 +140,46 @@ namespace { bool g_ChatDisabled = false; bool g_ChatConnected = false; bool g_WorldShutdownSequenceComplete = false; + // Where the chat server is (to connect again at once when a live update started a new one) + std::string g_ChatIP; + uint32_t g_ChatPort = 0; + // A live update restarted chat: send it who is here once connected + bool g_ChatResyncPending = false; }; // namespace anonymous +// Everyone loaded in here, to a chat server that just started (live update): friends see them online, whispers and +// teams reach them. Nothing is logged as a login or zone change. +void ResendPlayersToChat() { + uint32_t sent = 0; + for (auto* player : PlayerManager::GetAllPlayers()) { + auto* character = player ? player->GetCharacter() : nullptr; + auto* user = character ? character->GetParentUser() : nullptr; + if (!user) continue; + ChatPackets::LoginSessionNotify notify; + notify.playerID = player->GetObjectID(); + notify.playerName = character->GetName(); + notify.zoneID = Game::zoneManager->GetZone()->GetZoneID(); + notify.muteExpire = user->GetMuteExpire(); + notify.gmLevel = player->GetGMLevel(); + notify.resync = true; + ChatServerLink::Send(notify); + sent++; + } + LOG("Sent %u player(s) to the new chat server", sent); +} + +// CHAT_SERVER_READY (live update): connect to the new chat server now rather than at the next retry +void OnChatServerReady() { + g_ChatResyncPending = true; + if (g_ChatConnected) { + g_ChatResyncPending = false; + ResendPlayersToChat(); + return; + } + LOG("A new chat server is up; connecting"); + Game::chatServer->Connect(g_ChatIP.c_str(), g_ChatPort, NET_PASSWORD_EXTERNAL, strnlen(NET_PASSWORD_EXTERNAL, sizeof(NET_PASSWORD_EXTERNAL))); +} + void WorldShutdownSequence(); void WorldShutdownProcess(uint32_t zoneId); void FinalizeShutdown(); @@ -293,6 +331,8 @@ int main(int argc, char** argv) { Game::chatServer = RakNetworkFactory::GetRakPeerInterface(); Game::chatServer->Startup(1, 30, &chatSock, 1); Game::chatServer->Connect(masterIP.c_str(), chatPort, NET_PASSWORD_EXTERNAL, strnlen(NET_PASSWORD_EXTERNAL, sizeof(NET_PASSWORD_EXTERNAL))); + g_ChatIP = masterIP; + g_ChatPort = chatPort; //Set up other things: Game::randomEngine = std::mt19937(time(0)); @@ -483,6 +523,8 @@ int main(int argc, char** argv) { WorldMigration::Update(deltaTime); } + // Character selection has no entities, but its users are moved in a live update + if (zoneID == 0 && deltaTime > 0.0f) WorldMigration::Update(deltaTime); Metrics::StartMeasurement(MetricVariable::PacketHandling); @@ -691,6 +733,10 @@ void HandlePacketChat(Packet* packet) { Game::chatSysAddr = packet->systemAddress; g_ChatConnected = true; + if (g_ChatResyncPending) { + g_ChatResyncPending = false; + ResendPlayersToChat(); + } } if (packet->data[0] == ID_USER_PACKET_ENUM && packet->length >= 4) { @@ -832,6 +878,9 @@ namespace { handlers.On(Master::MIGRATE_PLAYERS, [](const MigratePlayersOrder& order, const SystemAddress&) { WorldMigration::HandleOrder(order); }); handlers.On(Master::MIGRATE_PLAYER_STATE, [](const CarriedPlayerState& state, const SystemAddress&) { WorldMigration::StoreCarriedState(state); }); handlers.On(Master::MIGRATE_STATUS, [](const MigrationStatus& status, const SystemAddress&) { WorldMigration::HandleStatus(status); }); + handlers.On(Master::MIGRATE_PREPARE, [](const MigratePrepare& prepare, const SystemAddress&) { WorldMigration::HandlePrepare(prepare); }); + handlers.On(Master::LIVE_UPDATE_STATUS, [](const LiveUpdateStatus& status, const SystemAddress&) { WorldMigration::HandleLiveUpdateStatus(status); }); + handlers.On(Master::CHAT_SERVER_READY, [](const ChatServerReady&, const SystemAddress&) { OnChatServerReady(); }); handlers.On(Master::PLAYER_ACTION, OnPlayerAction); handlers.On(Master::MESSAGE_CAPTURE_CONTROL, [](const MessageCaptureControl& control, const SystemAddress&) { MessageInspector::Control(control); }); handlers.On(Master::ANNOUNCE, [](const Announcement& announcement, const SystemAddress&) { DashboardNotify::Announce(announcement.title, announcement.message); }); @@ -866,6 +915,8 @@ void LoadPlayer(const SystemAddress& sysAddr) { EntityInfo info{}; info.lot = 1; Entity* player = Game::entityManager->CreateEntity(info, UserManager::Instance()->GetUser(sysAddr)); + // Moved here from another instance: where they stood there (properties don't save it) + WorldMigration::ApplyCarriedPosition(player); auto* characterComponent = player->GetComponent(); if (!characterComponent) return; diff --git a/resources/masterconfig.ini b/resources/masterconfig.ini index a02e78e93..6db5e89b8 100644 --- a/resources/masterconfig.ini +++ b/resources/masterconfig.ini @@ -22,3 +22,24 @@ enable_dashboard=0 # Start the UGC server, which makes and serves the meshes and icons of what players build (0 = disabled, 1 = enabled) # Its settings are in ugcconfig.ini; see docs/UgcServer.md for the client's boot.cfg settings enable_ugc_server=0 + +# Live updates (docs/LiveUpdate.md): moving every server and world instance onto a new build without a restart, +# started from the dashboard, with /liveupdate or with SIGUSR2 to master. Read when one starts. +# Seconds players are warned before they are moved (0-300) +live_update_warn_seconds=10 +# World instances replaced at the same time +live_update_parallel_worlds=4 +# Seconds dead or building players may take before they are moved anyway +live_update_player_wait=30 +# Seconds builders on a property may take before it is saved for its new instance anyway +live_update_property_build_wait=60 +# Seconds players at character select get before they are moved to the new one +live_update_char_select_wait=60 +# Seconds races, minigames and other activity zones get to finish before their players are moved +live_update_activity_wait=1800 +# Seconds the UGC server may take to finish the models it is making before it is stopped +live_update_ugc_drain_timeout=300 +# Seconds auth, chat, the UGC server or the dashboard may take to stop or come back before master starts it again +live_update_service_timeout=30 +# Run the new build's database migrations first (0 or 1) +live_update_run_migrations=1 diff --git a/tests/dCommonTests/CMakeLists.txt b/tests/dCommonTests/CMakeLists.txt index 905cab625..7d7535930 100644 --- a/tests/dCommonTests/CMakeLists.txt +++ b/tests/dCommonTests/CMakeLists.txt @@ -3,6 +3,7 @@ set(DCOMMONTEST_SOURCES "Amf3Tests.cpp" "ToUnderlyingTests.cpp" "InstanceMigrationTests.cpp" + "LiveUpdateTests.cpp" "TestCDFeatureGatingTable.cpp" "TestLDFFormat.cpp" "TestNiPoint3.cpp" diff --git a/tests/dCommonTests/LiveUpdateTests.cpp b/tests/dCommonTests/LiveUpdateTests.cpp new file mode 100644 index 000000000..bca3acb25 --- /dev/null +++ b/tests/dCommonTests/LiveUpdateTests.cpp @@ -0,0 +1,766 @@ +#include + +#include + +#include "LiveUpdateMachine.h" +#include "master/InstanceMigration.h" +#include "master/LiveUpdate.h" + +using namespace LiveUpdate; +using InstanceMigration::eKind; +using InstanceMigration::InstanceView; + +namespace { + InstanceView World(uint32_t zone, uint32_t instance, int32_t players, uint32_t clone = 0) { + InstanceView view; + view.zoneId = zone; + view.instanceId = instance; + view.cloneId = clone; + view.players = players; + view.softCap = 8; + view.hardCap = 12; + view.ready = true; + return view; + } + + // Plays master: records what the machine asks for, and lets the test change what "master" sees + struct FakeMaster final : public IActions { + Observed observed; + bool migrationsFail = false; + bool refuseMoves = false; + uint32_t nextInstance = 100; + uint32_t nextMigration = 1; + std::vector log; + std::vector moves; + std::map, bool> draining; + int chatReady = 0; + + FakeMaster() { + observed.ugc = { true, true, 1 }; + observed.auth = { true, true, 1 }; + observed.chat = { true, true, 1 }; + observed.dashboard = { true, true, 1 }; + } + + ServiceView& Service(eService service) { + switch (service) { + case eService::UGC: return observed.ugc; + case eService::AUTH: return observed.auth; + case eService::CHAT: return observed.chat; + case eService::DASHBOARD: return observed.dashboard; + } + return observed.auth; + } + + void Add(const InstanceView& view, bool activity = false) { observed.worlds.push_back({ view, activity }); } + + WorldView* Find(uint32_t zone, uint32_t instance) { + for (auto& world : observed.worlds) { + if (world.view.zoneId == zone && world.view.instanceId == instance) return &world; + } + return nullptr; + } + + void Remove(uint32_t zone, uint32_t instance) { + std::erase_if(observed.worlds, [&](const WorldView& w) { return w.view.zoneId == zone && w.view.instanceId == instance; }); + } + + // The old server went away and master started the new one, which connected + void Replace(eService service) { + auto& view = Service(service); + view.online = true; + view.connects++; + } + + bool RunMigrations(std::string& error) override { + log.push_back("migrations"); + if (migrationsFail) error = "boom"; + return !migrationsFail; + } + void RetireService(eService service) override { log.push_back(std::string("retire ") + ServiceName(service)); } + void StopService(eService service) override { log.push_back(std::string("stop ") + ServiceName(service)); } + void StartService(eService service) override { log.push_back(std::string("start ") + ServiceName(service)); } + void ChatReady() override { chatReady++; } + std::optional StartWorld(uint32_t zone) override { + const auto id = nextInstance++; + auto view = World(zone, id, 0); + view.ready = false; + Add(view); + log.push_back("start world " + std::to_string(zone) + "/" + std::to_string(id)); + return id; + } + void SetDraining(uint32_t zone, uint32_t instance, bool value) override { + draining[{ zone, instance }] = value; + if (auto* world = Find(zone, instance)) world->view.draining = value; + } + std::optional Migrate(const Move& move, std::string& refusal) override { + if (refuseMoves) { + refusal = "refused"; + return std::nullopt; + } + moves.push_back(move); + return nextMigration++; + } + void StopWorld(uint32_t zone, uint32_t instance) override { + log.push_back("stop world " + std::to_string(zone) + "/" + std::to_string(instance)); + } + + bool Draining(uint32_t zone, uint32_t instance) const { + const auto it = draining.find({ zone, instance }); + return it != draining.end() && it->second; + } + + bool Logged(const std::string& entry) const { return std::find(log.begin(), log.end(), entry) != log.end(); } + }; + + MigrationStatus Status(uint32_t id, InstanceMigration::eState state, uint32_t target = 0, uint16_t moved = 0) { + MigrationStatus status; + status.migrationId = id; + status.state = state; + status.targetInstance = target; + status.moved = moved; + return status; + } + + struct Harness { + FakeMaster master; + Machine machine; + Settings settings; + Clock::time_point now = Clock::time_point{} + std::chrono::hours(1); + + Harness() { + settings.parallelWorlds = 4; + settings.keepZones = { 0, 1000 }; + } + + void Start() { + std::string error; + ASSERT_TRUE(machine.Start(settings, master.observed, now, error)) << error; + } + + void Tick(std::chrono::seconds advance = std::chrono::seconds(0)) { + now += advance; + machine.Tick(master.observed, now, master); + } + + const LiveUpdate::Unit& Row(eUnitKind kind) const { + for (const auto& unit : machine.Units()) if (unit.kind == kind) return unit; + return machine.Units().front(); + } + + const LiveUpdate::Unit& WorldUnit(uint32_t zone, uint32_t instance) const { + for (const auto& unit : machine.Units()) if (unit.kind == eUnitKind::WORLD && unit.zone == zone && unit.instance == instance) return unit; + return machine.Units().front(); + } + + // Auth, chat and UGC come back; chat settles + void ServicesComeBack() { + Tick(); + master.Replace(eService::UGC); + master.Replace(eService::AUTH); + master.Replace(eService::CHAT); + Tick(); + Tick(settings.chatSettle); + } + }; +} + +TEST(LiveUpdateTest, PlanForEachKindOfInstance) { + EXPECT_EQ(PlanWorld(World(1100, 1, 0), false, false, false), eWorldPlan::STOP); + EXPECT_EQ(PlanWorld(World(1000, 1, 0), false, true, false), eWorldPlan::REPLACE_EMPTY); + EXPECT_EQ(PlanWorld(World(1000, 2, 0), false, true, true), eWorldPlan::STOP); // the zone has its new one already + EXPECT_EQ(PlanWorld(World(0, 1, 0), false, false, false), eWorldPlan::REPLACE_EMPTY); + EXPECT_EQ(PlanWorld(World(1100, 1, 3), false, false, false), eWorldPlan::MOVE); + EXPECT_EQ(PlanWorld(World(1150, 1, 2, 77), false, false, false), eWorldPlan::MOVE); + EXPECT_EQ(PlanWorld(World(1203, 1, 2), true, false, false), eWorldPlan::WAIT_THEN_MOVE); + EXPECT_EQ(PlanWorld(World(0, 1, 5), false, true, false), eWorldPlan::WAIT_THEN_MOVE); + // An empty property or private instance is never kept going + EXPECT_EQ(PlanWorld(World(1000, 1, 0, 5), false, true, false), eWorldPlan::STOP); + auto privateWorld = World(1000, 1, 0); + privateWorld.isPrivate = true; + EXPECT_EQ(PlanWorld(privateWorld, false, true, false), eWorldPlan::STOP); +} + +TEST(LiveUpdateTest, RoutingSkipsDrainingAndFullInstances) { + using InstanceMigration::AcceptsNewPlayers; + auto world = World(1100, 1, 3); + EXPECT_TRUE(AcceptsNewPlayers(world, 1100, 0, false)); + EXPECT_FALSE(AcceptsNewPlayers(world, 1200, 0, false)); + EXPECT_FALSE(AcceptsNewPlayers(world, 1100, 5, false)); + world.draining = true; + EXPECT_FALSE(AcceptsNewPlayers(world, 1100, 0, false)); + EXPECT_FALSE(AcceptsNewPlayers(world, 1100, 0, true)); + world.draining = false; + world.players = 8; + EXPECT_FALSE(AcceptsNewPlayers(world, 1100, 0, false)); // soft cap + EXPECT_TRUE(AcceptsNewPlayers(world, 1100, 0, true)); // a friend may go up to the hard cap + world.players = 6; + world.reserved = 2; // seats held for players being moved in + EXPECT_FALSE(AcceptsNewPlayers(world, 1100, 0, false)); + world.reserved = 0; + world.shuttingDown = true; + EXPECT_FALSE(AcceptsNewPlayers(world, 1100, 0, false)); + world.shuttingDown = false; + world.isPrivate = true; + EXPECT_FALSE(AcceptsNewPlayers(world, 1100, 0, true)); + // A replacement still starting takes players: they wait for it + auto starting = World(1100, 2, 0); + starting.ready = false; + EXPECT_TRUE(AcceptsNewPlayers(starting, 1100, 0, false)); +} + +TEST(LiveUpdateTest, LiveUpdateSourceCheck) { + using InstanceMigration::CheckLiveUpdateSource; + using InstanceMigration::eRefusal; + EXPECT_EQ(CheckLiveUpdateSource(World(0, 1, 3)), eRefusal::NONE); + EXPECT_EQ(CheckLiveUpdateSource(World(1150, 1, 3, 9)), eRefusal::NONE); + auto draining = World(1203, 1, 1); + draining.draining = true; + EXPECT_EQ(CheckLiveUpdateSource(draining), eRefusal::NONE); + auto starting = World(1100, 1, 0); + starting.ready = false; + EXPECT_EQ(CheckLiveUpdateSource(starting), eRefusal::NOT_READY); + auto stopping = World(1100, 1, 0); + stopping.shuttingDown = true; + EXPECT_EQ(CheckLiveUpdateSource(stopping), eRefusal::SHUTTING_DOWN); +} + +TEST(LiveUpdateTest, OnlyOneAtATime) { + Harness h; + h.Start(); + std::string error; + EXPECT_FALSE(h.machine.Start(h.settings, h.master.observed, h.now, error)); + EXPECT_FALSE(error.empty()); +} + +TEST(LiveUpdateTest, ServersFirstThenWorldsThenDashboard) { + Harness h; + h.master.Add(World(1100, 1, 3)); + h.Start(); + h.Tick(); + EXPECT_TRUE(h.master.Logged("migrations")); + EXPECT_TRUE(h.master.Logged("retire UGC server")); + EXPECT_TRUE(h.master.Logged("retire auth server")); + EXPECT_TRUE(h.master.Logged("retire chat server")); + EXPECT_FALSE(h.master.Logged("retire dashboard")); + EXPECT_EQ(h.Row(eUnitKind::UGC).state, eUnitState::DRAINING); + EXPECT_EQ(h.Row(eUnitKind::CHAT).state, eUnitState::STOPPING); + // No world moves before chat is back + EXPECT_TRUE(h.master.moves.empty()); + + // The old ones disconnect; master starts new ones + h.master.observed.auth.online = false; + h.master.observed.chat.online = false; + h.Tick(); + EXPECT_EQ(h.Row(eUnitKind::AUTH).state, eUnitState::STARTING); + EXPECT_EQ(h.Row(eUnitKind::CHAT).state, eUnitState::STARTING); + h.master.Replace(eService::AUTH); + h.master.Replace(eService::CHAT); + h.Tick(); + EXPECT_EQ(h.Row(eUnitKind::AUTH).state, eUnitState::STOPPED); + EXPECT_EQ(h.Row(eUnitKind::CHAT).state, eUnitState::READY); + EXPECT_EQ(h.master.chatReady, 1); + EXPECT_TRUE(h.master.moves.empty()) << "worlds wait for the worlds to reconnect to chat"; + h.Tick(h.settings.chatSettle); + EXPECT_EQ(h.Row(eUnitKind::CHAT).state, eUnitState::STOPPED); + h.Tick(); + ASSERT_EQ(h.master.moves.size(), 1u); + EXPECT_EQ(h.master.moves[0].kind, eKind::REPLACE); + + // The world is done; the dashboard waits for the UGC server too + h.machine.OnMigration(Status(1, InstanceMigration::eState::DONE, 100, 3)); + h.master.Remove(1100, 1); + h.Tick(); + EXPECT_EQ(h.WorldUnit(1100, 1).state, eUnitState::STOPPED); + EXPECT_FALSE(h.master.Logged("retire dashboard")); + h.master.Replace(eService::UGC); + h.Tick(); + h.Tick(); + EXPECT_TRUE(h.master.Logged("retire dashboard")); + EXPECT_EQ(h.machine.Phase(), ePhase::RUNNING); + h.master.Replace(eService::DASHBOARD); + h.Tick(); + EXPECT_EQ(h.Row(eUnitKind::DASHBOARD).state, eUnitState::STOPPED); + EXPECT_EQ(h.machine.Phase(), ePhase::DONE); +} + +TEST(LiveUpdateTest, WorldGoesThroughEveryState) { + Harness h; + h.master.Add(World(1100, 1, 3)); + h.Start(); + h.ServicesComeBack(); + const auto& unit = h.WorldUnit(1100, 1); + ASSERT_EQ(h.master.moves.size(), 1u); + EXPECT_EQ(unit.state, eUnitState::STARTING); + EXPECT_FALSE(h.master.moves[0].prepare); + EXPECT_EQ(h.master.moves[0].warnSeconds, h.settings.warnSeconds); + EXPECT_EQ(h.master.moves[0].playerWaitSeconds, h.settings.playerWaitSeconds); + + h.machine.OnMigration(Status(1, InstanceMigration::eState::STARTING_TARGET, 100)); + EXPECT_EQ(unit.state, eUnitState::STARTING); + EXPECT_EQ(unit.replacement, 100u); + h.machine.OnMigration(Status(1, InstanceMigration::eState::WARNING, 100)); + EXPECT_EQ(unit.state, eUnitState::READY); + h.machine.OnMigration(Status(1, InstanceMigration::eState::MOVING, 100, 1)); + EXPECT_EQ(unit.state, eUnitState::DRAINING); + EXPECT_EQ(unit.moved, 1u); + h.machine.OnMigration(Status(1, InstanceMigration::eState::DONE, 100, 3)); + EXPECT_EQ(unit.state, eUnitState::STOPPING); + h.Tick(); + EXPECT_EQ(unit.state, eUnitState::STOPPING) << "the old one is still shutting down"; + h.master.Remove(1100, 1); + h.Tick(); + EXPECT_EQ(unit.state, eUnitState::STOPPED); + EXPECT_EQ(unit.moved, 3u); +} + +TEST(LiveUpdateTest, OldInstanceThatNeverStopsFails) { + Harness h; + h.master.Add(World(1100, 1, 3)); + h.Start(); + h.ServicesComeBack(); + h.machine.OnMigration(Status(1, InstanceMigration::eState::DONE, 100, 3)); + h.Tick(); + h.Tick(h.settings.worldStartTimeout); + EXPECT_EQ(h.WorldUnit(1100, 1).state, eUnitState::FAILED); +} + +TEST(LiveUpdateTest, PropertiesAreSavedFirst) { + Harness h; + h.master.Add(World(1150, 1, 2, 42)); + h.Start(); + h.ServicesComeBack(); + ASSERT_EQ(h.master.moves.size(), 1u); + EXPECT_TRUE(h.master.moves[0].prepare); + EXPECT_EQ(h.master.moves[0].prepareWaitSeconds, h.settings.propertyBuildWaitSeconds); + const auto& unit = h.WorldUnit(1150, 1); + EXPECT_EQ(unit.state, eUnitState::PREPARING); + h.machine.OnMigration(Status(1, InstanceMigration::eState::PREPARED)); + EXPECT_EQ(unit.state, eUnitState::STARTING); + h.machine.OnMigration(Status(1, InstanceMigration::eState::FAILED)); + EXPECT_EQ(unit.state, eUnitState::FAILED); +} + +TEST(LiveUpdateTest, EmptyInstancesStopOrAreReplaced) { + Harness h; + h.master.Add(World(1100, 1, 0)); // nobody there, not kept: stopped + h.master.Add(World(1000, 2, 0)); // Venture Explorer is kept: replaced + h.master.Add(World(1000, 3, 0)); // a second empty one: just stopped + h.Start(); + h.ServicesComeBack(); + EXPECT_TRUE(h.master.Logged("stop world 1100/1")); + EXPECT_TRUE(h.master.Logged("start world 1000/100")); + EXPECT_TRUE(h.master.Logged("stop world 1000/3")); + EXPECT_FALSE(h.master.Logged("stop world 1000/2")) << "stopped only once the new one is ready"; + EXPECT_TRUE(h.master.Draining(1100, 1)); + EXPECT_EQ(h.WorldUnit(1000, 2).state, eUnitState::STARTING); + + h.master.Find(1000, 100)->view.ready = true; + h.Tick(); + EXPECT_TRUE(h.master.Logged("stop world 1000/2")); + EXPECT_EQ(h.WorldUnit(1000, 2).state, eUnitState::STOPPING); + h.master.Remove(1100, 1); + h.master.Remove(1000, 2); + h.master.Remove(1000, 3); + h.Tick(); + EXPECT_EQ(h.WorldUnit(1100, 1).state, eUnitState::STOPPED); + EXPECT_EQ(h.WorldUnit(1000, 2).state, eUnitState::STOPPED); + EXPECT_EQ(h.WorldUnit(1000, 3).state, eUnitState::STOPPED); + EXPECT_TRUE(h.master.moves.empty()); +} + +TEST(LiveUpdateTest, SomeoneArrivingBeforeTheReplacementIsReadyIsMoved) { + Harness h; + h.master.Add(World(1000, 2, 0)); + h.Start(); + h.ServicesComeBack(); + h.master.Find(1000, 2)->view.players = 1; + h.master.Find(1000, 100)->view.ready = true; + h.Tick(); + ASSERT_EQ(h.master.moves.size(), 1u); + EXPECT_EQ(h.master.moves[0].kind, eKind::MERGE); + EXPECT_EQ(h.master.moves[0].target, 100u); + EXPECT_TRUE(h.master.Draining(1000, 2)); +} + +TEST(LiveUpdateTest, ReplacementThatNeverStartsFails) { + Harness h; + h.master.Add(World(1000, 2, 0)); + h.Start(); + h.ServicesComeBack(); + h.Tick(h.settings.worldStartTimeout); + EXPECT_EQ(h.WorldUnit(1000, 2).state, eUnitState::FAILED); + EXPECT_FALSE(h.master.Logged("stop world 1000/2")) << "the old one keeps running"; +} + +TEST(LiveUpdateTest, CharacterSelectDrainsThenMovesWhoIsLeft) { + Harness h; + h.master.Add(World(0, 1, 3)); + h.Start(); + h.ServicesComeBack(); + const auto& unit = h.WorldUnit(0, 1); + EXPECT_EQ(unit.state, eUnitState::WAITING); + EXPECT_TRUE(h.master.Logged("start world 0/100")) << "logins go to a new one at once"; + EXPECT_TRUE(h.master.Draining(0, 1)); + EXPECT_TRUE(h.master.moves.empty()); + EXPECT_EQ(h.machine.Busy(), 0u) << "waiting takes no slot"; + + h.master.Find(0, 100)->view.ready = true; + h.Tick(h.settings.charSelectWait); + ASSERT_EQ(h.master.moves.size(), 1u); + EXPECT_EQ(h.master.moves[0].kind, eKind::MERGE); + EXPECT_EQ(h.master.moves[0].target, 100u); + EXPECT_EQ(unit.state, eUnitState::STARTING); +} + +TEST(LiveUpdateTest, CharacterSelectThatEmptiesStops) { + Harness h; + h.master.Add(World(0, 1, 3)); + h.Start(); + h.ServicesComeBack(); + h.master.Find(0, 1)->view.players = 0; + h.Tick(); + EXPECT_TRUE(h.master.Logged("stop world 0/1")); + EXPECT_EQ(h.WorldUnit(0, 1).state, eUnitState::STOPPING); + EXPECT_TRUE(h.master.moves.empty()); +} + +TEST(LiveUpdateTest, ActivitiesFinishFirst) { + Harness h; + h.master.Add(World(1203, 1, 2), true); + h.Start(); + h.ServicesComeBack(); + EXPECT_EQ(h.WorldUnit(1203, 1).state, eUnitState::WAITING); + EXPECT_TRUE(h.master.Draining(1203, 1)); + h.Tick(h.settings.activityWait - std::chrono::seconds(1)); + EXPECT_TRUE(h.master.moves.empty()); + h.Tick(std::chrono::seconds(1)); + ASSERT_EQ(h.master.moves.size(), 1u); + EXPECT_EQ(h.master.moves[0].kind, eKind::REPLACE); +} + +TEST(LiveUpdateTest, AFewWorldsAtATime) { + Harness h; + h.settings.parallelWorlds = 2; + for (uint32_t i = 1; i <= 5; i++) h.master.Add(World(1100, i, 2)); + h.Start(); + h.ServicesComeBack(); + EXPECT_EQ(h.master.moves.size(), 2u); + EXPECT_EQ(h.machine.Busy(), 2u); + h.Tick(); + EXPECT_EQ(h.master.moves.size(), 2u); + h.machine.OnMigration(Status(1, InstanceMigration::eState::DONE, 100, 2)); + h.Tick(); + EXPECT_EQ(h.master.moves.size(), 3u) << "a stopping instance frees its slot"; +} + +TEST(LiveUpdateTest, BusiestFirstAndCharacterSelectBeforeAll) { + Harness h; + h.settings.parallelWorlds = 1; + h.master.Add(World(1100, 1, 1)); + h.master.Add(World(1200, 2, 6)); + h.master.Add(World(0, 3, 2)); + h.Start(); + const auto& units = h.machine.Units(); + std::vector order; + for (const auto& unit : units) if (unit.kind == eUnitKind::WORLD) order.push_back(unit.zone); + EXPECT_EQ(order, (std::vector{ 0, 1200, 1100 })); +} + +TEST(LiveUpdateTest, RefusedMoveFailsAndTakesPlayersAgain) { + Harness h; + h.master.refuseMoves = true; + h.master.Add(World(1203, 1, 2), true); + h.Start(); + h.ServicesComeBack(); + h.Tick(h.settings.activityWait); + EXPECT_EQ(h.WorldUnit(1203, 1).state, eUnitState::FAILED); + EXPECT_FALSE(h.master.Draining(1203, 1)); +} + +TEST(LiveUpdateTest, FailedMigrationsStopEverything) { + Harness h; + h.master.migrationsFail = true; + h.master.Add(World(1100, 1, 3)); + h.Start(); + h.Tick(); + EXPECT_EQ(h.machine.Phase(), ePhase::FAILED); + EXPECT_EQ(h.Row(eUnitKind::DATABASE).state, eUnitState::FAILED); + EXPECT_FALSE(h.master.Logged("retire auth server")); + EXPECT_TRUE(h.master.moves.empty()); + for (const auto& unit : h.machine.Units()) EXPECT_TRUE(IsFinished(unit.state)); + // And another can be started after + std::string error; + h.master.migrationsFail = false; + EXPECT_TRUE(h.machine.Start(h.settings, h.master.observed, h.now, error)); +} + +TEST(LiveUpdateTest, MigrationsCanBeSwitchedOff) { + Harness h; + h.settings.runMigrations = false; + h.Start(); + h.Tick(); + EXPECT_FALSE(h.master.Logged("migrations")); + EXPECT_EQ(h.Row(eUnitKind::DATABASE).state, eUnitState::SKIPPED); +} + +TEST(LiveUpdateTest, ServerThatDoesntComeBackIsStartedAgainThenFails) { + Harness h; + h.Start(); + h.Tick(); + h.master.observed.auth.online = false; + h.Tick(); + EXPECT_EQ(h.Row(eUnitKind::AUTH).state, eUnitState::STARTING); + for (uint32_t i = 1; i <= h.settings.maxServiceStarts; i++) { + h.Tick(h.settings.serviceTimeout); + EXPECT_EQ(std::count(h.master.log.begin(), h.master.log.end(), "start auth server"), static_cast(i)); + } + h.Tick(h.settings.serviceTimeout); + EXPECT_EQ(h.Row(eUnitKind::AUTH).state, eUnitState::FAILED); +} + +TEST(LiveUpdateTest, ServerThatDoesntStopFails) { + Harness h; + h.Start(); + h.Tick(); + h.Tick(h.settings.serviceTimeout); + EXPECT_EQ(h.Row(eUnitKind::AUTH).state, eUnitState::FAILED); +} + +TEST(LiveUpdateTest, SlowUgcServerIsStopped) { + Harness h; + h.Start(); + h.Tick(); + EXPECT_EQ(h.Row(eUnitKind::UGC).state, eUnitState::DRAINING); + h.Tick(h.settings.ugcDrainTimeout - std::chrono::seconds(1)); + EXPECT_FALSE(h.master.Logged("stop UGC server")); + h.Tick(std::chrono::seconds(1)); + EXPECT_TRUE(h.master.Logged("stop UGC server")); + EXPECT_EQ(h.Row(eUnitKind::UGC).state, eUnitState::STOPPING); + h.master.observed.ugc.online = false; + h.Tick(); + h.master.Replace(eService::UGC); + h.Tick(); + EXPECT_EQ(h.Row(eUnitKind::UGC).state, eUnitState::STOPPED); +} + +TEST(LiveUpdateTest, ServersNotRunningAreSkipped) { + Harness h; + h.master.observed.ugc.enabled = false; + h.master.observed.chat.online = false; + h.Start(); + h.Tick(); + EXPECT_EQ(h.Row(eUnitKind::UGC).state, eUnitState::SKIPPED); + EXPECT_EQ(h.Row(eUnitKind::CHAT).state, eUnitState::SKIPPED); + EXPECT_FALSE(h.master.Logged("retire chat server")); +} + +TEST(LiveUpdateTest, CancelFinishesWhatIsUnderWay) { + Harness h; + h.settings.parallelWorlds = 1; + h.master.Add(World(1100, 1, 2)); + h.master.Add(World(1200, 2, 1)); + h.Start(); + h.ServicesComeBack(); + ASSERT_EQ(h.master.moves.size(), 1u); + EXPECT_TRUE(h.machine.Cancel()); + EXPECT_EQ(h.machine.Phase(), ePhase::CANCELLING); + EXPECT_EQ(h.WorldUnit(1200, 2).state, eUnitState::SKIPPED); + EXPECT_EQ(h.Row(eUnitKind::DASHBOARD).state, eUnitState::SKIPPED); + h.machine.OnMigration(Status(1, InstanceMigration::eState::DONE, 100, 2)); + h.master.Remove(1100, 1); + h.master.Replace(eService::UGC); + h.Tick(); + h.Tick(); + EXPECT_EQ(h.master.moves.size(), 1u); + EXPECT_FALSE(h.master.Logged("retire dashboard")); + EXPECT_EQ(h.machine.Phase(), ePhase::CANCELLED); + EXPECT_FALSE(h.machine.Cancel()); +} + +TEST(LiveUpdateTest, AbortEndsIt) { + Harness h; + h.master.Add(World(1100, 1, 2)); + h.Start(); + h.Tick(); + h.machine.Abort("Master is shutting down"); + EXPECT_EQ(h.machine.Phase(), ePhase::CANCELLED); + for (const auto& unit : h.machine.Units()) EXPECT_TRUE(IsFinished(unit.state)); +} + +TEST(LiveUpdateTest, ShuttingDownInstancesAreLeftOut) { + Harness h; + auto stopping = World(1100, 1, 0); + stopping.shuttingDown = true; + h.master.Add(stopping); + h.master.Add(World(1200, 2, 1)); + h.Start(); + size_t worlds = 0; + for (const auto& unit : h.machine.Units()) if (unit.kind == eUnitKind::WORLD) worlds++; + EXPECT_EQ(worlds, 1u); +} + +TEST(LiveUpdateTest, StatusCarriesEveryUnit) { + Harness h; + h.master.Add(World(1150, 4, 2, 42)); + h.Start(); + LiveUpdateStatus status; + h.machine.FillStatus(status); + status.updateId = 3; + status.by = "Aaron"; + status.startedAt = 1700000000; + ASSERT_EQ(status.units.size(), 6u); + + RakNet::BitStream stream; + status.Serialize(stream); + LiveUpdateStatus read; + ASSERT_TRUE(read.Deserialize(stream)); + EXPECT_EQ(read.updateId, 3u); + EXPECT_EQ(read.phase, ePhase::RUNNING); + EXPECT_EQ(read.by, "Aaron"); + EXPECT_EQ(read.startedAt, 1700000000); + ASSERT_EQ(read.units.size(), 6u); + EXPECT_EQ(read.units[0].kind, eUnitKind::DATABASE); + EXPECT_EQ(read.units[4].kind, eUnitKind::WORLD); + EXPECT_EQ(read.units[4].zoneId, 1150u); + EXPECT_EQ(read.units[4].instanceId, 4u); + EXPECT_EQ(read.units[4].cloneId, 42u); + EXPECT_EQ(read.units[4].players, 2u); + EXPECT_EQ(read.units[5].kind, eUnitKind::DASHBOARD); +} + +TEST(LiveUpdateTest, RequestRoundTripAndLimits) { + LiveUpdateRequest request; + request.action = eAction::START; + request.warnSeconds = 30; + request.requesterId = 1152921510436607007LL; + request.requestedBy = "Aaron"; + RakNet::BitStream stream; + request.Serialize(stream); + LiveUpdateRequest read; + ASSERT_TRUE(read.Deserialize(stream)); + EXPECT_EQ(read.action, eAction::START); + EXPECT_EQ(read.warnSeconds, 30); + EXPECT_EQ(read.requesterId, 1152921510436607007LL); + EXPECT_EQ(read.requestedBy, "Aaron"); + + RakNet::BitStream bad; + bad.Write(9); + EXPECT_FALSE(read.Deserialize(bad)); + RakNet::BitStream tooLong; + tooLong.Write(0); + tooLong.Write(301); + tooLong.Write(0); + tooLong.Write(0); + EXPECT_FALSE(read.Deserialize(tooLong)); +} + +TEST(LiveUpdateTest, ChatHandoffRoundTrip) { + ChatHandoff handoff; + auto& team = handoff.teams.emplace_back(); + team.teamId = 7; + team.leaderId = 100; + team.members = { 100, 101, 102 }; + team.lootFlag = 1; + team.local = true; + team.zoneId = 1100; + team.instanceId = 4; + team.cloneId = 0; + RakNet::BitStream stream; + handoff.Serialize(stream); + ChatHandoff read; + ASSERT_TRUE(read.Deserialize(stream)); + ASSERT_EQ(read.teams.size(), 1u); + EXPECT_EQ(read.teams[0].teamId, 7); + EXPECT_EQ(read.teams[0].leaderId, 100); + EXPECT_EQ(read.teams[0].members, (std::vector{ 100, 101, 102 })); + EXPECT_EQ(read.teams[0].lootFlag, 1u); + EXPECT_TRUE(read.teams[0].local); + EXPECT_EQ(read.teams[0].zoneId, 1100u); + EXPECT_EQ(read.teams[0].instanceId, 4u); +} + +TEST(LiveUpdateTest, CarriedPositionIsOptional) { + CarriedPlayerState state; + state.targetZone = 1150; + state.targetInstance = 3; + state.characterId = 5; + state.hasPosition = true; + state.x = 1.5f; + state.y = -2.0f; + state.z = 300.25f; + state.rotW = 0.5f; + state.rotY = 0.5f; + RakNet::BitStream stream; + state.Serialize(stream); + CarriedPlayerState read; + ASSERT_TRUE(read.Deserialize(stream)); + EXPECT_TRUE(read.hasPosition); + EXPECT_FLOAT_EQ(read.x, 1.5f); + EXPECT_FLOAT_EQ(read.y, -2.0f); + EXPECT_FLOAT_EQ(read.z, 300.25f); + EXPECT_FLOAT_EQ(read.rotW, 0.5f); + EXPECT_FLOAT_EQ(read.rotY, 0.5f); + + // As an older world wrote it: no position at all + RakNet::BitStream old; + old.Write(1150); + old.Write(3); + old.Write(5); + old.Write(0); + old.Write(0); + CarriedPlayerState oldRead; + ASSERT_TRUE(oldRead.Deserialize(old)); + EXPECT_FALSE(oldRead.hasPosition); + EXPECT_EQ(oldRead.characterId, 5); +} + +TEST(LiveUpdateTest, OrderWaitIsOptional) { + MigratePlayersOrder order; + order.migrationId = 1; + order.targetIp = "localhost"; + order.targetPort = 3000; + order.maxWaitSeconds = 45; + RakNet::BitStream stream; + order.Serialize(stream); + MigratePlayersOrder read; + ASSERT_TRUE(read.Deserialize(stream)); + EXPECT_EQ(read.maxWaitSeconds, 45u); + + // Without it (an older master): the default + RakNet::BitStream old; + old.Write(1); + old.Write(1100); + old.Write(2); + old.Write(0); + InstanceMigration::WriteText(old, "localhost", MigratePlayersOrder::MAX_IP); + old.Write(3000); + old.Write(10); + old.Write(10); + old.Write(1); + old.Write(0); + MigratePlayersOrder oldRead; + ASSERT_TRUE(oldRead.Deserialize(old)); + EXPECT_EQ(oldRead.maxWaitSeconds, MigratePlayersOrder::DEFAULT_MAX_WAIT_SECONDS); +} + +TEST(LiveUpdateTest, PrepareAndPreparedStatus) { + MigratePrepare prepare; + prepare.migrationId = 9; + prepare.maxWaitSeconds = 90; + RakNet::BitStream stream; + prepare.Serialize(stream); + MigratePrepare read; + ASSERT_TRUE(read.Deserialize(stream)); + EXPECT_EQ(read.migrationId, 9u); + EXPECT_EQ(read.maxWaitSeconds, 90u); + + MigrationStatus status; + status.state = InstanceMigration::eState::PREPARED; + RakNet::BitStream statusStream; + status.Serialize(statusStream); + MigrationStatus statusRead; + ASSERT_TRUE(statusRead.Deserialize(statusStream)); + EXPECT_EQ(statusRead.state, InstanceMigration::eState::PREPARED); + EXPECT_FALSE(statusRead.Finished()); +} diff --git a/tests/dCommonTests/MessageIdPinTests.cpp b/tests/dCommonTests/MessageIdPinTests.cpp index 1f742dbf9..9676a96fc 100644 --- a/tests/dCommonTests/MessageIdPinTests.cpp +++ b/tests/dCommonTests/MessageIdPinTests.cpp @@ -1763,7 +1763,7 @@ static_assert(static_cast(MessageType::Game::CAN_ITEMS_BE_REFORGED) == static_assert(static_cast(MessageType::Game::NOTIFY_CLIENT_RAIL_START_FAILED) == 1771); static_assert(static_cast(MessageType::Game::GET_IS_ON_RAIL) == 1772); -// MessageType::Master: 19 enumerators +// MessageType::Master: 43 enumerators static_assert(static_cast(MessageType::Master::REQUEST_ZONE_TRANSFER) == 1); static_assert(static_cast(MessageType::Master::REQUEST_ZONE_TRANSFER_RESPONSE) == 2); static_assert(static_cast(MessageType::Master::SERVER_INFO) == 3); @@ -1783,6 +1783,30 @@ static_assert(static_cast(MessageType::Master::SHUTDOWN_UNIVERSE) == 16 static_assert(static_cast(MessageType::Master::AFFIRM_TRANSFER_REQUEST) == 17); static_assert(static_cast(MessageType::Master::AFFIRM_TRANSFER_RESPONSE) == 18); static_assert(static_cast(MessageType::Master::NEW_SESSION_ALERT) == 19); +static_assert(static_cast(MessageType::Master::REQUEST_SERVER_LIST) == 20); +static_assert(static_cast(MessageType::Master::SERVER_LIST_RESPONSE) == 21); +static_assert(static_cast(MessageType::Master::PLAYER_ACTION) == 22); +static_assert(static_cast(MessageType::Master::PLAYER_ACTION_RESULT) == 23); +static_assert(static_cast(MessageType::Master::DATA_CHANGED) == 24); +static_assert(static_cast(MessageType::Master::PLAYER_POSITIONS) == 25); +static_assert(static_cast(MessageType::Master::ANNOUNCE) == 26); +static_assert(static_cast(MessageType::Master::DASHBOARD_SHUTDOWN) == 27); +static_assert(static_cast(MessageType::Master::CONFIG_RELOAD) == 28); +static_assert(static_cast(MessageType::Master::INSTANCE_SHUTDOWN) == 29); +static_assert(static_cast(MessageType::Master::MESSAGE_CAPTURE_CONTROL) == 30); +static_assert(static_cast(MessageType::Master::MESSAGE_CAPTURE_DATA) == 31); +static_assert(static_cast(MessageType::Master::INSTANCE_MIGRATE) == 32); +static_assert(static_cast(MessageType::Master::MIGRATE_PLAYERS) == 33); +static_assert(static_cast(MessageType::Master::MIGRATE_STATUS) == 34); +static_assert(static_cast(MessageType::Master::MIGRATE_PLAYER_STATE) == 35); +static_assert(static_cast(MessageType::Master::SERVER_TRAFFIC) == 36); +static_assert(static_cast(MessageType::Master::UGC_MODELS_MADE) == 37); +static_assert(static_cast(MessageType::Master::MIGRATE_PREPARE) == 38); +static_assert(static_cast(MessageType::Master::LIVE_UPDATE_REQUEST) == 39); +static_assert(static_cast(MessageType::Master::LIVE_UPDATE_STATUS) == 40); +static_assert(static_cast(MessageType::Master::LIVE_UPDATE_RETIRE) == 41); +static_assert(static_cast(MessageType::Master::CHAT_HANDOFF) == 42); +static_assert(static_cast(MessageType::Master::CHAT_SERVER_READY) == 43); // MessageType::Server: 3 enumerators static_assert(static_cast(MessageType::Server::VERSION_CONFIRM) == 0); diff --git a/tests/dGameTests/dNetTests/ChatPacketsTests.cpp b/tests/dGameTests/dNetTests/ChatPacketsTests.cpp index 66b9fd960..b1114309b 100644 --- a/tests/dGameTests/dNetTests/ChatPacketsTests.cpp +++ b/tests/dGameTests/dNetTests/ChatPacketsTests.cpp @@ -659,6 +659,25 @@ TEST_F(ChatPacketsTests, WorldToChatMatchesLegacy) { EXPECT_FALSE(read.Deserialize(stream)); } +// Live updates: players sent again to a new chat server carry a resync byte; every other notify stays as it was +TEST_F(ChatPacketsTests, LoginSessionNotifyResync) { + ChatPackets::LoginSessionNotify plain; + plain.playerID = 0x42; + plain.playerName = "Name"; + ChatPackets::LoginSessionNotify resync = plain; + resync.resync = true; + RakNet::BitStream plainStream; plain.Serialize(plainStream); + RakNet::BitStream resyncStream; resync.Serialize(resyncStream); + EXPECT_EQ(resyncStream.GetNumberOfBitsUsed(), plainStream.GetNumberOfBitsUsed() + 8); + + ChatPackets::LoginSessionNotify readPlain, readResync; + ASSERT_TRUE(readPlain.Deserialize(plainStream)); + ASSERT_TRUE(readResync.Deserialize(resyncStream)); + EXPECT_FALSE(readPlain.resync); + EXPECT_TRUE(readResync.resync); + EXPECT_EQ(readResync.playerName, "Name"); +} + TEST_F(ChatPacketsTests, CreateTeamMatchesLegacy) { for (size_t count = 0; count <= 3; count++) { for (const auto& zone : g_Zones) {