feat(master): outdated instances take nobody new and stop once empty

Instances on an old binary or old zone files are marked outdated: routing,
private passwords and merges skip them. Outdated properties remind their
players every 10 minutes that an update is waiting (ANNOUNCE) and stop once
empty. A new instance of the same property waits for the old one to be gone
before its world server starts, so two worlds never save one property.
The server list carries the flag after the endpoints (read when present).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-09-30 04:03:15 -05:00
parent c6831ac80e
commit 8f891229e6
9 changed files with 327 additions and 9 deletions

View File

@@ -1,6 +1,7 @@
#include "InstanceManager.h"
#include <string>
#include <algorithm>
#include <cctype>
#include <chrono>
#include "Game.h"
#include "dServer.h"
@@ -14,6 +15,8 @@
#include "BitStreamUtils.h"
#include "ServiceType.h"
#include "MessageType/Master.h"
#include "OutdatedInstances.h"
#include "master/DashboardMessages.h"
#include "Start.h"
@@ -76,8 +79,19 @@ const InstancePtr& InstanceManager::CreateInstance(LWOMAPID mapID, LWOCLONEID cl
uint32_t port = GetFreePort();
auto newInstance = make_unique<Instance>(mExternalIP, port, mapID, ++m_LastInstanceID, cloneID, softCap, maxPlayers);
//Start the actual process:
StartWorldServer(mapID, port, m_LastInstanceID, maxPlayers, cloneID);
// A property whose old instance still runs: its world server starts once that one is gone (never two instances
// saving the same property). Requests for it wait meanwhile.
std::vector<InstanceMigration::InstanceView> views;
for (const auto& instance : m_Instances) {
if (instance && !instance->GetShutdownComplete()) views.push_back(instance->View());
}
if (OutdatedInstances::MustWaitForOld(views, mapID, cloneID)) {
newInstance->SetIsWaitingForOld(true);
LOG("Instance %i/%i/%i waits for the old instance of the property to stop before it starts", mapID, m_LastInstanceID, cloneID);
} else {
//Start the actual process:
StartWorldServer(mapID, port, m_LastInstanceID, maxPlayers, cloneID);
}
m_Instances.push_back(std::move(newInstance));
if (m_OnInstancesChanged) m_OnInstancesChanged();
@@ -163,6 +177,69 @@ void InstanceManager::RemoveInstance(const InstancePtr& instance) {
break;
}
}
if (!m_IsShuttingDown && !Game::ShouldShutdown()) StartWaitingInstances();
}
void InstanceManager::StartWaitingInstances() {
std::vector<InstanceMigration::InstanceView> views;
for (const auto& instance : m_Instances) {
if (instance && !instance->GetShutdownComplete()) views.push_back(instance->View());
}
for (const auto& instance : m_Instances) {
if (!instance || !instance->GetIsWaitingForOld()) continue;
if (OutdatedInstances::MustWaitForOld(views, instance->GetMapID(), instance->GetCloneID())) continue;
instance->SetIsWaitingForOld(false);
LOG("The old instance of property %i/%i is gone; starting instance %i", instance->GetMapID(), instance->GetCloneID(), instance->GetInstanceID());
StartWorldServer(instance->GetMapID(), instance->GetPort(), instance->GetInstanceID(), instance->GetHardCap(), instance->GetCloneID());
}
}
uint32_t InstanceManager::MarkOutdated(const std::function<bool(const Instance&)>& which) {
uint32_t marked = 0;
for (const auto& instance : m_Instances) {
// One still waiting to start runs the new binaries and files already
if (!instance || instance->GetShutdownComplete() || instance->GetIsWaitingForOld() || instance->GetIsOutdated() || !which(*instance)) continue;
instance->SetIsOutdated(true);
marked++;
}
if (marked && m_OnInstancesChanged) m_OnInstancesChanged();
return marked;
}
void InstanceManager::UpdateOutdatedInstances() {
const auto now = std::chrono::steady_clock::now();
if (now < m_NextOutdatedCheck) return;
m_NextOutdatedCheck = now + std::chrono::seconds(1);
if (m_IsShuttingDown) return;
bool changed = false;
std::optional<std::set<uint32_t>> keepZones;
for (const auto& instance : m_Instances) {
if (!instance || !instance->GetIsOutdated() || instance->GetShutdownComplete()) continue;
if (!keepZones) {
keepZones.emplace();
for (auto part : GeneralUtils::SplitString(Game::config->GetValue("prestart_worlds", "0,1000"), ',')) {
std::erase_if(part, [](const char c) { return std::isspace(static_cast<unsigned char>(c)); });
if (const auto zone = GeneralUtils::TryParse<uint32_t>(part)) keepZones->insert(*zone);
}
}
const auto view = instance->View();
if (OutdatedInstances::NoticeDue(view, instance->GetLastUpdateNotice(), now)) {
Announcement notice;
notice.title = OutdatedInstances::NOTICE_TITLE;
notice.message = OutdatedInstances::NOTICE_MESSAGE;
MasterPackets::SendTo(instance->GetSysAddr(), notice);
instance->SetLastUpdateNotice(now);
LOG("Told the %i player(s) on outdated property %i/%i/%i that an update is waiting", view.players, view.zoneId, view.cloneId, view.instanceId);
}
const bool pending = !instance->GetPendingAffirmations().empty() || !instance->GetPendingRequests().empty();
if (OutdatedInstances::ShouldStop(view, keepZones->contains(view.zoneId), pending)) {
LOG("Outdated instance %i/%i/%i is empty; stopping it", view.zoneId, view.cloneId, view.instanceId);
instance->Shutdown();
instance->SetIsShuttingDown(true);
changed = true;
}
}
if (changed && m_OnInstancesChanged) m_OnInstancesChanged();
}
void InstanceManager::ReadyInstance(const InstancePtr& instance) {
@@ -304,8 +381,9 @@ const InstancePtr& InstanceManager::FindPrivateInstance(const std::string& passw
for (const auto& instance : m_Instances) {
if (!instance) continue;
// A private instance being replaced (live update) takes nobody new: its replacement has the same password
if (!instance->GetIsPrivate() || instance->GetIsDraining() || instance->GetIsShuttingDown() || instance->GetShutdownComplete()) {
// A private instance being replaced (live update) or on an old version takes nobody new: its replacement has
// the same password
if (!instance->GetIsPrivate() || instance->GetIsDraining() || instance->GetIsOutdated() || instance->GetIsShuttingDown() || instance->GetShutdownComplete()) {
continue;
}

View File

@@ -1,9 +1,12 @@
#pragma once
#include <chrono>
#include <functional>
#include <algorithm>
#include <optional>
#include "SpareBackoff.h"
#include <map>
#include <set>
#include <vector>
#include "dCommonVars.h"
#include "IServerOperations.h"
@@ -62,6 +65,17 @@ public:
// Its players are being moved to another instance (InstanceMigration.h): nobody new is sent here
bool GetIsDraining() const { return m_IsDraining; }
void SetIsDraining(bool value) { m_IsDraining = value; }
// Started before a live update, or on zone files that changed since: nobody new is sent here, and it stops once
// empty (OutdatedInstances.h)
bool GetIsOutdated() const { return m_IsOutdated; }
void SetIsOutdated(bool value) { m_IsOutdated = value; }
// A property instance waiting for the outdated instance of the same property to stop before its world server is
// started (so the two never both save it); requests for it wait as pending requests
bool GetIsWaitingForOld() const { return m_WaitingForOld; }
void SetIsWaitingForOld(bool value) { m_WaitingForOld = value; }
// When its players were last told an update is waiting (outdated properties)
std::optional<std::chrono::steady_clock::time_point> GetLastUpdateNotice() const { return m_LastUpdateNotice; }
void SetLastUpdateNotice(std::chrono::steady_clock::time_point value) { m_LastUpdateNotice = value; }
// Seats held for players being moved in; they count towards the caps until the move is over
int GetReserved() const { return m_Reserved; }
void SetReserved(int value) { m_Reserved = std::max(0, value); }
@@ -113,6 +127,7 @@ public:
view.isPrivate = GetIsPrivate();
view.shuttingDown = GetIsShuttingDown() || GetShutdownComplete();
view.draining = GetIsDraining();
view.outdated = GetIsOutdated();
return view;
}
@@ -129,6 +144,9 @@ private:
bool m_Ready{};
bool m_IsShuttingDown{};
bool m_IsDraining{};
bool m_IsOutdated{};
bool m_WaitingForOld{};
std::optional<std::chrono::steady_clock::time_point> m_LastUpdateNotice{};
int m_Reserved{};
std::vector<PendingInstanceRequest> m_PendingRequests{};
std::vector<PendingInstanceRequest> m_PendingAffirmations{};
@@ -192,6 +210,16 @@ public:
// Start an instance of each zone that has fewer instances with room than its spare_instances (one per call)
void KeepSpareInstances();
/**
* Outdated instances (OutdatedInstances.h): reminds players on outdated properties that an update is waiting, and
* stops outdated instances nobody is in or on the way to (not public instances of prestart_worlds zones: the live
* update or reload replaces those). Called every frame; does its work once a second.
*/
void UpdateOutdatedInstances();
// Mark the running instances which() picks outdated (old binary or old zone files); returns how many were new
uint32_t MarkOutdated(const std::function<bool(const Instance&)>& which);
private:
std::string mExternalIP;
std::vector<std::unique_ptr<Instance>> m_Instances;
@@ -210,5 +238,8 @@ private:
int GetSoftCap(LWOMAPID mapID);
int GetHardCap(LWOMAPID mapID);
const InstancePtr& CreateInstance(LWOMAPID mapID, LWOCLONEID cloneID);
// Start the world servers of property instances whose outdated predecessor is gone
void StartWaitingInstances();
std::function<void()> m_OnInstancesChanged;
std::chrono::steady_clock::time_point m_NextOutdatedCheck{};
};

View File

@@ -158,6 +158,7 @@ namespace {
entry.isPrivate = inst->GetIsPrivate() ? 1 : 0;
entry.state = inst->GetIsShuttingDown() ? eState::STOPPING : !inst->GetIsReady() ? eState::STARTING :
inst->GetIsDraining() ? eState::DRAINING : eState::READY;
entry.outdated = inst->GetIsOutdated() ? 1 : 0;
auto& endpoint = response.endpoints.emplace_back();
endpoint.type = ServiceType::WORLD;
endpoint.zoneID = entry.mapID;
@@ -632,6 +633,8 @@ int main(int argc, char** argv) {
LiveUpdateCoordinator::Update();
CDClientReloader::Update();
WorldReloader::Update();
// Instances on an old binary or old zone files: reminders on properties, and stopping empty ones
Game::im->UpdateOutdatedInstances();
CheckPlayerActionTimeouts();
// Spare instances for busy zones (zone_limits), checked every few seconds

View File

@@ -0,0 +1,78 @@
#ifndef __OUTDATEDINSTANCES__H__
#define __OUTDATEDINSTANCES__H__
#include <chrono>
#include <cstdint>
#include <optional>
#include <set>
#include <vector>
#include "master/InstanceMigration.h"
/**
* World instances on an old version: started before a live update (old binary) or on zone files that changed since
* (docs/LiveUpdate.md, docs/WorldHotReload.md). Master marks them outdated; nobody new is sent to one (AcceptsNewPlayers)
* and a request for its zone goes to a new instance instead.
*
* Properties (clone instances) are never replaced or moved: players may have building in progress that isn't saved.
* An outdated property keeps running until everyone left, reminding its players now and then that an update is
* waiting, then stops. A new instance of the same property only starts once the old one is gone, so two instances of
* one property never both save its models; whoever asks for it meanwhile waits for the new one.
*
* Header only, no master state, so it is unit tested.
*/
namespace OutdatedInstances {
using Clock = std::chrono::steady_clock;
// Between two reminders to players on an outdated property
constexpr auto NOTICE_INTERVAL = std::chrono::minutes(10);
inline constexpr const char* NOTICE_TITLE = "Server update";
inline constexpr const char* NOTICE_MESSAGE =
"A server update is available. This property keeps running on the old version until everyone has left it: "
"leave and come back to get the update. Nothing you built is lost.";
// Property instances hold building in progress that lives only in that world
inline bool IsProperty(const InstanceMigration::InstanceView& view) {
return view.cloneId != 0;
}
/**
* Whether players on this instance are due a reminder: an outdated property with players, once at first and then
* every interval while they stay. lastNotice is when the last one went out (none yet: nullopt).
*/
inline bool NoticeDue(const InstanceMigration::InstanceView& view, std::optional<Clock::time_point> lastNotice, Clock::time_point now,
Clock::duration interval = NOTICE_INTERVAL) {
if (!view.outdated || !IsProperty(view) || view.players <= 0 || !view.ready || view.shuttingDown) return false;
return !lastNotice || now - *lastNotice >= interval;
}
/**
* Whether an outdated instance nobody is in or on the way to should stop now. Instances a live update or reload
* handles itself are left to it: ones being emptied (draining), character selection, and public instances of zones
* that always keep one (they get their new instance first).
* hasPending: players on their way (affirmations or requests waiting for it)
*/
inline bool ShouldStop(const InstanceMigration::InstanceView& view, bool keepZone, bool hasPending) {
if (!view.outdated || view.shuttingDown || view.draining || !view.ready) return false;
if (view.Load() > 0 || hasPending) return false;
if (view.zoneId == 0) return false;
if (!IsProperty(view) && !view.isPrivate && keepZone) return false;
return true;
}
/**
* Whether a new instance of property zone/clone must wait before it starts: an instance of the same property is still
* running (outdated, possibly still saving on its way out). Starting the new one then would have two worlds save the
* same property.
*/
inline bool MustWaitForOld(const std::vector<InstanceMigration::InstanceView>& running, uint32_t zone, uint32_t clone) {
if (clone == 0) return false;
for (const auto& view : running) {
if (view.zoneId == zone && view.cloneId == clone && view.outdated) return true;
}
return false;
}
}
#endif //!__OUTDATEDINSTANCES__H__

View File

@@ -257,6 +257,7 @@ namespace MasterPackets {
bitStream.Write(endpoint.port);
bitStream.Write(endpoint.host);
}
for (const auto& instance : instances) bitStream.Write(instance.outdated);
}
bool ServerListResponse::Deserialize(RakNet::BitStream& bitStream) {
@@ -296,6 +297,9 @@ namespace MasterPackets {
VALIDATE_READ(bitStream.Read(endpoint.port));
VALIDATE_READ(bitStream.Read(endpoint.host));
}
if (bitStream.GetNumberOfUnreadBits() >= instances.size() * 8) {
for (auto& instance : instances) VALIDATE_READ(bitStream.Read(instance.outdated));
}
return true;
}

View File

@@ -267,6 +267,9 @@ namespace MasterPackets {
uint32_t port{};
uint8_t isPrivate{};
eState state{ eState::READY }; // written after the UGC fields, one byte per instance
// On an old binary or old zone files: takes nobody new and stops once empty. Written after the endpoints,
// one byte per instance, and read only when there (older masters didn't write it)
uint8_t outdated{};
};
uint8_t authOnline{};

View File

@@ -89,6 +89,9 @@ namespace InstanceMigration {
bool isPrivate{};
bool shuttingDown{};
bool draining{}; // its players are being moved away; nobody new is sent there
// Started before a live update or on zone files that changed since (old binary or old files): nobody new is
// sent there, and it stops once empty (OutdatedInstances.h)
bool outdated{};
int32_t Load() const { return players + reserved; }
};
@@ -108,6 +111,7 @@ namespace InstanceMigration {
TARGET_FULL,
NO_TARGET,
MASTER_SHUTTING_DOWN,
TARGET_OUTDATED,
};
inline const char* Describe(eRefusal refusal) {
@@ -126,6 +130,7 @@ namespace InstanceMigration {
case eRefusal::TARGET_FULL: return "The other instance doesn't have room for everyone";
case eRefusal::NO_TARGET: return "No other instance of that zone has room for everyone";
case eRefusal::MASTER_SHUTTING_DOWN: return "The server is shutting down";
case eRefusal::TARGET_OUTDATED: return "That instance runs an old version and takes nobody new";
}
return "Unknown reason";
}
@@ -145,8 +150,8 @@ namespace InstanceMigration {
/**
* A live update replaces every instance, so it moves what CheckSource refuses too: character selection, private
* instances, properties (after MIGRATE_PREPARE) and activity zones (once their players had time to finish). Only
* an instance that isn't running properly is left alone. A draining one is fine: the live update drains activity
* instances and activity zones (once their players had time to finish). Properties are never moved (live updates
* and reloads leave them out). Only an instance that isn't running properly is left alone. A draining one is fine: the live update drains activity
* zones and character selection itself before moving whoever is left.
*/
inline eRefusal CheckLiveUpdateSource(const InstanceView& source) {
@@ -157,12 +162,13 @@ namespace InstanceMigration {
/**
* Whether master may send a new player to this instance of zone/clone (InstanceManager::FindInstance): not
* private, not shutting down, not draining (being emptied for a live update or migration), and with room: under
* private, not shutting down, not draining (being emptied for a live update or migration), not outdated (on an old
* binary or old zone files), and with room: under
* the soft cap, or under the hard cap for players following a friend. Seats held for players being moved in count.
*/
inline bool AcceptsNewPlayers(const InstanceView& instance, uint32_t zone, uint32_t clone, bool friendTransfer) {
if (instance.zoneId != zone || instance.cloneId != clone) return false;
if (instance.isPrivate || instance.shuttingDown || instance.draining) return false;
if (instance.isPrivate || instance.shuttingDown || instance.draining || instance.outdated) return false;
return instance.Load() < (friendTransfer ? instance.hardCap : instance.softCap);
}
@@ -174,6 +180,7 @@ namespace InstanceMigration {
if (target.isPrivate) return eRefusal::PRIVATE_INSTANCE;
if (target.shuttingDown) return eRefusal::SHUTTING_DOWN;
if (target.draining) return eRefusal::ALREADY_MIGRATING;
if (target.outdated) return eRefusal::TARGET_OUTDATED;
if (!target.ready) return eRefusal::NOT_READY;
if (target.Load() + source.players > target.hardCap) return eRefusal::TARGET_FULL;
return eRefusal::NONE;
@@ -227,7 +234,7 @@ namespace InstanceMigration {
if (takingIn[i] || list[i].players <= 0) continue;
// Into the fullest one it still fits in; only ones at least as full (later in the list)
for (size_t j = list.size(); j-- > i + 1;) {
if (mergedAway[j] || list[j].Load() + list[i].players > list[j].softCap) continue;
if (mergedAway[j] || list[j].outdated || list[j].Load() + list[i].players > list[j].softCap) continue;
suggestions.push_back({ zone, list[i].instanceId, list[j].instanceId, list[i].players, list[j].Load() + list[i].players });
list[j].reserved += list[i].players;
mergedAway[i] = true;

View File

@@ -5,6 +5,7 @@ set(DCOMMONTEST_SOURCES
"ToUnderlyingTests.cpp"
"InstanceMigrationTests.cpp"
"LiveUpdateTests.cpp"
"OutdatedInstancesTests.cpp"
"TestCDFeatureGatingTable.cpp"
"TestLDFFormat.cpp"
"TestNiPoint3.cpp"

View File

@@ -0,0 +1,113 @@
#include <gtest/gtest.h>
#include "OutdatedInstances.h"
using namespace OutdatedInstances;
using InstanceMigration::InstanceView;
namespace {
InstanceView View(uint32_t zone, uint32_t instance, int32_t players, uint32_t clone = 0) {
InstanceView view;
view.zoneId = zone;
view.instanceId = instance;
view.cloneId = clone;
view.players = players;
view.softCap = 8;
view.hardCap = 12;
view.ready = true;
return view;
}
const Clock::time_point START = Clock::time_point{} + std::chrono::hours(1);
}
TEST(OutdatedInstancesTest, NoticeOnceThenEveryTenMinutes) {
auto property = View(1150, 1, 2, 42);
EXPECT_FALSE(NoticeDue(property, std::nullopt, START)) << "not outdated";
property.outdated = true;
EXPECT_TRUE(NoticeDue(property, std::nullopt, START)) << "at once";
EXPECT_FALSE(NoticeDue(property, START, START + std::chrono::minutes(1)));
EXPECT_FALSE(NoticeDue(property, START, START + std::chrono::minutes(9) + std::chrono::seconds(59)));
EXPECT_TRUE(NoticeDue(property, START, START + std::chrono::minutes(10)));
EXPECT_TRUE(NoticeDue(property, START, START + std::chrono::minutes(25)));
EXPECT_EQ(std::chrono::duration_cast<std::chrono::seconds>(NOTICE_INTERVAL).count(), 600);
}
TEST(OutdatedInstancesTest, NoticeOnlyForPropertiesWithPlayers) {
auto world = View(1100, 1, 3);
world.outdated = true;
EXPECT_FALSE(NoticeDue(world, std::nullopt, START)) << "other worlds' players are moved instead";
auto empty = View(1150, 2, 0, 42);
empty.outdated = true;
EXPECT_FALSE(NoticeDue(empty, std::nullopt, START));
auto stopping = View(1150, 3, 1, 42);
stopping.outdated = true;
stopping.shuttingDown = true;
EXPECT_FALSE(NoticeDue(stopping, std::nullopt, START));
auto starting = View(1150, 4, 1, 42);
starting.outdated = true;
starting.ready = false;
EXPECT_FALSE(NoticeDue(starting, std::nullopt, START));
}
TEST(OutdatedInstancesTest, StopsOnceEmpty) {
auto property = View(1150, 1, 2, 42);
property.outdated = true;
EXPECT_FALSE(ShouldStop(property, false, false)) << "players still there";
property.players = 0;
EXPECT_TRUE(ShouldStop(property, false, false));
EXPECT_TRUE(ShouldStop(property, true, false)) << "a property is never kept, even in a kept zone";
EXPECT_FALSE(ShouldStop(property, false, true)) << "someone on the way";
property.reserved = 1;
EXPECT_FALSE(ShouldStop(property, false, false)) << "a seat held for someone moving in";
property.reserved = 0;
property.outdated = false;
EXPECT_FALSE(ShouldStop(property, false, false)) << "not outdated";
}
TEST(OutdatedInstancesTest, LeavesWhatTheUpdateHandlesAlone) {
auto world = View(1100, 1, 0);
world.outdated = true;
EXPECT_TRUE(ShouldStop(world, false, false));
EXPECT_FALSE(ShouldStop(world, true, false)) << "a kept zone's public instance gets its replacement first";
auto privateWorld = world;
privateWorld.isPrivate = true;
EXPECT_TRUE(ShouldStop(privateWorld, true, false));
auto charSelect = View(0, 2, 0);
charSelect.outdated = true;
EXPECT_FALSE(ShouldStop(charSelect, false, false));
auto draining = world;
draining.draining = true;
EXPECT_FALSE(ShouldStop(draining, false, false)) << "being emptied by a move";
auto stopping = world;
stopping.shuttingDown = true;
EXPECT_FALSE(ShouldStop(stopping, false, false)) << "already stopping";
}
TEST(OutdatedInstancesTest, SamePropertyWaitsForTheOldInstance) {
std::vector<InstanceView> running{ View(1150, 1, 2, 42), View(1150, 2, 1, 43), View(1100, 3, 4) };
EXPECT_FALSE(MustWaitForOld(running, 1150, 42)) << "the old one isn't outdated";
running[0].outdated = true;
EXPECT_TRUE(MustWaitForOld(running, 1150, 42));
running[0].shuttingDown = true;
EXPECT_TRUE(MustWaitForOld(running, 1150, 42)) << "still saving on its way out";
EXPECT_FALSE(MustWaitForOld(running, 1150, 43)) << "another property";
EXPECT_FALSE(MustWaitForOld(running, 1151, 42)) << "another zone";
running[2].outdated = true;
EXPECT_FALSE(MustWaitForOld(running, 1100, 0)) << "public worlds start a new instance at once";
running.erase(running.begin());
EXPECT_FALSE(MustWaitForOld(running, 1150, 42)) << "the old one is gone";
}
TEST(OutdatedInstancesTest, MergesNeverGoIntoAnOutdatedInstance) {
using namespace InstanceMigration;
const auto source = View(1100, 1, 2);
auto old = View(1100, 2, 3);
auto fresh = View(1100, 3, 1);
EXPECT_EQ(CheckMergeTarget(old, source), eRefusal::NONE);
old.outdated = true;
EXPECT_EQ(CheckMergeTarget(old, source), eRefusal::TARGET_OUTDATED);
EXPECT_EQ(PickMergeTarget({ old, fresh }, source), std::optional<uint32_t>(3u));
EXPECT_EQ(PickMergeTarget({ old }, source), std::nullopt);
for (const auto& suggestion : SuggestMerges({ source, old })) EXPECT_NE(suggestion.targetInstance, 2u);
}