Merge pull request #887 from EmosewaMC/ServerShutdown

Improve server shutdown process
This commit is contained in:
Gie "Max" Vanommeslaeghe 2022-12-18 15:41:58 +01:00 committed by GitHub
commit f41dfaebdf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 114 additions and 54 deletions

View File

@ -25,6 +25,7 @@ namespace Game {
dLogger* logger = nullptr;
dServer* server = nullptr;
dConfig* config = nullptr;
bool shouldShutdown = false;
}
dLogger* SetupLogger();
@ -83,7 +84,7 @@ int main(int argc, char** argv) {
if (Game::config->GetValue("max_clients") != "") maxClients = std::stoi(Game::config->GetValue("max_clients"));
if (Game::config->GetValue("port") != "") ourPort = std::atoi(Game::config->GetValue("port").c_str());
Game::server = new dServer(Game::config->GetValue("external_ip"), ourPort, 0, maxClients, false, true, Game::logger, masterIP, masterPort, ServerType::Auth, Game::config);
Game::server = new dServer(Game::config->GetValue("external_ip"), ourPort, 0, maxClients, false, true, Game::logger, masterIP, masterPort, ServerType::Auth, Game::config, &Game::shouldShutdown);
//Run it until server gets a kill message from Master:
auto t = std::chrono::high_resolution_clock::now();
@ -92,7 +93,7 @@ int main(int argc, char** argv) {
int framesSinceMasterDisconnect = 0;
int framesSinceLastSQLPing = 0;
while (true) {
while (!Game::shouldShutdown) {
//Check if we're still connected to master:
if (!Game::server->GetIsConnectedToMaster()) {
framesSinceMasterDisconnect++;

View File

@ -25,6 +25,7 @@ namespace Game {
dConfig* config = nullptr;
dChatFilter* chatFilter = nullptr;
AssetManager* assetManager = nullptr;
bool shouldShutdown = false;
}
//RakNet includes:
@ -103,7 +104,7 @@ int main(int argc, char** argv) {
if (Game::config->GetValue("max_clients") != "") maxClients = std::stoi(Game::config->GetValue("max_clients"));
if (Game::config->GetValue("port") != "") ourPort = std::atoi(Game::config->GetValue("port").c_str());
Game::server = new dServer(Game::config->GetValue("external_ip"), ourPort, 0, maxClients, false, true, Game::logger, masterIP, masterPort, ServerType::Chat, Game::config);
Game::server = new dServer(Game::config->GetValue("external_ip"), ourPort, 0, maxClients, false, true, Game::logger, masterIP, masterPort, ServerType::Chat, Game::config, &Game::shouldShutdown);
Game::chatFilter = new dChatFilter(Game::assetManager->GetResPath().string() + "/chatplus_en_us", bool(std::stoi(Game::config->GetValue("dont_generate_dcf"))));
@ -114,7 +115,7 @@ int main(int argc, char** argv) {
int framesSinceMasterDisconnect = 0;
int framesSinceLastSQLPing = 0;
while (true) {
while (!Game::shouldShutdown) {
//Check if we're still connected to master:
if (!Game::server->GetIsConnectedToMaster()) {
framesSinceMasterDisconnect++;

View File

@ -21,4 +21,5 @@ namespace Game {
extern RakPeerInterface* chatServer;
extern AssetManager* assetManager;
extern SystemAddress chatSysAddr;
extern bool shouldShutdown;
}

View File

@ -150,7 +150,7 @@ void InstanceManager::RemoveInstance(Instance* instance) {
if (m_Instances[i] == instance) {
instance->SetShutdownComplete(true);
RedirectPendingRequests(instance);
if (!Game::shouldShutdown) RedirectPendingRequests(instance);
delete m_Instances[i];
@ -391,5 +391,5 @@ void Instance::Shutdown() {
Game::server->Send(&bitStream, this->m_SysAddr, false);
Game::logger->Log("Instance", "Triggered world shutdown");
Game::logger->Log("Instance", "Triggered world shutdown for zone/clone/instance %i/%i/%i", GetMapID(), GetCloneID(), GetInstanceID());
}

View File

@ -48,17 +48,18 @@ namespace Game {
InstanceManager* im = nullptr;
dConfig* config = nullptr;
AssetManager* assetManager = nullptr;
bool shouldShutdown = false;
} //namespace Game
bool shutdownSequenceStarted = false;
void ShutdownSequence();
int FinalizeShutdown();
void ShutdownSequence(int signal = -1);
int FinalizeShutdown(int signal = -1);
dLogger* SetupLogger();
void StartAuthServer();
void StartChatServer();
void HandlePacket(Packet* packet);
std::map<uint32_t, std::string> activeSessions;
bool shouldShutdown = false;
SystemAddress authServerMasterPeerSysAddr;
SystemAddress chatServerMasterPeerSysAddr;
int main(int argc, char** argv) {
@ -71,9 +72,9 @@ int main(int argc, char** argv) {
#endif
//Triggers the shutdown sequence at application exit
std::atexit(ShutdownSequence);
signal(SIGINT, [](int) { ShutdownSequence(); });
signal(SIGTERM, [](int) { ShutdownSequence(); });
std::atexit([]() { ShutdownSequence(); });
signal(SIGINT, [](int signal) { ShutdownSequence(EXIT_FAILURE); });
signal(SIGTERM, [](int signal) { ShutdownSequence(EXIT_FAILURE); });
//Create all the objects we need to run our service:
Game::logger = SetupLogger();
@ -241,7 +242,7 @@ int main(int argc, char** argv) {
if (Game::config->GetValue("max_clients") != "") maxClients = std::stoi(Game::config->GetValue("max_clients"));
if (Game::config->GetValue("port") != "") ourPort = std::stoi(Game::config->GetValue("port"));
Game::server = new dServer(Game::config->GetValue("external_ip"), ourPort, 0, maxClients, true, false, Game::logger, "", 0, ServerType::Master, Game::config);
Game::server = new dServer(Game::config->GetValue("external_ip"), ourPort, 0, maxClients, true, false, Game::logger, "", 0, ServerType::Master, Game::config, &Game::shouldShutdown);
//Query for the database for a server labeled "master"
auto* masterLookupStatement = Database::CreatePreppedStmt("SELECT id FROM `servers` WHERE `name` = 'master'");
@ -278,8 +279,8 @@ int main(int argc, char** argv) {
if (Game::config->GetValue("prestart_servers") != "" && Game::config->GetValue("prestart_servers") == "1") {
StartChatServer();
Game::im->GetInstance(0, false, 0)->SetIsReady(true);
Game::im->GetInstance(1000, false, 0)->SetIsReady(true);
Game::im->GetInstance(0, false, 0);
Game::im->GetInstance(1000, false, 0);
StartAuthServer();
}
@ -328,7 +329,7 @@ int main(int argc, char** argv) {
framesSinceLastSQLPing++;
//10m shutdown for universe kill command
if (shouldShutdown) {
if (Game::shouldShutdown) {
if (framesSinceKillUniverseCommand >= 40000) {
//Break main loop and exit
break;
@ -375,9 +376,7 @@ int main(int argc, char** argv) {
t += std::chrono::milliseconds(highFrameRate);
std::this_thread::sleep_until(t);
}
FinalizeShutdown();
exit(EXIT_SUCCESS);
return EXIT_SUCCESS;
return FinalizeShutdown(EXIT_SUCCESS);
}
dLogger* SetupLogger() {
@ -406,9 +405,15 @@ void HandlePacket(Packet* packet) {
Game::im->RemoveInstance(instance); //Delete the old
}
if (packet->systemAddress == chatServerMasterPeerSysAddr && !shouldShutdown) {
if (packet->systemAddress == chatServerMasterPeerSysAddr) {
chatServerMasterPeerSysAddr = UNASSIGNED_SYSTEM_ADDRESS;
StartChatServer();
}
if (packet->systemAddress == authServerMasterPeerSysAddr) {
authServerMasterPeerSysAddr = UNASSIGNED_SYSTEM_ADDRESS;
StartAuthServer();
}
}
if (packet->data[0] == ID_CONNECTION_LOST) {
@ -422,9 +427,15 @@ void HandlePacket(Packet* packet) {
//Game::im->GetInstance(zoneID.GetMapID(), false, 0); //Create the new
}
if (packet->systemAddress == chatServerMasterPeerSysAddr && !shouldShutdown) {
if (packet->systemAddress == chatServerMasterPeerSysAddr) {
chatServerMasterPeerSysAddr = UNASSIGNED_SYSTEM_ADDRESS;
StartChatServer();
}
if (packet->systemAddress == authServerMasterPeerSysAddr) {
authServerMasterPeerSysAddr = UNASSIGNED_SYSTEM_ADDRESS;
StartAuthServer();
}
}
if (packet->data[1] == MASTER) {
@ -520,6 +531,14 @@ void HandlePacket(Packet* packet) {
chatServerMasterPeerSysAddr = copy;
}
if (theirServerType == ServerType::Auth) {
SystemAddress copy;
copy.binaryAddress = packet->systemAddress.binaryAddress;
copy.port = packet->systemAddress.port;
authServerMasterPeerSysAddr = copy;
}
Game::logger->Log("MasterServer", "Received server info, instance: %i port: %i", theirInstanceID, theirPort);
break;
@ -744,17 +763,21 @@ void HandlePacket(Packet* packet) {
case MSG_MASTER_SHUTDOWN_UNIVERSE: {
Game::logger->Log("MasterServer", "Received shutdown universe command, shutting down in 10 minutes.");
shouldShutdown = true;
Game::shouldShutdown = true;
break;
}
default:
Game::logger->Log("MasterServer", "Unknown master packet ID from server: %i", packet->data[3]);
}
}
}
}
void StartChatServer() {
if (Game::shouldShutdown) {
Game::logger->Log("MasterServer", "Currently shutting down. Chat will not be restarted.");
return;
}
#ifdef __APPLE__
//macOS doesn't need sudo to run on ports < 1024
auto result = system(((BinaryPathFinder::GetBinaryDir() / "ChatServer").string() + "&").c_str());
@ -767,9 +790,13 @@ void StartChatServer() {
auto result = system(((BinaryPathFinder::GetBinaryDir() / "ChatServer").string() + "&").c_str());
}
#endif
}
}
void StartAuthServer() {
if (Game::shouldShutdown) {
Game::logger->Log("MasterServer", "Currently shutting down. Auth will not be restarted.");
return;
}
#ifdef __APPLE__
auto result = system(((BinaryPathFinder::GetBinaryDir() / "AuthServer").string() + "&").c_str());
#elif _WIN32
@ -783,21 +810,19 @@ void StartAuthServer() {
#endif
}
void ShutdownSequence() {
void ShutdownSequence(int signal) {
if (shutdownSequenceStarted) {
return;
}
shutdownSequenceStarted = true;
Game::shouldShutdown = true;
if (Game::im) {
for (auto* instance : Game::im->GetInstances()) {
if (instance == nullptr) {
continue;
}
instance->Shutdown();
}
{
CBITSTREAM;
PacketUtils::WriteHeader(bitStream, MASTER, MSG_MASTER_SHUTDOWN);
Game::server->Send(&bitStream, UNASSIGNED_SYSTEM_ADDRESS, true);
Game::logger->Log("MasterServer", "Triggered master shutdown");
}
auto* objIdManager = ObjectIDManager::TryInstance();
@ -810,13 +835,23 @@ void ShutdownSequence() {
auto ticks = 0;
if (!Game::im) {
exit(EXIT_SUCCESS);
FinalizeShutdown(EXIT_FAILURE);
}
// A server might not be finished spinning up yet, remove all of those here.
for (auto instance : Game::im->GetInstances()) {
if (!instance->GetIsReady()) {
Game::im->RemoveInstance(instance);
}
}
for (auto instance : Game::im->GetInstances()) {
instance->SetIsShuttingDown(true);
}
Game::logger->Log("MasterServer", "Attempting to shutdown instances, max 60 seconds...");
while (true) {
auto packet = Game::server->Receive();
if (packet) {
HandlePacket(packet);
@ -835,8 +870,8 @@ void ShutdownSequence() {
done = false;
}
}
if (done) {
if (done && authServerMasterPeerSysAddr == UNASSIGNED_SYSTEM_ADDRESS && chatServerMasterPeerSysAddr == UNASSIGNED_SYSTEM_ADDRESS) {
Game::logger->Log("MasterServer", "Finished shutting down MasterServer!");
break;
}
@ -852,10 +887,10 @@ void ShutdownSequence() {
}
}
FinalizeShutdown();
FinalizeShutdown(signal);
}
int FinalizeShutdown() {
int FinalizeShutdown(int signal) {
//Delete our objects here:
Database::Destroy("MasterServer");
if (Game::config) delete Game::config;
@ -863,6 +898,6 @@ int FinalizeShutdown() {
if (Game::server) delete Game::server;
if (Game::logger) delete Game::logger;
exit(EXIT_SUCCESS);
return EXIT_SUCCESS;
if (signal != EXIT_SUCCESS) exit(signal);
return signal;
}

View File

@ -36,7 +36,7 @@ public:
}
} ReceiveDownloadCompleteCB;
dServer::dServer(const std::string& ip, int port, int instanceID, int maxConnections, bool isInternal, bool useEncryption, dLogger* logger, const std::string masterIP, int masterPort, ServerType serverType, dConfig* config, unsigned int zoneID) {
dServer::dServer(const std::string& ip, int port, int instanceID, int maxConnections, bool isInternal, bool useEncryption, dLogger* logger, const std::string masterIP, int masterPort, ServerType serverType, dConfig* config, bool* shouldShutdown, unsigned int zoneID) {
mIP = ip;
mPort = port;
mZoneID = zoneID;
@ -52,6 +52,7 @@ dServer::dServer(const std::string& ip, int port, int instanceID, int maxConnect
mReplicaManager = nullptr;
mServerType = serverType;
mConfig = config;
mShouldShutdown = shouldShutdown;
//Attempt to start our server here:
mIsOkay = Startup();
@ -124,8 +125,11 @@ Packet* dServer::ReceiveFromMaster() {
ZoneInstanceManager::Instance()->HandleRequestZoneTransferResponse(requestID, packet);
break;
}
case MSG_MASTER_SHUTDOWN:
*mShouldShutdown = true;
break;
//When we handle these packets in World instead dServer, we just return the packet's pointer.
//When we handle these packets in World instead dServer, we just return the packet's pointer.
default:
return packet;

View File

@ -18,7 +18,20 @@ class dServer {
public:
// Default constructor should only used for testing!
dServer() {};
dServer(const std::string& ip, int port, int instanceID, int maxConnections, bool isInternal, bool useEncryption, dLogger* logger, const std::string masterIP, int masterPort, ServerType serverType, dConfig* config, unsigned int zoneID = 0);
dServer(
const std::string& ip,
int port,
int instanceID,
int maxConnections,
bool isInternal,
bool useEncryption,
dLogger* logger,
const std::string masterIP,
int masterPort,
ServerType serverType,
dConfig* config,
bool* shouldShutdown,
unsigned int zoneID = 0);
~dServer();
Packet* ReceiveFromMaster();
@ -64,6 +77,11 @@ private:
RakPeerInterface* mPeer = nullptr;
ReplicaManager* mReplicaManager = nullptr;
NetworkIDManager* mNetIDManager = nullptr;
/**
* Whether or not to shut down the server. Pointer to Game::shouldShutdown.
*/
bool* mShouldShutdown = nullptr;
SocketDescriptor mSocketDescriptor;
std::string mIP;
int mPort;

View File

@ -70,11 +70,11 @@ namespace Game {
RakPeerInterface* chatServer = nullptr;
std::mt19937 randomEngine;
SystemAddress chatSysAddr;
}
bool shouldShutdown = false;
} // namespace Game
bool chatDisabled = false;
bool chatConnected = false;
bool worldShutdownSequenceStarted = false;
bool worldShutdownSequenceComplete = false;
void WorldShutdownSequence();
void WorldShutdownProcess(uint32_t zoneId);
@ -200,7 +200,7 @@ int main(int argc, char** argv) {
LootGenerator::Instance();
Game::chatFilter = new dChatFilter(Game::assetManager->GetResPath().string() + "/chatplus_en_us", bool(std::stoi(Game::config->GetValue("dont_generate_dcf"))));
Game::server = new dServer(masterIP, ourPort, instanceID, maxClients, false, true, Game::logger, masterIP, masterPort, ServerType::World, Game::config, zoneID);
Game::server = new dServer(masterIP, ourPort, instanceID, maxClients, false, true, Game::logger, masterIP, masterPort, ServerType::World, Game::config, &Game::shouldShutdown, zoneID);
//Connect to the chat server:
int chatPort = 1501;
@ -315,9 +315,9 @@ int main(int argc, char** argv) {
framesSinceMasterDisconnect++;
int framesToWaitForMaster = ready ? 10 : 200;
if (framesSinceMasterDisconnect >= framesToWaitForMaster && !worldShutdownSequenceStarted) {
if (framesSinceMasterDisconnect >= framesToWaitForMaster && !Game::shouldShutdown) {
Game::logger->Log("WorldServer", "Game loop running but no connection to master for %d frames, shutting down", framesToWaitForMaster);
worldShutdownSequenceStarted = true;
Game::shouldShutdown = true;
}
} else framesSinceMasterDisconnect = 0;
@ -413,7 +413,7 @@ int main(int argc, char** argv) {
//If we haven't had any players for a while, time out and shut down:
if (framesSinceLastUser == (cloneID != 0 ? 4000 : 40000)) {
worldShutdownSequenceStarted = true;
Game::shouldShutdown = true;
}
} else {
framesSinceLastUser = 0;
@ -470,7 +470,7 @@ int main(int argc, char** argv) {
}
}
if (worldShutdownSequenceStarted && !worldShutdownSequenceComplete) {
if (Game::shouldShutdown && !worldShutdownSequenceComplete) {
WorldShutdownProcess(zoneID);
break;
}
@ -789,7 +789,7 @@ void HandlePacket(Packet* packet) {
}
case MSG_MASTER_SHUTDOWN: {
worldShutdownSequenceStarted = true;
Game::shouldShutdown = true;
Game::logger->Log("WorldServer", "Got shutdown request from master, zone (%i), instance (%i)", Game::server->GetZoneID(), Game::server->GetInstanceID());
break;
}
@ -1259,11 +1259,11 @@ void WorldShutdownProcess(uint32_t zoneId) {
}
void WorldShutdownSequence() {
if (worldShutdownSequenceStarted || worldShutdownSequenceComplete) {
if (Game::shouldShutdown || worldShutdownSequenceComplete) {
return;
}
worldShutdownSequenceStarted = true;
Game::shouldShutdown = true;
Game::logger->Log("WorldServer", "Zone (%i) instance (%i) shutting down outside of main loop!", Game::server->GetZoneID(), instanceID);
WorldShutdownProcess(Game::server->GetZoneID());