diff --git a/dDashboardServer/DashboardServer.cpp b/dDashboardServer/DashboardServer.cpp index a68548853..236808788 100644 --- a/dDashboardServer/DashboardServer.cpp +++ b/dDashboardServer/DashboardServer.cpp @@ -115,6 +115,9 @@ namespace Game { namespace ServerState { ServerStatus g_AuthStatus{}; ServerStatus g_ChatStatus{}; + ServerStatus g_UgcStatus{}; + bool g_UgcEnabled{}; + uint32_t g_UgcPid{}; std::vector g_WorldInstances{}; std::mutex g_StatusMutex{}; } @@ -197,10 +200,11 @@ namespace { void OnServerList(const MasterPackets::ServerListResponse& list, const SystemAddress&) { std::lock_guard lock(ServerState::g_StatusMutex); - ServerState::g_AuthStatus.online = list.authOnline != 0; - ServerState::g_AuthStatus.lastSeen = std::chrono::steady_clock::now(); - ServerState::g_ChatStatus.online = list.chatOnline != 0; - ServerState::g_ChatStatus.lastSeen = std::chrono::steady_clock::now(); + ServerState::g_AuthStatus.Set(list.authOnline != 0); + ServerState::g_ChatStatus.Set(list.chatOnline != 0); + ServerState::g_UgcStatus.Set(list.ugcOnline != 0); + ServerState::g_UgcEnabled = list.ugcEnabled != 0; + ServerState::g_UgcPid = list.ugcPid; ServerState::g_WorldInstances.clear(); for (const auto& instance : list.instances) { @@ -217,30 +221,28 @@ namespace { ServerState::g_WorldInstances.push_back(info); } - LOG_DEBUG("Received server list: auth=%s chat=%s worlds=%u", + LOG_DEBUG("Received server list: auth=%s chat=%s ugc=%s worlds=%u", list.authOnline ? "online" : "offline", list.chatOnline ? "online" : "offline", + list.ugcOnline ? "online" : list.ugcEnabled ? "offline" : "off", static_cast(list.instances.size())); } void OnServerInfo(const MasterPackets::ServerInfo& serverInfo, const SystemAddress&) { std::lock_guard lock(ServerState::g_StatusMutex); + // Master sends "offline" as the address when one goes away + const bool up = serverInfo.ip.string != "offline"; switch (serverInfo.serverType) { case ServiceType::AUTH: - if (serverInfo.ip.string == "offline") { - ServerState::g_AuthStatus.online = false; - } else { - ServerState::g_AuthStatus.online = true; - ServerState::g_AuthStatus.lastSeen = std::chrono::steady_clock::now(); - } + ServerState::g_AuthStatus.Set(up); break; case ServiceType::CHAT: - if (serverInfo.ip.string == "offline") { - ServerState::g_ChatStatus.online = false; - } else { - ServerState::g_ChatStatus.online = true; - ServerState::g_ChatStatus.lastSeen = std::chrono::steady_clock::now(); - } + ServerState::g_ChatStatus.Set(up); + break; + case ServiceType::UGC: + ServerState::g_UgcStatus.Set(up); + // Only started when enabled; the pid comes with the next server list + if (up) ServerState::g_UgcEnabled = true; break; default: break; diff --git a/dDashboardServer/notify/Alerts.cpp b/dDashboardServer/notify/Alerts.cpp index ef108eb90..681f8c46c 100644 --- a/dDashboardServer/notify/Alerts.cpp +++ b/dDashboardServer/notify/Alerts.cpp @@ -47,7 +47,13 @@ namespace { bool g_Started = false; uint32_t g_LastBugReportId = 0; - std::optional> g_LastServerStatus; + struct LastServerStatus { + bool auth{}; + bool chat{}; + bool ugcEnabled{}; + bool ugc{}; + }; + std::optional g_LastServerStatus; size_t Discard(char*, size_t size, size_t count, void*) { return size * count; } @@ -287,21 +293,26 @@ namespace Alerts { "/accounts/" + std::to_string(accountId)); } - void ServerStatus(bool authOnline, bool chatOnline) { - const std::pair current{ authOnline, chatOnline }; + void ServerStatus(bool authOnline, bool chatOnline, bool ugcEnabled, bool ugcOnline) { + const LastServerStatus current{ authOnline, chatOnline, ugcEnabled, ugcOnline }; if (!g_LastServerStatus) { g_LastServerStatus = current; return; } const auto previous = *g_LastServerStatus; g_LastServerStatus = current; - if (previous.first != authOnline) { + if (previous.auth != authOnline) { Emit("server", authOnline ? "Auth server is back" : "Auth server went offline", authOnline ? "Players can log in again." : "Players cannot log in until it is back.", {}, "/"); } - if (previous.second != chatOnline) { + if (previous.chat != chatOnline) { Emit("server", chatOnline ? "Chat server is back" : "Chat server went offline", chatOnline ? "Chat, friends and teams work again." : "Chat, friends and teams are unavailable.", {}, "/"); } + // Turning enable_ugc_server on or off is not an outage + if (previous.ugcEnabled && ugcEnabled && previous.ugc != ugcOnline) { + Emit("server", ugcOnline ? "UGC server is back" : "UGC server went offline", + ugcOnline ? "Player models' meshes and icons are made and downloaded again." : "Game clients cannot download player models' meshes and icons until it is back.", {}, "/ugc"); + } } } diff --git a/dDashboardServer/notify/Alerts.h b/dDashboardServer/notify/Alerts.h index 4b3c63e95..2447c1100 100644 --- a/dDashboardServer/notify/Alerts.h +++ b/dDashboardServer/notify/Alerts.h @@ -44,6 +44,6 @@ namespace Alerts { // Bans and mutes done in game with slash commands void InGameAccountAction(const std::string& action, uint32_t accountId); - // Auth/chat availability changes (first call only records the state) - void ServerStatus(bool authOnline, bool chatOnline); + // Auth/chat/UGC availability changes (first call only records the state); the UGC server only while it is enabled + void ServerStatus(bool authOnline, bool chatOnline, bool ugcEnabled = false, bool ugcOnline = false); } diff --git a/dDashboardServer/notify/WebhookFormat.cpp b/dDashboardServer/notify/WebhookFormat.cpp index 89253fe6e..48c5c7073 100644 --- a/dDashboardServer/notify/WebhookFormat.cpp +++ b/dDashboardServer/notify/WebhookFormat.cpp @@ -55,7 +55,7 @@ namespace WebhookFormat { { "moderation", "Bans, locks, mutes, kicks and restrictions, from the dashboard or in game" }, { "security", "GM level changes, two-factor login changes, API tokens, recovery code use" }, { "economy_flag", "The nightly checks flagged unusual income, an item spike or a duplicated item" }, - { "server", "Auth or chat went offline or came back, restarts scheduled or cancelled" }, + { "server", "Auth, chat or the UGC server went offline or came back, restarts scheduled or cancelled" }, { "test", "Test messages sent from the webhooks page" }, }; return events; diff --git a/dDashboardServer/routes/PrometheusMetrics.cpp b/dDashboardServer/routes/PrometheusMetrics.cpp index cd60477b8..b4ae33376 100644 --- a/dDashboardServer/routes/PrometheusMetrics.cpp +++ b/dDashboardServer/routes/PrometheusMetrics.cpp @@ -97,16 +97,20 @@ namespace { // What the dashboard knows without asking the database: master's server list, its own connections, /proc void AddLive(MetricsFormat::Writer& w) { std::vector worlds; - bool authUp = false, chatUp = false; + bool authUp = false, chatUp = false, ugcEnabled = false, ugcUp = false; { std::lock_guard lock(ServerState::g_StatusMutex); worlds = ServerState::g_WorldInstances; authUp = ServerState::g_AuthStatus.online; chatUp = ServerState::g_ChatStatus.online; + ugcEnabled = ServerState::g_UgcEnabled; + ugcUp = ServerState::g_UgcStatus.online; } w.Add("darkflame_master_connected", "Whether the dashboard is connected to the master server", "gauge", Game::server && Game::server->GetIsConnectedToMaster() ? 1 : 0); w.Add("darkflame_auth_up", "Whether the auth server is up, as master last reported", "gauge", authUp ? 1 : 0); w.Add("darkflame_chat_up", "Whether the chat server is up, as master last reported", "gauge", chatUp ? 1 : 0); + w.Add("darkflame_ugc_enabled", "Whether master starts the UGC server (enable_ugc_server)", "gauge", ugcEnabled ? 1 : 0); + w.Add("darkflame_ugc_up", "Whether the UGC server is up, as master last reported", "gauge", ugcUp ? 1 : 0); uint32_t players = 0; std::map>> zones; // id -> name, {instances, players} @@ -215,6 +219,20 @@ namespace { w.Add("darkflame_moderation_queue", "", "gauge", { {"queue", "player_reports"} }, db->CountPlayerReports(q)); }); + // The UGC server's work list: models and modular builds by processing state + ok &= Timed(w, "ugc", [&] { + w.Declare("darkflame_ugc_items", "Player models (model) and cars and rockets (modular) by UGC processing state", "gauge"); + const auto add = [&](const char* kind, const std::vector>& rows) { + std::map states{ {"pending", 0}, {"done", 0}, {"failed", 0} }; + for (const auto& [state, count] : rows) { + states[state == IUgc::eProcessState::PENDING ? "pending" : state == IUgc::eProcessState::DONE ? "done" : "failed"] += count; + } + for (const auto& [state, count] : states) w.Add("darkflame_ugc_items", "", "gauge", { {"kind", kind}, {"state", state} }, static_cast(count)); + }; + add("model", db->GetUgcProcessCounts()); + add("modular", db->GetModularBuildProcessCounts()); + }); + ok &= Timed(w, "strikes", [&] { w.Add("darkflame_active_strikes", "Strikes that still count, all accounts together", "gauge", db->CountActiveStrikes(0, Strikes::CountsSince())); }); diff --git a/dDashboardServer/routes/ServerRoutes.cpp b/dDashboardServer/routes/ServerRoutes.cpp index 5b882e3fd..f466cd4f3 100644 --- a/dDashboardServer/routes/ServerRoutes.cpp +++ b/dDashboardServer/routes/ServerRoutes.cpp @@ -2,6 +2,8 @@ #include #include +#include +#include #include #include #include @@ -11,6 +13,8 @@ #include "RouteUtils.h" #include "ServerState.h" +#include "Traffic.h" +#include "dServer.h" #include "Background.h" #include "PlayerActions.h" #include "BinaryPathFinder.h" @@ -21,6 +25,10 @@ #include "GeneralUtils.h" #include "eHTTPMethod.h" +#ifdef __linux__ +#include +#endif + using namespace RouteUtils; namespace { @@ -29,34 +37,64 @@ namespace { constexpr size_t MAX_MATCHES = 1000; std::chrono::steady_clock::time_point g_NextSample{}; - // All DarkflameServer processes from this build together, in kB (Linux: /proc; elsewhere 0) - uint64_t ServerMemoryKb() { + uint32_t OwnPid() { #ifdef __linux__ - const auto binaryDir = BinaryPathFinder::GetBinaryDir().string(); - uint64_t total = 0; - std::error_code ec; - for (const auto& entry : fs::directory_iterator("/proc", ec)) { - const auto pid = entry.path().filename().string(); - if (pid.empty() || !std::all_of(pid.begin(), pid.end(), ::isdigit)) continue; - std::error_code linkError; - const auto exe = fs::read_symlink(entry.path() / "exe", linkError).string(); - if (linkError || !exe.starts_with(binaryDir) || !exe.ends_with("Server")) continue; - std::ifstream status(entry.path() / "status"); - std::string line; - while (std::getline(status, line)) { - if (!line.starts_with("VmRSS:")) continue; - uint64_t kb = 0; - std::istringstream(line.substr(6)) >> kb; // "VmRSS: 12345 kB" - total += kb; - break; - } - } - return total; + return static_cast(getpid()); #else return 0; #endif } + // CPU seconds of each process when it was last looked at, for its CPU use since then + std::map> g_LastCpu; + + // All DarkflameServer processes from this build together, in kB + uint64_t ServerMemoryKb() { + uint64_t total = 0; + for (const auto& process : ServerRoutes::Processes()) total += process.memoryKb; + return total; + } + + nlohmann::json ProcessJson(const ServerRoutes::Process& p) { + return { {"pid", p.pid}, {"memory_kb", p.memoryKb}, {"cpu_percent", std::round(p.cpuPercent * 10) / 10}, {"started_at", p.startedAt} }; + } + + // The UGC server's queue in the database (its work list), from the model and modular build tables + nlohmann::json UgcCounts() { + const auto counts = [](const std::vector>& rows) { + nlohmann::json out = { {"pending", 0}, {"done", 0}, {"failed", 0} }; + for (const auto& [state, count] : rows) { + out[state == IUgc::eProcessState::PENDING ? "pending" : state == IUgc::eProcessState::DONE ? "done" : "failed"] = count; + } + return out; + }; + return { {"model", counts(Database::Get()->GetUgcProcessCounts())}, {"modular", counts(Database::Get()->GetModularBuildProcessCounts())} }; + } + + // What /api/servers and the home page show of the UGC server + nlohmann::json UgcSummary(bool withCounts) { + const auto state = ServerState::GetServerStateJson()["ugc"]; + nlohmann::json out = state; + uint32_t pid = 0; + { + std::lock_guard lock(ServerState::g_StatusMutex); + pid = ServerState::g_UgcPid; + } + out["pid"] = pid; + // Its traffic report (every few seconds, via master) carries its workers, totals and storage + const auto traffic = Traffic::Server("ugc"); + out["gauges"] = traffic.value("gauges", nlohmann::json::object()); + out["last_report"] = traffic.value("last_seen", int64_t{}); + if (withCounts) { + try { + out["counts"] = UgcCounts(); + } catch (const std::exception& ex) { + LOG_DEBUG("Could not count the UGC queue: %s", ex.what()); + } + } + return out; + } + IServerHealth::HealthSample CurrentSample() { const auto state = ServerState::GetServerStateJson(); IServerHealth::HealthSample sample; @@ -66,6 +104,8 @@ namespace { sample.authOnline = state["auth"].value("online", false); sample.chatOnline = state["chat"].value("online", false); sample.memoryKb = ServerMemoryKb(); + sample.ugcEnabled = state["ugc"].value("enabled", false); + sample.ugcOnline = state["ugc"].value("online", false); return sample; } @@ -86,6 +126,76 @@ namespace { } namespace ServerRoutes { + std::vector Processes() { + std::vector processes; +#ifdef __linux__ + const auto binaryDir = BinaryPathFinder::GetBinaryDir().string(); + const auto ticks = static_cast(sysconf(_SC_CLK_TCK)); + const auto now = std::chrono::steady_clock::now(); + // Process start times are in ticks since boot + int64_t bootTime = 0; + { + std::ifstream stat("/proc/stat"); + std::string line; + while (std::getline(stat, line)) { + if (line.starts_with("btime ")) { bootTime = GeneralUtils::TryParse(line.substr(6)).value_or(0); break; } + } + } + std::map> seen; + std::error_code ec; + for (const auto& entry : fs::directory_iterator("/proc", ec)) { + const auto pidText = entry.path().filename().string(); + if (pidText.empty() || !std::all_of(pidText.begin(), pidText.end(), ::isdigit)) continue; + std::error_code linkError; + const auto exe = fs::read_symlink(entry.path() / "exe", linkError); + const auto exeText = exe.string(); + if (linkError || !exeText.starts_with(binaryDir) || !exeText.ends_with("Server")) continue; + Process process; + process.program = exe.filename().string(); + process.pid = GeneralUtils::TryParse(pidText).value_or(0); + + std::ifstream status(entry.path() / "status"); + std::string line; + while (std::getline(status, line)) { + if (!line.starts_with("VmRSS:")) continue; + std::istringstream(line.substr(6)) >> process.memoryKb; // "VmRSS: 12345 kB" + break; + } + + // stat: "pid (name) state ..." then utime, stime (fields 14, 15) and starttime (22), counted after the name + std::ifstream statFile(entry.path() / "stat"); + std::string stat((std::istreambuf_iterator(statFile)), std::istreambuf_iterator()); + if (const auto close = stat.rfind(')'); close != std::string::npos) { + std::istringstream fields(stat.substr(close + 2)); + std::vector values{ std::istream_iterator(fields), std::istream_iterator() }; + if (values.size() > 19 && ticks > 0) { + const auto cpu = (GeneralUtils::TryParse(values[11]).value_or(0) + GeneralUtils::TryParse(values[12]).value_or(0)) / ticks; + process.startedAt = bootTime + static_cast(GeneralUtils::TryParse(values[19]).value_or(0) / ticks); + const auto last = g_LastCpu.find(process.pid); + if (last != g_LastCpu.end()) { + const auto wall = std::chrono::duration(now - last->second.second).count(); + if (wall > 0.5) process.cpuPercent = std::max(0.0, (cpu - last->second.first) / wall * 100); + } + // Keep the older reading when looked at again too soon for a fair number + seen[process.pid] = last != g_LastCpu.end() && std::chrono::duration(now - last->second.second).count() <= 0.5 ? last->second : std::pair{ cpu, now }; + } + } + + // World servers: WorldServer -zone -port -instance ... + std::ifstream cmdline(entry.path() / "cmdline"); + std::vector args; + for (std::string arg; std::getline(cmdline, arg, '\0');) args.push_back(arg); + for (size_t i = 0; i + 1 < args.size(); i++) { + if (args[i] == "-zone") process.zoneId = GeneralUtils::TryParse(args[i + 1]).value_or(0); + if (args[i] == "-instance") process.instanceId = GeneralUtils::TryParse(args[i + 1]).value_or(0); + } + processes.push_back(std::move(process)); + } + g_LastCpu = std::move(seen); +#endif + return processes; + } + void Update() { const auto now = std::chrono::steady_clock::now(); // The first sample waits a minute, so auth and chat have time to connect after a start @@ -109,13 +219,71 @@ void RegisterServerRoutes() { const auto now = static_cast(std::time(nullptr)); nlohmann::json samples = nlohmann::json::array(); for (const auto& s : Database::Get()->GetHealthSamples(now - span, now, bucket)) { - samples.push_back({ {"time", s.time}, {"players", s.players}, {"worlds", s.worlds}, {"auth", s.authOnline}, {"chat", s.chatOnline}, {"memory_kb", s.memoryKb} }); + samples.push_back({ {"time", s.time}, {"players", s.players}, {"worlds", s.worlds}, {"auth", s.authOnline}, {"chat", s.chatOnline}, {"memory_kb", s.memoryKb}, + {"ugc_enabled", s.ugcEnabled}, {"ugc", s.ugcOnline} }); } const auto current = CurrentSample(); JsonSuccess(reply, { {"from", now - span}, {"to", now}, {"bucket", bucket}, {"samples", samples}, - {"current", { {"players", current.players}, {"worlds", current.worlds}, {"auth", current.authOnline}, {"chat", current.chatOnline}, {"memory_kb", current.memoryKb} }} }); + {"current", { {"players", current.players}, {"worlds", current.worlds}, {"auth", current.authOnline}, {"chat", current.chatOnline}, {"memory_kb", current.memoryKb}, + {"ugc_enabled", current.ugcEnabled}, {"ugc", current.ugcOnline} }} }); }); + Route(eHTTPMethod::GET, "/api/servers", Perm("health_view"), + "Every server: {servers: [{key, label, kind, online, since, players, process: {pid, memory_kb, cpu_percent, started_at}, " + "traffic: {online, last_seen, connections, ping_ms, gauges}}], ugc: {enabled, online, since, pid, gauges, last_report, counts}}. " + "cpu_percent is of one core since the last time the processes were read", + [](HTTPReply& reply, const HTTPContext&) { + const auto state = ServerState::GetServerStateJson(); + auto processes = ServerRoutes::Processes(); + const auto ugc = UgcSummary(true); + // The process of a server: by pid when known, else the only one of its program (worlds: by zone and instance) + const auto take = [&](const std::string& program, uint32_t pid, uint32_t zone, uint32_t instance) -> nlohmann::json { + for (auto it = processes.begin(); it != processes.end(); ++it) { + const bool match = pid ? it->pid == pid : it->program == program && (program != "WorldServer" || (it->zoneId == zone && it->instanceId == instance)); + if (!match) continue; + auto json = ProcessJson(*it); + processes.erase(it); + return json; + } + return nullptr; + }; + const auto row = [&](const std::string& key, const std::string& kind, bool online, int64_t since, nlohmann::json process) { + const auto traffic = Traffic::Server(key); + const auto link = traffic.value("link", nlohmann::json::object()); + return nlohmann::json{ {"key", key}, {"label", traffic.value("label", key)}, {"kind", kind}, {"online", online}, {"since", since}, {"process", process}, + {"traffic", { {"online", traffic.value("online", false)}, {"last_seen", traffic.value("last_seen", int64_t{})}, + {"connections", link.value("connections", 0u)}, {"ping_ms", link.value("ping_ms", 0u)}, {"gauges", traffic.value("gauges", nlohmann::json::object())} }} }; + }; + nlohmann::json servers = nlohmann::json::array(); + const bool masterUp = Game::server && Game::server->GetIsConnectedToMaster(); + servers.push_back(row("master", "MASTER", masterUp, 0, take("MasterServer", 0, 0, 0))); + servers.push_back(row("auth", "AUTH", state["auth"].value("online", false), state["auth"].value("since", int64_t{}), take("AuthServer", 0, 0, 0))); + servers.push_back(row("chat", "CHAT", state["chat"].value("online", false), state["chat"].value("since", int64_t{}), take("ChatServer", 0, 0, 0))); + servers.push_back(row("dashboard", "DASHBOARD", true, 0, take("DashboardServer", OwnPid(), 0, 0))); + if (ugc.value("enabled", false) || ugc.value("online", false)) { + auto ugcRow = row("ugc", "UGC", ugc.value("online", false), ugc.value("since", int64_t{}), take("UgcServer", ugc.value("pid", 0u), 0, 0)); + if (ugcRow["process"].is_null()) ugcRow["process"] = take("UgcServer", 0, 0, 0); + servers.push_back(ugcRow); + } + for (const auto& world : state.value("worlds", nlohmann::json::array())) { + const auto zone = world.value("mapID", 0u), instance = world.value("instanceID", 0u); + auto worldRow = row("world:" + std::to_string(zone) + ":" + std::to_string(instance), "WORLD", true, 0, take("WorldServer", 0, zone, instance)); + worldRow["players"] = world.value("players", 0u); + servers.push_back(worldRow); + } + // Anything left over, e.g. a world master no longer lists + for (const auto& process : processes) { + servers.push_back({ {"key", process.program + ":" + std::to_string(process.pid)}, {"label", process.program}, {"kind", "OTHER"}, {"online", false}, + {"since", 0}, {"process", ProcessJson(process)}, {"traffic", nullptr} }); + } + JsonSuccess(reply, { {"servers", servers}, {"ugc", ugc} }); + }); + + Route(eHTTPMethod::GET, "/api/servers/ugc", Perm("health_view"), + "The UGC server for the home page: {enabled, online, since, pid, gauges: {workers_busy, workers_queued, workers_threads, ugc_made_total, " + "ugc_failed_total, ugc_evicted_total, ugc_stored_bytes, ugc_max_storage_bytes}, last_report, counts: {model, modular: {pending, done, failed}}}", + [](HTTPReply& reply, const HTTPContext&) { JsonSuccess(reply, UgcSummary(true)); }); + Route(eHTTPMethod::GET, "/api/crash_dumps", Perm("logs_system"), "Crash dumps in dump_folder, newest first", [](HTTPReply& reply, const HTTPContext&) { const auto folder = DumpFolder(); diff --git a/dDashboardServer/routes/ServerRoutes.h b/dDashboardServer/routes/ServerRoutes.h index 5f6116c83..e897e876b 100644 --- a/dDashboardServer/routes/ServerRoutes.h +++ b/dDashboardServer/routes/ServerRoutes.h @@ -1,12 +1,31 @@ #pragma once +#include +#include +#include + /** - * Running the server: health history (a sample a minute: players, worlds, auth/chat up, memory), crash dumps from - * dump_folder, and searching the servers' log files. Prometheus metrics are in PrometheusMetrics.h. + * Running the server: health history (a sample a minute: players, worlds, auth/chat/UGC up, memory), the running + * server processes, crash dumps from dump_folder, and searching the servers' log files. Prometheus metrics are in + * PrometheusMetrics.h. */ void RegisterServerRoutes(); namespace ServerRoutes { // Main loop: take a health sample once a minute void Update(); + + // A running DarkflameServer process from this build + struct Process { + std::string program; // the binary: "WorldServer", "UgcServer", ... + uint32_t pid{}; + uint64_t memoryKb{}; // resident + double cpuPercent{}; // of one core, since this process was last looked at (0 the first time) + int64_t startedAt{}; // unix time + uint32_t zoneId{}; // world servers: from the command line + uint32_t instanceId{}; + }; + + // Main thread: every server process of this build (Linux: /proc; elsewhere none) + std::vector Processes(); } diff --git a/dDashboardServer/routes/ServerState.h b/dDashboardServer/routes/ServerState.h index fd9ab91f5..a21048fde 100644 --- a/dDashboardServer/routes/ServerState.h +++ b/dDashboardServer/routes/ServerState.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -13,6 +14,15 @@ struct ServerStatus { uint32_t players{0}; std::string version{}; std::chrono::steady_clock::time_point lastSeen{}; + int64_t since{}; // unix time the dashboard first saw it up this time (0: down) + + // Online or not, keeping `since` for as long as it stays up + void Set(bool up) { + if (up && !online) since = static_cast(std::time(nullptr)); + if (!up) since = 0; + online = up; + if (up) lastSeen = std::chrono::steady_clock::now(); + } }; struct WorldInstanceInfo { @@ -34,6 +44,10 @@ struct WorldInstanceInfo { namespace ServerState { extern ServerStatus g_AuthStatus; extern ServerStatus g_ChatStatus; + // The UGC server (docs/UgcServer.md): enabled is master's enable_ugc_server, pid the process master last started + extern ServerStatus g_UgcStatus; + extern bool g_UgcEnabled; + extern uint32_t g_UgcPid; extern std::vector g_WorldInstances; extern std::mutex g_StatusMutex; @@ -45,6 +59,9 @@ namespace ServerState { data["auth"]["players"] = g_AuthStatus.players; data["chat"]["online"] = g_ChatStatus.online; data["chat"]["players"] = g_ChatStatus.players; + data["auth"]["since"] = g_AuthStatus.since; + data["chat"]["since"] = g_ChatStatus.since; + data["ugc"] = { {"enabled", g_UgcEnabled}, {"online", g_UgcStatus.online}, {"since", g_UgcStatus.since} }; uint32_t totalOnlinePlayers = 0; data["worlds"] = nlohmann::json::array(); diff --git a/dDashboardServer/routes/Traffic.cpp b/dDashboardServer/routes/Traffic.cpp index 454e056de..3d53f39c7 100644 --- a/dDashboardServer/routes/Traffic.cpp +++ b/dDashboardServer/routes/Traffic.cpp @@ -308,6 +308,10 @@ namespace { } namespace Traffic { + nlohmann::json Server(const std::string& key) { + return ServerInfo(key, TrafficStats::Now()); + } + void Ingest(const ServerTraffic& report) { g_History.Ingest(static_cast(report.serverType), report.zoneId, report.instanceId, report.report, TrafficStats::Now()); g_Changed = true; diff --git a/dDashboardServer/routes/Traffic.h b/dDashboardServer/routes/Traffic.h index ae970d886..51aed38a7 100644 --- a/dDashboardServer/routes/Traffic.h +++ b/dDashboardServer/routes/Traffic.h @@ -1,5 +1,8 @@ #pragma once +#include +#include "json.hpp" + struct ServerTraffic; namespace MetricsFormat { class Writer; } @@ -16,6 +19,9 @@ namespace Traffic { // Main loop: write finished minutes, tell open pages there is news void Update(); + // Main thread: one server's last report ("ugc", "chat", ...): {key, label, online, last_seen, link, gauges} + nlohmann::json Server(const std::string& key); + // Counters for /metrics void AddMetrics(MetricsFormat::Writer& w); } diff --git a/dDashboardServer/routes/WSRoutes.cpp b/dDashboardServer/routes/WSRoutes.cpp index 1498de6e6..d91c77d2e 100644 --- a/dDashboardServer/routes/WSRoutes.cpp +++ b/dDashboardServer/routes/WSRoutes.cpp @@ -60,7 +60,8 @@ void RegisterWSRoutes() { void BroadcastDashboardUpdate() { // Sent to every signed-in account, so it only has what players may see; staff load /api/worlds for details nlohmann::json status = ServerState::PlayerSafe(ServerState::GetServerStateJson()); - Alerts::ServerStatus(status["auth"].value("online", false), status["chat"].value("online", false)); + Alerts::ServerStatus(status["auth"].value("online", false), status["chat"].value("online", false), + status["ugc"].value("enabled", false), status["ugc"].value("online", false)); status["restart"] = LiveWorld::RestartStatus(); try { diff --git a/dDatabase/GameDatabase/ITables/IServerHealth.h b/dDatabase/GameDatabase/ITables/IServerHealth.h index 4018b103e..723541d9a 100644 --- a/dDatabase/GameDatabase/ITables/IServerHealth.h +++ b/dDatabase/GameDatabase/ITables/IServerHealth.h @@ -16,6 +16,8 @@ public: bool authOnline{}; bool chatOnline{}; uint64_t memoryKb{}; // all server processes together (Linux only, else 0) + bool ugcEnabled{}; // master starts the UGC server (in a bucket: at any sample) + bool ugcOnline{}; // in a bucket: at every sample }; virtual void InsertHealthSample(const HealthSample& sample) = 0; diff --git a/dDatabase/GameDatabase/MySQL/Tables/ServerHealth.cpp b/dDatabase/GameDatabase/MySQL/Tables/ServerHealth.cpp index d1a7e21ae..32ccdd672 100644 --- a/dDatabase/GameDatabase/MySQL/Tables/ServerHealth.cpp +++ b/dDatabase/GameDatabase/MySQL/Tables/ServerHealth.cpp @@ -1,17 +1,18 @@ #include "MySQLDatabase.h" void MySQLDatabase::InsertHealthSample(const HealthSample& sample) { - ExecuteInsert("REPLACE INTO server_health (time, players, worlds, auth_online, chat_online, memory_kb) VALUES (?, ?, ?, ?, ?, ?);", - sample.time, sample.players, sample.worlds, sample.authOnline, sample.chatOnline, static_cast(sample.memoryKb)); + ExecuteInsert("REPLACE INTO server_health (time, players, worlds, auth_online, chat_online, memory_kb, ugc_enabled, ugc_online) VALUES (?, ?, ?, ?, ?, ?, ?, ?);", + sample.time, sample.players, sample.worlds, sample.authOnline, sample.chatOnline, static_cast(sample.memoryKb), sample.ugcEnabled, sample.ugcOnline); } std::vector MySQLDatabase::GetHealthSamples(int64_t from, int64_t to, int64_t bucketSeconds) { std::vector samples; - auto result = ExecuteSelect("SELECT FLOOR(time / ?) * ? AS bucket, MAX(players) AS players, MAX(worlds) AS worlds, MIN(auth_online) AS auth, MIN(chat_online) AS chat, AVG(memory_kb) AS memory " + auto result = ExecuteSelect("SELECT FLOOR(time / ?) * ? AS bucket, MAX(players) AS players, MAX(worlds) AS worlds, MIN(auth_online) AS auth, MIN(chat_online) AS chat, AVG(memory_kb) AS memory, MAX(ugc_enabled) AS ugc_enabled, MIN(ugc_online) AS ugc " "FROM server_health WHERE time >= ? AND time <= ? GROUP BY bucket ORDER BY bucket;", bucketSeconds, bucketSeconds, from, to); while (result->next()) { samples.push_back({ result->getInt64("bucket"), result->getUInt("players"), result->getUInt("worlds"), - result->getInt("auth") != 0, result->getInt("chat") != 0, static_cast(result->getDouble("memory")) }); + result->getInt("auth") != 0, result->getInt("chat") != 0, static_cast(result->getDouble("memory")), + result->getInt("ugc_enabled") != 0, result->getInt("ugc") != 0 }); } return samples; } diff --git a/dDatabase/GameDatabase/SQLite/Tables/ServerHealth.cpp b/dDatabase/GameDatabase/SQLite/Tables/ServerHealth.cpp index 1174ff666..babb3c21b 100644 --- a/dDatabase/GameDatabase/SQLite/Tables/ServerHealth.cpp +++ b/dDatabase/GameDatabase/SQLite/Tables/ServerHealth.cpp @@ -1,17 +1,18 @@ #include "SQLiteDatabase.h" void SQLiteDatabase::InsertHealthSample(const HealthSample& sample) { - ExecuteInsert("INSERT OR REPLACE INTO server_health (time, players, worlds, auth_online, chat_online, memory_kb) VALUES (?, ?, ?, ?, ?, ?);", - sample.time, sample.players, sample.worlds, sample.authOnline, sample.chatOnline, static_cast(sample.memoryKb)); + ExecuteInsert("INSERT OR REPLACE INTO server_health (time, players, worlds, auth_online, chat_online, memory_kb, ugc_enabled, ugc_online) VALUES (?, ?, ?, ?, ?, ?, ?, ?);", + sample.time, sample.players, sample.worlds, sample.authOnline, sample.chatOnline, static_cast(sample.memoryKb), sample.ugcEnabled, sample.ugcOnline); } std::vector SQLiteDatabase::GetHealthSamples(int64_t from, int64_t to, int64_t bucketSeconds) { std::vector samples; - auto [_, result] = ExecuteSelect("SELECT (time / ?) * ? AS bucket, MAX(players) AS players, MAX(worlds) AS worlds, MIN(auth_online) AS auth, MIN(chat_online) AS chat, AVG(memory_kb) AS memory " + auto [_, result] = ExecuteSelect("SELECT (time / ?) * ? AS bucket, MAX(players) AS players, MAX(worlds) AS worlds, MIN(auth_online) AS auth, MIN(chat_online) AS chat, AVG(memory_kb) AS memory, MAX(ugc_enabled) AS ugc_enabled, MIN(ugc_online) AS ugc " "FROM server_health WHERE time >= ? AND time <= ? GROUP BY bucket ORDER BY bucket;", bucketSeconds, bucketSeconds, from, to); while (!result.eof()) { samples.push_back({ result.getInt64Field("bucket"), static_cast(result.getIntField("players")), static_cast(result.getIntField("worlds")), - result.getIntField("auth") != 0, result.getIntField("chat") != 0, static_cast(result.getFloatField("memory")) }); + result.getIntField("auth") != 0, result.getIntField("chat") != 0, static_cast(result.getFloatField("memory")), + result.getIntField("ugc_enabled") != 0, result.getIntField("ugc") != 0 }); result.nextRow(); } return samples; diff --git a/migrations/dlu/mysql/85_server_health_ugc.sql b/migrations/dlu/mysql/85_server_health_ugc.sql new file mode 100644 index 000000000..e6aaed281 --- /dev/null +++ b/migrations/dlu/mysql/85_server_health_ugc.sql @@ -0,0 +1,3 @@ +/* The UGC server in the health history: whether master was set to start it (enable_ugc_server) and whether it was up. */ +ALTER TABLE server_health ADD COLUMN IF NOT EXISTS ugc_enabled TINYINT NOT NULL DEFAULT 0; +ALTER TABLE server_health ADD COLUMN IF NOT EXISTS ugc_online TINYINT NOT NULL DEFAULT 0; diff --git a/migrations/dlu/sqlite/68_server_health_ugc.sql b/migrations/dlu/sqlite/68_server_health_ugc.sql new file mode 100644 index 000000000..3606d68e1 --- /dev/null +++ b/migrations/dlu/sqlite/68_server_health_ugc.sql @@ -0,0 +1,3 @@ +/* The UGC server in the health history. See the MySQL migration. */ +ALTER TABLE server_health ADD COLUMN ugc_enabled INTEGER NOT NULL DEFAULT 0; +ALTER TABLE server_health ADD COLUMN ugc_online INTEGER NOT NULL DEFAULT 0; diff --git a/tests/dDatabaseTests/DatabaseParityTests.cpp b/tests/dDatabaseTests/DatabaseParityTests.cpp index cee5d4d8d..f841af1c5 100644 --- a/tests/dDatabaseTests/DatabaseParityTests.cpp +++ b/tests/dDatabaseTests/DatabaseParityTests.cpp @@ -78,7 +78,7 @@ NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(IScheduledTasks::TaskSettings, name, schedule NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(IScheduledTasks::TaskRun, id, task, trigger, actor, startedAt, finishedAt, status, summary, log); NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(IServerConfig::Setting, file, name, fileValue, fileSource, webValue, webWins, secret, description, seenAt, updatedAt, updatedBy); NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(IServerConfig::SettingChange, id, file, name, oldValue, oldWebWins, newValue, newWebWins, fileValue, secret, removed, revertOf, changedAt, accountId, changedBy); -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(IServerHealth::HealthSample, time, players, worlds, authOnline, chatOnline, memoryKb); +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(IServerHealth::HealthSample, time, players, worlds, authOnline, chatOnline, memoryKb, ugcEnabled, ugcOnline); NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(IServerHealth::InstanceSample, time, zoneId, instanceId, cloneId, players); NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(IPlayerPositions::PositionSample, time, characterId, zoneId, instanceId, cloneId, x, y, z); NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(IPlayerPositions::PositionInstance, zoneId, instanceId, cloneId, first, last, players); @@ -1230,8 +1230,8 @@ TEST_F(ParitySeeded, ChatLog) { TEST_F(ParitySeeded, ServerHealth) { Both("InsertHealthSample", [](GameDatabase& db) { db.InsertHealthSample({ 1700000000, 5, 2, true, true, 100000 }); - db.InsertHealthSample({ 1700000060, 7, 3, true, false, 110000 }); - db.InsertHealthSample({ 1700000120, 3, 3, false, true, 90000 }); + db.InsertHealthSample({ 1700000060, 7, 3, true, false, 110000, true, true }); + db.InsertHealthSample({ 1700000120, 3, 3, false, true, 90000, true, false }); db.InsertHealthSample({ 1700000500, 1, 1, true, true, 50000 }); }); Both("GetHealthSamples", [](GameDatabase& db) { return json{ db.GetHealthSamples(1700000000, 1700001000, 60), db.GetHealthSamples(1700000000, 1700001000, 300), db.GetHealthSamples(1700000050, 1700000130, 60) }; }); diff --git a/tests/dWebTests/ServerStateTests.cpp b/tests/dWebTests/ServerStateTests.cpp index 1e0104752..ded2f8a39 100644 --- a/tests/dWebTests/ServerStateTests.cpp +++ b/tests/dWebTests/ServerStateTests.cpp @@ -6,6 +6,9 @@ namespace ServerState { ServerStatus g_AuthStatus{}; ServerStatus g_ChatStatus{}; + ServerStatus g_UgcStatus{}; + bool g_UgcEnabled{}; + uint32_t g_UgcPid{}; std::vector g_WorldInstances{}; std::mutex g_StatusMutex{}; } @@ -16,6 +19,8 @@ protected: std::lock_guard lock(ServerState::g_StatusMutex); ServerState::g_AuthStatus = {}; ServerState::g_ChatStatus = {}; + ServerState::g_UgcStatus = {}; + ServerState::g_UgcEnabled = false; ServerState::g_WorldInstances.clear(); } }; @@ -25,6 +30,8 @@ TEST_F(ServerStateTest, DefaultStateAllOffline) { EXPECT_FALSE(json["auth"]["online"].get()); EXPECT_FALSE(json["chat"]["online"].get()); + EXPECT_FALSE(json["ugc"]["online"].get()); + EXPECT_FALSE(json["ugc"]["enabled"].get()); EXPECT_TRUE(json["worlds"].empty()); EXPECT_EQ(json["stats"]["onlinePlayers"].get(), 0); } @@ -165,3 +172,25 @@ TEST_F(ServerStateTest, ServerStatusDefaults) { EXPECT_EQ(status.players, 0); EXPECT_EQ(status.version, ""); } + +TEST_F(ServerStateTest, UgcServerComesAndGoes) { + { + std::lock_guard lock(ServerState::g_StatusMutex); + ServerState::g_UgcEnabled = true; + ServerState::g_UgcStatus.Set(true); + } + auto json = ServerState::GetServerStateJson(); + EXPECT_TRUE(json["ugc"]["enabled"].get()); + EXPECT_TRUE(json["ugc"]["online"].get()); + const auto since = json["ugc"]["since"].get(); + EXPECT_GT(since, 0); + + // Still up: it keeps the time it came up + ServerState::g_UgcStatus.Set(true); + EXPECT_EQ(ServerState::GetServerStateJson()["ugc"]["since"].get(), since); + + ServerState::g_UgcStatus.Set(false); + json = ServerState::GetServerStateJson(); + EXPECT_FALSE(json["ugc"]["online"].get()); + EXPECT_EQ(json["ugc"]["since"].get(), 0); +}