feat(dashboard): track the UGC server like auth and chat

Online state with the time it came up, down/up alerts, a UGC column in the
health history, /api/servers (every server with its process memory and CPU)
and /api/servers/ugc, and UGC gauges for Prometheus.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dDashboardServer/DashboardServer.cpp

@@ -115,6 +115,9 @@ namespace Game {
 namespace ServerState {
 	ServerStatus g_AuthStatus{};
 	ServerStatus g_ChatStatus{};
+	ServerStatus g_UgcStatus{};
+	bool g_UgcEnabled{};
+	uint32_t g_UgcPid{};
 	std::vector<WorldInstanceInfo> g_WorldInstances{};
 	std::mutex g_StatusMutex{};
 }
@@ -197,10 +200,11 @@ namespace {
 
 	void OnServerList(const MasterPackets::ServerListResponse& list, const SystemAddress&) {
 		std::lock_guard lock(ServerState::g_StatusMutex);
-		ServerState::g_AuthStatus.online = list.authOnline != 0;
-		ServerState::g_AuthStatus.lastSeen = std::chrono::steady_clock::now();
-		ServerState::g_ChatStatus.online = list.chatOnline != 0;
-		ServerState::g_ChatStatus.lastSeen = std::chrono::steady_clock::now();
+		ServerState::g_AuthStatus.Set(list.authOnline != 0);
+		ServerState::g_ChatStatus.Set(list.chatOnline != 0);
+		ServerState::g_UgcStatus.Set(list.ugcOnline != 0);
+		ServerState::g_UgcEnabled = list.ugcEnabled != 0;
+		ServerState::g_UgcPid = list.ugcPid;
 
 		ServerState::g_WorldInstances.clear();
 		for (const auto& instance : list.instances) {
@@ -217,30 +221,28 @@ namespace {
 			ServerState::g_WorldInstances.push_back(info);
 		}
 
-		LOG_DEBUG("Received server list: auth=%s chat=%s worlds=%u",
+		LOG_DEBUG("Received server list: auth=%s chat=%s ugc=%s worlds=%u",
 			list.authOnline ? "online" : "offline",
 			list.chatOnline ? "online" : "offline",
+			list.ugcOnline ? "online" : list.ugcEnabled ? "offline" : "off",
 			static_cast<uint32_t>(list.instances.size()));
 	}
 
 	void OnServerInfo(const MasterPackets::ServerInfo& serverInfo, const SystemAddress&) {
 		std::lock_guard lock(ServerState::g_StatusMutex);
+		// Master sends "offline" as the address when one goes away
+		const bool up = serverInfo.ip.string != "offline";
 		switch (serverInfo.serverType) {
 		case ServiceType::AUTH:
-			if (serverInfo.ip.string == "offline") {
-				ServerState::g_AuthStatus.online = false;
-			} else {
-				ServerState::g_AuthStatus.online = true;
-				ServerState::g_AuthStatus.lastSeen = std::chrono::steady_clock::now();
-			}
+			ServerState::g_AuthStatus.Set(up);
 			break;
 		case ServiceType::CHAT:
-			if (serverInfo.ip.string == "offline") {
-				ServerState::g_ChatStatus.online = false;
-			} else {
-				ServerState::g_ChatStatus.online = true;
-				ServerState::g_ChatStatus.lastSeen = std::chrono::steady_clock::now();
-			}
+			ServerState::g_ChatStatus.Set(up);
+			break;
+		case ServiceType::UGC:
+			ServerState::g_UgcStatus.Set(up);
+			// Only started when enabled; the pid comes with the next server list
+			if (up) ServerState::g_UgcEnabled = true;
 			break;
 		default:
 			break;

dDashboardServer/notify/Alerts.cpp

@@ -47,7 +47,13 @@ namespace {
 	bool g_Started = false;
 
 	uint32_t g_LastBugReportId = 0;
-	std::optional<std::pair<bool, bool>> g_LastServerStatus;
+	struct LastServerStatus {
+		bool auth{};
+		bool chat{};
+		bool ugcEnabled{};
+		bool ugc{};
+	};
+	std::optional<LastServerStatus> g_LastServerStatus;
 
 	size_t Discard(char*, size_t size, size_t count, void*) { return size * count; }
 
@@ -287,21 +293,26 @@ namespace Alerts {
 			"/accounts/" + std::to_string(accountId));
 	}
 
-	void ServerStatus(bool authOnline, bool chatOnline) {
-		const std::pair<bool, bool> current{ authOnline, chatOnline };
+	void ServerStatus(bool authOnline, bool chatOnline, bool ugcEnabled, bool ugcOnline) {
+		const LastServerStatus current{ authOnline, chatOnline, ugcEnabled, ugcOnline };
 		if (!g_LastServerStatus) {
 			g_LastServerStatus = current;
 			return;
 		}
 		const auto previous = *g_LastServerStatus;
 		g_LastServerStatus = current;
-		if (previous.first != authOnline) {
+		if (previous.auth != authOnline) {
 			Emit("server", authOnline ? "Auth server is back" : "Auth server went offline",
 				authOnline ? "Players can log in again." : "Players cannot log in until it is back.", {}, "/");
 		}
-		if (previous.second != chatOnline) {
+		if (previous.chat != chatOnline) {
 			Emit("server", chatOnline ? "Chat server is back" : "Chat server went offline",
 				chatOnline ? "Chat, friends and teams work again." : "Chat, friends and teams are unavailable.", {}, "/");
 		}
+		// Turning enable_ugc_server on or off is not an outage
+		if (previous.ugcEnabled && ugcEnabled && previous.ugc != ugcOnline) {
+			Emit("server", ugcOnline ? "UGC server is back" : "UGC server went offline",
+				ugcOnline ? "Player models' meshes and icons are made and downloaded again." : "Game clients cannot download player models' meshes and icons until it is back.", {}, "/ugc");
+		}
 	}
 }

dDashboardServer/notify/Alerts.h

@@ -44,6 +44,6 @@ namespace Alerts {
 	// Bans and mutes done in game with slash commands
 	void InGameAccountAction(const std::string& action, uint32_t accountId);
 
-	// Auth/chat availability changes (first call only records the state)
-	void ServerStatus(bool authOnline, bool chatOnline);
+	// Auth/chat/UGC availability changes (first call only records the state); the UGC server only while it is enabled
+	void ServerStatus(bool authOnline, bool chatOnline, bool ugcEnabled = false, bool ugcOnline = false);
 }

dDashboardServer/notify/WebhookFormat.cpp

@@ -55,7 +55,7 @@ namespace WebhookFormat {
 			{ "moderation", "Bans, locks, mutes, kicks and restrictions, from the dashboard or in game" },
 			{ "security", "GM level changes, two-factor login changes, API tokens, recovery code use" },
 			{ "economy_flag", "The nightly checks flagged unusual income, an item spike or a duplicated item" },
-			{ "server", "Auth or chat went offline or came back, restarts scheduled or cancelled" },
+			{ "server", "Auth, chat or the UGC server went offline or came back, restarts scheduled or cancelled" },
 			{ "test", "Test messages sent from the webhooks page" },
 		};
 		return events;

dDashboardServer/routes/PrometheusMetrics.cpp

@@ -97,16 +97,20 @@ namespace {
 	// What the dashboard knows without asking the database: master's server list, its own connections, /proc
 	void AddLive(MetricsFormat::Writer& w) {
 		std::vector<WorldInstanceInfo> worlds;
-		bool authUp = false, chatUp = false;
+		bool authUp = false, chatUp = false, ugcEnabled = false, ugcUp = false;
 		{
 			std::lock_guard lock(ServerState::g_StatusMutex);
 			worlds = ServerState::g_WorldInstances;
 			authUp = ServerState::g_AuthStatus.online;
 			chatUp = ServerState::g_ChatStatus.online;
+			ugcEnabled = ServerState::g_UgcEnabled;
+			ugcUp = ServerState::g_UgcStatus.online;
 		}
 		w.Add("darkflame_master_connected", "Whether the dashboard is connected to the master server", "gauge", Game::server && Game::server->GetIsConnectedToMaster() ? 1 : 0);
 		w.Add("darkflame_auth_up", "Whether the auth server is up, as master last reported", "gauge", authUp ? 1 : 0);
 		w.Add("darkflame_chat_up", "Whether the chat server is up, as master last reported", "gauge", chatUp ? 1 : 0);
+		w.Add("darkflame_ugc_enabled", "Whether master starts the UGC server (enable_ugc_server)", "gauge", ugcEnabled ? 1 : 0);
+		w.Add("darkflame_ugc_up", "Whether the UGC server is up, as master last reported", "gauge", ugcUp ? 1 : 0);
 
 		uint32_t players = 0;
 		std::map<uint32_t, std::pair<std::string, std::pair<uint32_t, uint32_t>>> zones; // id -> name, {instances, players}
@@ -215,6 +219,20 @@ namespace {
 			w.Add("darkflame_moderation_queue", "", "gauge", { {"queue", "player_reports"} }, db->CountPlayerReports(q));
 		});
 
+		// The UGC server's work list: models and modular builds by processing state
+		ok &= Timed(w, "ugc", [&] {
+			w.Declare("darkflame_ugc_items", "Player models (model) and cars and rockets (modular) by UGC processing state", "gauge");
+			const auto add = [&](const char* kind, const std::vector<std::pair<IUgc::eProcessState, uint64_t>>& rows) {
+				std::map<std::string, uint64_t> states{ {"pending", 0}, {"done", 0}, {"failed", 0} };
+				for (const auto& [state, count] : rows) {
+					states[state == IUgc::eProcessState::PENDING ? "pending" : state == IUgc::eProcessState::DONE ? "done" : "failed"] += count;
+				}
+				for (const auto& [state, count] : states) w.Add("darkflame_ugc_items", "", "gauge", { {"kind", kind}, {"state", state} }, static_cast<double>(count));
+			};
+			add("model", db->GetUgcProcessCounts());
+			add("modular", db->GetModularBuildProcessCounts());
+		});
+
 		ok &= Timed(w, "strikes", [&] {
 			w.Add("darkflame_active_strikes", "Strikes that still count, all accounts together", "gauge", db->CountActiveStrikes(0, Strikes::CountsSince()));
 		});

dDashboardServer/routes/ServerRoutes.cpp

@@ -2,6 +2,8 @@
 
 #include <algorithm>
 #include <chrono>
+#include <cmath>
+#include <iterator>
 #include <ctime>
 #include <filesystem>
 #include <fstream>
@@ -11,6 +13,8 @@
 
 #include "RouteUtils.h"
 #include "ServerState.h"
+#include "Traffic.h"
+#include "dServer.h"
 #include "Background.h"
 #include "PlayerActions.h"
 #include "BinaryPathFinder.h"
@@ -21,6 +25,10 @@
 #include "GeneralUtils.h"
 #include "eHTTPMethod.h"
 
+#ifdef __linux__
+#include <unistd.h>
+#endif
+
 using namespace RouteUtils;
 
 namespace {
@@ -29,34 +37,64 @@ namespace {
 	constexpr size_t MAX_MATCHES = 1000;
 	std::chrono::steady_clock::time_point g_NextSample{};
 
-	// All DarkflameServer processes from this build together, in kB (Linux: /proc; elsewhere 0)
-	uint64_t ServerMemoryKb() {
+	uint32_t OwnPid() {
 #ifdef __linux__
-		const auto binaryDir = BinaryPathFinder::GetBinaryDir().string();
-		uint64_t total = 0;
-		std::error_code ec;
-		for (const auto& entry : fs::directory_iterator("/proc", ec)) {
-			const auto pid = entry.path().filename().string();
-			if (pid.empty() || !std::all_of(pid.begin(), pid.end(), ::isdigit)) continue;
-			std::error_code linkError;
-			const auto exe = fs::read_symlink(entry.path() / "exe", linkError).string();
-			if (linkError || !exe.starts_with(binaryDir) || !exe.ends_with("Server")) continue;
-			std::ifstream status(entry.path() / "status");
-			std::string line;
-			while (std::getline(status, line)) {
-				if (!line.starts_with("VmRSS:")) continue;
-				uint64_t kb = 0;
-				std::istringstream(line.substr(6)) >> kb; // "VmRSS:   12345 kB"
-				total += kb;
-				break;
-			}
-		}
-		return total;
+		return static_cast<uint32_t>(getpid());
 #else
 		return 0;
 #endif
 	}
 
+	// CPU seconds of each process when it was last looked at, for its CPU use since then
+	std::map<uint32_t, std::pair<double, std::chrono::steady_clock::time_point>> g_LastCpu;
+
+	// All DarkflameServer processes from this build together, in kB
+	uint64_t ServerMemoryKb() {
+		uint64_t total = 0;
+		for (const auto& process : ServerRoutes::Processes()) total += process.memoryKb;
+		return total;
+	}
+
+	nlohmann::json ProcessJson(const ServerRoutes::Process& p) {
+		return { {"pid", p.pid}, {"memory_kb", p.memoryKb}, {"cpu_percent", std::round(p.cpuPercent * 10) / 10}, {"started_at", p.startedAt} };
+	}
+
+	// The UGC server's queue in the database (its work list), from the model and modular build tables
+	nlohmann::json UgcCounts() {
+		const auto counts = [](const std::vector<std::pair<IUgc::eProcessState, uint64_t>>& rows) {
+			nlohmann::json out = { {"pending", 0}, {"done", 0}, {"failed", 0} };
+			for (const auto& [state, count] : rows) {
+				out[state == IUgc::eProcessState::PENDING ? "pending" : state == IUgc::eProcessState::DONE ? "done" : "failed"] = count;
+			}
+			return out;
+		};
+		return { {"model", counts(Database::Get()->GetUgcProcessCounts())}, {"modular", counts(Database::Get()->GetModularBuildProcessCounts())} };
+	}
+
+	// What /api/servers and the home page show of the UGC server
+	nlohmann::json UgcSummary(bool withCounts) {
+		const auto state = ServerState::GetServerStateJson()["ugc"];
+		nlohmann::json out = state;
+		uint32_t pid = 0;
+		{
+			std::lock_guard lock(ServerState::g_StatusMutex);
+			pid = ServerState::g_UgcPid;
+		}
+		out["pid"] = pid;
+		// Its traffic report (every few seconds, via master) carries its workers, totals and storage
+		const auto traffic = Traffic::Server("ugc");
+		out["gauges"] = traffic.value("gauges", nlohmann::json::object());
+		out["last_report"] = traffic.value("last_seen", int64_t{});
+		if (withCounts) {
+			try {
+				out["counts"] = UgcCounts();
+			} catch (const std::exception& ex) {
+				LOG_DEBUG("Could not count the UGC queue: %s", ex.what());
+			}
+		}
+		return out;
+	}
+
 	IServerHealth::HealthSample CurrentSample() {
 		const auto state = ServerState::GetServerStateJson();
 		IServerHealth::HealthSample sample;
@@ -66,6 +104,8 @@ namespace {
 		sample.authOnline = state["auth"].value("online", false);
 		sample.chatOnline = state["chat"].value("online", false);
 		sample.memoryKb = ServerMemoryKb();
+		sample.ugcEnabled = state["ugc"].value("enabled", false);
+		sample.ugcOnline = state["ugc"].value("online", false);
 		return sample;
 	}
 
@@ -86,6 +126,76 @@ namespace {
 }
 
 namespace ServerRoutes {
+	std::vector<Process> Processes() {
+		std::vector<Process> processes;
+#ifdef __linux__
+		const auto binaryDir = BinaryPathFinder::GetBinaryDir().string();
+		const auto ticks = static_cast<double>(sysconf(_SC_CLK_TCK));
+		const auto now = std::chrono::steady_clock::now();
+		// Process start times are in ticks since boot
+		int64_t bootTime = 0;
+		{
+			std::ifstream stat("/proc/stat");
+			std::string line;
+			while (std::getline(stat, line)) {
+				if (line.starts_with("btime ")) { bootTime = GeneralUtils::TryParse<int64_t>(line.substr(6)).value_or(0); break; }
+			}
+		}
+		std::map<uint32_t, std::pair<double, std::chrono::steady_clock::time_point>> seen;
+		std::error_code ec;
+		for (const auto& entry : fs::directory_iterator("/proc", ec)) {
+			const auto pidText = entry.path().filename().string();
+			if (pidText.empty() || !std::all_of(pidText.begin(), pidText.end(), ::isdigit)) continue;
+			std::error_code linkError;
+			const auto exe = fs::read_symlink(entry.path() / "exe", linkError);
+			const auto exeText = exe.string();
+			if (linkError || !exeText.starts_with(binaryDir) || !exeText.ends_with("Server")) continue;
+			Process process;
+			process.program = exe.filename().string();
+			process.pid = GeneralUtils::TryParse<uint32_t>(pidText).value_or(0);
+
+			std::ifstream status(entry.path() / "status");
+			std::string line;
+			while (std::getline(status, line)) {
+				if (!line.starts_with("VmRSS:")) continue;
+				std::istringstream(line.substr(6)) >> process.memoryKb; // "VmRSS:   12345 kB"
+				break;
+			}
+
+			// stat: "pid (name) state ..." then utime, stime (fields 14, 15) and starttime (22), counted after the name
+			std::ifstream statFile(entry.path() / "stat");
+			std::string stat((std::istreambuf_iterator<char>(statFile)), std::istreambuf_iterator<char>());
+			if (const auto close = stat.rfind(')'); close != std::string::npos) {
+				std::istringstream fields(stat.substr(close + 2));
+				std::vector<std::string> values{ std::istream_iterator<std::string>(fields), std::istream_iterator<std::string>() };
+				if (values.size() > 19 && ticks > 0) {
+					const auto cpu = (GeneralUtils::TryParse<double>(values[11]).value_or(0) + GeneralUtils::TryParse<double>(values[12]).value_or(0)) / ticks;
+					process.startedAt = bootTime + static_cast<int64_t>(GeneralUtils::TryParse<double>(values[19]).value_or(0) / ticks);
+					const auto last = g_LastCpu.find(process.pid);
+					if (last != g_LastCpu.end()) {
+						const auto wall = std::chrono::duration<double>(now - last->second.second).count();
+						if (wall > 0.5) process.cpuPercent = std::max(0.0, (cpu - last->second.first) / wall * 100);
+					}
+					// Keep the older reading when looked at again too soon for a fair number
+					seen[process.pid] = last != g_LastCpu.end() && std::chrono::duration<double>(now - last->second.second).count() <= 0.5 ? last->second : std::pair{ cpu, now };
+				}
+			}
+
+			// World servers: WorldServer -zone <id> -port <port> -instance <id> ...
+			std::ifstream cmdline(entry.path() / "cmdline");
+			std::vector<std::string> args;
+			for (std::string arg; std::getline(cmdline, arg, '\0');) args.push_back(arg);
+			for (size_t i = 0; i + 1 < args.size(); i++) {
+				if (args[i] == "-zone") process.zoneId = GeneralUtils::TryParse<uint32_t>(args[i + 1]).value_or(0);
+				if (args[i] == "-instance") process.instanceId = GeneralUtils::TryParse<uint32_t>(args[i + 1]).value_or(0);
+			}
+			processes.push_back(std::move(process));
+		}
+		g_LastCpu = std::move(seen);
+#endif
+		return processes;
+	}
+
 	void Update() {
 		const auto now = std::chrono::steady_clock::now();
 		// The first sample waits a minute, so auth and chat have time to connect after a start
@@ -109,13 +219,71 @@ void RegisterServerRoutes() {
 			const auto now = static_cast<int64_t>(std::time(nullptr));
 			nlohmann::json samples = nlohmann::json::array();
 			for (const auto& s : Database::Get()->GetHealthSamples(now - span, now, bucket)) {
-				samples.push_back({ {"time", s.time}, {"players", s.players}, {"worlds", s.worlds}, {"auth", s.authOnline}, {"chat", s.chatOnline}, {"memory_kb", s.memoryKb} });
+				samples.push_back({ {"time", s.time}, {"players", s.players}, {"worlds", s.worlds}, {"auth", s.authOnline}, {"chat", s.chatOnline}, {"memory_kb", s.memoryKb},
+					{"ugc_enabled", s.ugcEnabled}, {"ugc", s.ugcOnline} });
 			}
 			const auto current = CurrentSample();
 			JsonSuccess(reply, { {"from", now - span}, {"to", now}, {"bucket", bucket}, {"samples", samples},
-				{"current", { {"players", current.players}, {"worlds", current.worlds}, {"auth", current.authOnline}, {"chat", current.chatOnline}, {"memory_kb", current.memoryKb} }} });
+				{"current", { {"players", current.players}, {"worlds", current.worlds}, {"auth", current.authOnline}, {"chat", current.chatOnline}, {"memory_kb", current.memoryKb},
+					{"ugc_enabled", current.ugcEnabled}, {"ugc", current.ugcOnline} }} });
 		});
 
+	Route(eHTTPMethod::GET, "/api/servers", Perm("health_view"),
+		"Every server: {servers: [{key, label, kind, online, since, players, process: {pid, memory_kb, cpu_percent, started_at}, "
+		"traffic: {online, last_seen, connections, ping_ms, gauges}}], ugc: {enabled, online, since, pid, gauges, last_report, counts}}. "
+		"cpu_percent is of one core since the last time the processes were read",
+		[](HTTPReply& reply, const HTTPContext&) {
+			const auto state = ServerState::GetServerStateJson();
+			auto processes = ServerRoutes::Processes();
+			const auto ugc = UgcSummary(true);
+			// The process of a server: by pid when known, else the only one of its program (worlds: by zone and instance)
+			const auto take = [&](const std::string& program, uint32_t pid, uint32_t zone, uint32_t instance) -> nlohmann::json {
+				for (auto it = processes.begin(); it != processes.end(); ++it) {
+					const bool match = pid ? it->pid == pid : it->program == program && (program != "WorldServer" || (it->zoneId == zone && it->instanceId == instance));
+					if (!match) continue;
+					auto json = ProcessJson(*it);
+					processes.erase(it);
+					return json;
+				}
+				return nullptr;
+			};
+			const auto row = [&](const std::string& key, const std::string& kind, bool online, int64_t since, nlohmann::json process) {
+				const auto traffic = Traffic::Server(key);
+				const auto link = traffic.value("link", nlohmann::json::object());
+				return nlohmann::json{ {"key", key}, {"label", traffic.value("label", key)}, {"kind", kind}, {"online", online}, {"since", since}, {"process", process},
+					{"traffic", { {"online", traffic.value("online", false)}, {"last_seen", traffic.value("last_seen", int64_t{})},
+						{"connections", link.value("connections", 0u)}, {"ping_ms", link.value("ping_ms", 0u)}, {"gauges", traffic.value("gauges", nlohmann::json::object())} }} };
+			};
+			nlohmann::json servers = nlohmann::json::array();
+			const bool masterUp = Game::server && Game::server->GetIsConnectedToMaster();
+			servers.push_back(row("master", "MASTER", masterUp, 0, take("MasterServer", 0, 0, 0)));
+			servers.push_back(row("auth", "AUTH", state["auth"].value("online", false), state["auth"].value("since", int64_t{}), take("AuthServer", 0, 0, 0)));
+			servers.push_back(row("chat", "CHAT", state["chat"].value("online", false), state["chat"].value("since", int64_t{}), take("ChatServer", 0, 0, 0)));
+			servers.push_back(row("dashboard", "DASHBOARD", true, 0, take("DashboardServer", OwnPid(), 0, 0)));
+			if (ugc.value("enabled", false) || ugc.value("online", false)) {
+				auto ugcRow = row("ugc", "UGC", ugc.value("online", false), ugc.value("since", int64_t{}), take("UgcServer", ugc.value("pid", 0u), 0, 0));
+				if (ugcRow["process"].is_null()) ugcRow["process"] = take("UgcServer", 0, 0, 0);
+				servers.push_back(ugcRow);
+			}
+			for (const auto& world : state.value("worlds", nlohmann::json::array())) {
+				const auto zone = world.value("mapID", 0u), instance = world.value("instanceID", 0u);
+				auto worldRow = row("world:" + std::to_string(zone) + ":" + std::to_string(instance), "WORLD", true, 0, take("WorldServer", 0, zone, instance));
+				worldRow["players"] = world.value("players", 0u);
+				servers.push_back(worldRow);
+			}
+			// Anything left over, e.g. a world master no longer lists
+			for (const auto& process : processes) {
+				servers.push_back({ {"key", process.program + ":" + std::to_string(process.pid)}, {"label", process.program}, {"kind", "OTHER"}, {"online", false},
+					{"since", 0}, {"process", ProcessJson(process)}, {"traffic", nullptr} });
+			}
+			JsonSuccess(reply, { {"servers", servers}, {"ugc", ugc} });
+		});
+
+	Route(eHTTPMethod::GET, "/api/servers/ugc", Perm("health_view"),
+		"The UGC server for the home page: {enabled, online, since, pid, gauges: {workers_busy, workers_queued, workers_threads, ugc_made_total, "
+		"ugc_failed_total, ugc_evicted_total, ugc_stored_bytes, ugc_max_storage_bytes}, last_report, counts: {model, modular: {pending, done, failed}}}",
+		[](HTTPReply& reply, const HTTPContext&) { JsonSuccess(reply, UgcSummary(true)); });
+
 	Route(eHTTPMethod::GET, "/api/crash_dumps", Perm("logs_system"), "Crash dumps in dump_folder, newest first",
 		[](HTTPReply& reply, const HTTPContext&) {
 			const auto folder = DumpFolder();

dDashboardServer/routes/ServerRoutes.h

@@ -1,12 +1,31 @@
 #pragma once
 
+#include <cstdint>
+#include <string>
+#include <vector>
+
 /**
- * Running the server: health history (a sample a minute: players, worlds, auth/chat up, memory), crash dumps from
- * dump_folder, and searching the servers' log files. Prometheus metrics are in PrometheusMetrics.h.
+ * Running the server: health history (a sample a minute: players, worlds, auth/chat/UGC up, memory), the running
+ * server processes, crash dumps from dump_folder, and searching the servers' log files. Prometheus metrics are in
+ * PrometheusMetrics.h.
  */
 void RegisterServerRoutes();
 
 namespace ServerRoutes {
 	// Main loop: take a health sample once a minute
 	void Update();
+
+	// A running DarkflameServer process from this build
+	struct Process {
+		std::string program; // the binary: "WorldServer", "UgcServer", ...
+		uint32_t pid{};
+		uint64_t memoryKb{};  // resident
+		double cpuPercent{};  // of one core, since this process was last looked at (0 the first time)
+		int64_t startedAt{};  // unix time
+		uint32_t zoneId{};    // world servers: from the command line
+		uint32_t instanceId{};
+	};
+
+	// Main thread: every server process of this build (Linux: /proc; elsewhere none)
+	std::vector<Process> Processes();
 }

dDashboardServer/routes/ServerState.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <chrono>
+#include <ctime>
 #include <map>
 #include <mutex>
 #include <vector>
@@ -13,6 +14,15 @@ struct ServerStatus {
 	uint32_t players{0};
 	std::string version{};
 	std::chrono::steady_clock::time_point lastSeen{};
+	int64_t since{}; // unix time the dashboard first saw it up this time (0: down)
+
+	// Online or not, keeping `since` for as long as it stays up
+	void Set(bool up) {
+		if (up && !online) since = static_cast<int64_t>(std::time(nullptr));
+		if (!up) since = 0;
+		online = up;
+		if (up) lastSeen = std::chrono::steady_clock::now();
+	}
 };
 
 struct WorldInstanceInfo {
@@ -34,6 +44,10 @@ struct WorldInstanceInfo {
 namespace ServerState {
 	extern ServerStatus g_AuthStatus;
 	extern ServerStatus g_ChatStatus;
+	// The UGC server (docs/UgcServer.md): enabled is master's enable_ugc_server, pid the process master last started
+	extern ServerStatus g_UgcStatus;
+	extern bool g_UgcEnabled;
+	extern uint32_t g_UgcPid;
 	extern std::vector<WorldInstanceInfo> g_WorldInstances;
 	extern std::mutex g_StatusMutex;
 
@@ -45,6 +59,9 @@ namespace ServerState {
 		data["auth"]["players"] = g_AuthStatus.players;
 		data["chat"]["online"] = g_ChatStatus.online;
 		data["chat"]["players"] = g_ChatStatus.players;
+		data["auth"]["since"] = g_AuthStatus.since;
+		data["chat"]["since"] = g_ChatStatus.since;
+		data["ugc"] = { {"enabled", g_UgcEnabled}, {"online", g_UgcStatus.online}, {"since", g_UgcStatus.since} };
 
 		uint32_t totalOnlinePlayers = 0;
 		data["worlds"] = nlohmann::json::array();

dDashboardServer/routes/Traffic.cpp

@@ -308,6 +308,10 @@ namespace {
 }
 
 namespace Traffic {
+	nlohmann::json Server(const std::string& key) {
+		return ServerInfo(key, TrafficStats::Now());
+	}
+
 	void Ingest(const ServerTraffic& report) {
 		g_History.Ingest(static_cast<uint16_t>(report.serverType), report.zoneId, report.instanceId, report.report, TrafficStats::Now());
 		g_Changed = true;

dDashboardServer/routes/Traffic.h

@@ -1,5 +1,8 @@
 #pragma once
 
+#include <string>
+#include "json.hpp"
+
 struct ServerTraffic;
 namespace MetricsFormat { class Writer; }
 
@@ -16,6 +19,9 @@ namespace Traffic {
 	// Main loop: write finished minutes, tell open pages there is news
 	void Update();
 
+	// Main thread: one server's last report ("ugc", "chat", ...): {key, label, online, last_seen, link, gauges}
+	nlohmann::json Server(const std::string& key);
+
 	// Counters for /metrics
 	void AddMetrics(MetricsFormat::Writer& w);
 }

dDashboardServer/routes/WSRoutes.cpp

@@ -60,7 +60,8 @@ void RegisterWSRoutes() {
 void BroadcastDashboardUpdate() {
 	// Sent to every signed-in account, so it only has what players may see; staff load /api/worlds for details
 	nlohmann::json status = ServerState::PlayerSafe(ServerState::GetServerStateJson());
-	Alerts::ServerStatus(status["auth"].value("online", false), status["chat"].value("online", false));
+	Alerts::ServerStatus(status["auth"].value("online", false), status["chat"].value("online", false),
+		status["ugc"].value("enabled", false), status["ugc"].value("online", false));
 	status["restart"] = LiveWorld::RestartStatus();
 
 	try {