#include "PerfHistory.h" #include #include #include #include #include "ServiceType.h" #include "TrafficStats.h" namespace { double Ms(uint64_t microseconds) { return std::round(static_cast(microseconds) / 10.0) / 100.0; } double Round2(double value) { return std::round(value * 100.0) / 100.0; } std::string LabelOf(const Profiler::Node& node, const PerfHistory::Label& label) { return label ? label(node) : Profiler::DefaultLabel(node); } // The seconds of a server from `from` (inclusive) to `to` (exclusive) template void ForSeconds(const PerfHistory::Server& server, int64_t from, int64_t to, Fn&& fn) { auto it = std::lower_bound(server.seconds.begin(), server.seconds.end(), from, [](const Profiler::Second& s, int64_t t) { return s.time < t; }); for (; it != server.seconds.end() && it->time < to; ++it) fn(*it); } } bool PerfHistory::ParseKey(const std::string& key, uint16_t& serviceType, uint32_t& zoneId, uint32_t& instanceId) { zoneId = 0; instanceId = 0; if (key == "master") serviceType = static_cast(ServiceType::MASTER); else if (key == "auth") serviceType = static_cast(ServiceType::AUTH); else if (key == "chat") serviceType = static_cast(ServiceType::CHAT); else if (key == "dashboard") serviceType = static_cast(ServiceType::DASHBOARD); else if (key == "ugc") serviceType = static_cast(ServiceType::UGC); else if (key.starts_with("world:")) { const auto colon = key.find(':', 6); if (colon == std::string::npos) return false; const auto zone = key.substr(6, colon - 6), instance = key.substr(colon + 1); const auto digits = [](const std::string& s) { return !s.empty() && s.size() <= 9 && std::all_of(s.begin(), s.end(), [](char c) { return c >= '0' && c <= '9'; }); }; if (!digits(zone) || !digits(instance)) return false; serviceType = static_cast(ServiceType::WORLD); zoneId = static_cast(std::stoul(zone)); instanceId = static_cast(std::stoul(instance)); } else { return false; } return true; } void PerfHistory::Ingest(const std::string& key, const Profiler::Report& report, int64_t now) { if (!report.present) return; auto [it, added] = m_Servers.try_emplace(key); auto& server = it->second; if (added) server.key = key; server.lastSeen = now; server.slowThresholdMs = report.slowThresholdMs; for (const auto& second : report.seconds) { if (server.seconds.empty() || server.seconds.back().time < second.time) { server.seconds.push_back(second); continue; } // Late or repeated: added to the one it belongs to auto at = std::lower_bound(server.seconds.begin(), server.seconds.end(), second.time, [](const Profiler::Second& s, int64_t t) { return s.time < t; }); if (at != server.seconds.end() && at->time == second.time) at->Merge(second); else server.seconds.insert(at, second); } if (!report.messages.empty()) server.messages.emplace_back(now, report.messages); for (const auto& frame : report.worst) { server.worst.push_back(frame); if (server.worst.size() > WORST_KEPT) { // The shortest one goes const auto shortest = std::min_element(server.worst.begin(), server.worst.end(), [](const Profiler::Frame& a, const Profiler::Frame& b) { return a.durationUs < b.durationUs; }); server.worst.erase(shortest); } } for (const auto& frame : report.slow) { m_Slow.push_back({ key, frame }); if (m_Slow.size() > SLOW_KEPT) m_Slow.pop_front(); } } void PerfHistory::Forget(int64_t now) { for (auto it = m_Servers.begin(); it != m_Servers.end();) { auto& server = it->second; if (now - server.lastSeen > FORGET_AFTER) { it = m_Servers.erase(it); continue; } while (!server.seconds.empty() && server.seconds.front().time < now - SECONDS_KEPT) server.seconds.pop_front(); while (!server.messages.empty() && server.messages.front().first < now - RECENT_SECONDS) server.messages.pop_front(); std::erase_if(server.worst, [now](const Profiler::Frame& frame) { return frame.timeMs / 1000 < now - RECENT_SECONDS; }); ++it; } } nlohmann::json PerfHistory::ServersJson(int64_t now, int64_t span, int64_t onlineSeconds, const ServerLabel& label) const { nlohmann::json out = nlohmann::json::array(); for (const auto& [key, server] : m_Servers) { uint64_t ticks = 0, totalUs = 0; uint32_t maxUs = 0; int64_t covered = 0; TrafficStats::Histogram frames; ForSeconds(server, now - span, now, [&](const Profiler::Second& s) { covered++; ticks += s.ticks; totalUs += s.totalUs; maxUs = std::max(maxUs, s.maxUs); frames.Merge(s.frames); }); size_t slow = 0; for (const auto& entry : m_Slow) if (entry.server == key && entry.frame.timeMs / 1000 >= now - span) slow++; nlohmann::json row = { {"key", key}, {"label", label ? label(key) : key}, {"online", now - server.lastSeen <= onlineSeconds}, {"last_seen", server.lastSeen}, {"slow_threshold_ms", server.slowThresholdMs}, {"slow", slow}, {"seconds", covered}, {"ticks_per_second", covered ? Round2(static_cast(ticks) / static_cast(covered)) : 0.0}, {"avg_ms", ticks ? nlohmann::json(Ms(totalUs / ticks)) : nlohmann::json(nullptr)}, {"p95_ms", ticks ? nlohmann::json(Ms(frames.Percentile(0.95))) : nlohmann::json(nullptr)}, {"max_ms", ticks ? nlohmann::json(Ms(maxUs)) : nlohmann::json(nullptr)}, {"busy_percent", covered ? Round2(static_cast(totalUs) / (static_cast(covered) * 1e4)) : 0.0}, }; out.push_back(std::move(row)); } std::stable_sort(out.begin(), out.end(), [](const nlohmann::json& a, const nlohmann::json& b) { return a["busy_percent"].get() > b["busy_percent"].get(); }); return out; } nlohmann::json PerfHistory::Series(const std::string& key, int64_t from, int64_t to, int64_t step) const { step = std::max(step, 1); const auto count = static_cast(std::max((to - from) / step, 0)); nlohmann::json times = nlohmann::json::array(), ticks = nlohmann::json::array(), avg = nlohmann::json::array(), p95 = nlohmann::json::array(), max = nlohmann::json::array(); std::vector phases(Profiler::PHASES, nlohmann::json::array()); const auto it = m_Servers.find(key); for (size_t i = 0; i < count; i++) { const int64_t start = from + static_cast(i) * step; times.push_back(start); Profiler::Second sum; int64_t covered = 0; if (it != m_Servers.end()) { ForSeconds(it->second, start, start + step, [&](const Profiler::Second& s) { covered++; sum.Merge(s); }); } // Not reported: gaps. Reported without frames (a stuck loop): no frames. if (!covered) { ticks.push_back(nullptr); avg.push_back(nullptr); p95.push_back(nullptr); max.push_back(nullptr); for (auto& phase : phases) phase.push_back(nullptr); continue; } ticks.push_back(Round2(static_cast(sum.ticks) / static_cast(covered))); avg.push_back(sum.ticks ? nlohmann::json(Ms(sum.totalUs / sum.ticks)) : nlohmann::json(nullptr)); p95.push_back(sum.ticks ? nlohmann::json(Ms(sum.frames.Percentile(0.95))) : nlohmann::json(nullptr)); max.push_back(sum.ticks ? nlohmann::json(Ms(sum.maxUs)) : nlohmann::json(nullptr)); for (size_t p = 0; p < Profiler::PHASES; p++) phases[p].push_back(Ms(sum.phaseUs[p] / static_cast(covered))); } nlohmann::json phaseJson = nlohmann::json::object(); for (size_t p = 0; p < Profiler::PHASES; p++) phaseJson[Profiler::PhaseName(p)] = std::move(phases[p]); return { {"key", key}, {"from", from}, {"to", to}, {"step", step}, {"times", times}, {"ticks_per_second", ticks}, {"avg_ms", avg}, {"p95_ms", p95}, {"max_ms", max}, {"phases_ms_per_second", phaseJson}, {"slow_threshold_ms", it != m_Servers.end() ? it->second.slowThresholdMs : 0} }; } nlohmann::json PerfHistory::FrameJson(const Profiler::Frame& frame, const Label& label) { nlohmann::json phases = nlohmann::json::object(); for (size_t p = 0; p < Profiler::PHASES; p++) { if (frame.phaseUs[p]) phases[Profiler::PhaseName(p)] = Ms(frame.phaseUs[p]); } nlohmann::json scopes = nlohmann::json::array(); for (size_t i = 0; i < frame.scopes.size(); i++) { const auto& node = frame.scopes[i]; uint64_t children = 0; for (size_t j = i + 1; j < frame.scopes.size() && frame.scopes[j].depth > node.depth; j++) { if (frame.scopes[j].depth == node.depth + 1) children += frame.scopes[j].totalUs; } scopes.push_back({ {"label", LabelOf(node, label)}, {"name", node.name}, {"arg", node.arg}, {"depth", node.depth}, {"count", node.count}, {"total_ms", Ms(node.totalUs)}, {"self_ms", Ms(node.totalUs > children ? node.totalUs - children : 0)}, {"start_ms", Ms(node.startUs)} }); } return { {"time_ms", frame.timeMs}, {"duration_ms", Ms(frame.durationUs)}, {"implicit", frame.implicit}, {"phases", phases}, {"path", frame.Path([&label](const Profiler::Node& node) { return LabelOf(node, label); })}, {"scopes", scopes} }; } nlohmann::json PerfHistory::Worst(const std::string& key, int64_t since, size_t limit, const Label& label) const { nlohmann::json out = nlohmann::json::array(); const auto it = m_Servers.find(key); if (it == m_Servers.end()) return out; std::vector frames; for (const auto& frame : it->second.worst) if (frame.timeMs / 1000 >= since) frames.push_back(&frame); std::stable_sort(frames.begin(), frames.end(), [](const Profiler::Frame* a, const Profiler::Frame* b) { return a->durationUs > b->durationUs; }); if (frames.size() > limit) frames.resize(limit); for (const auto* frame : frames) out.push_back(FrameJson(*frame, label)); return out; } nlohmann::json PerfHistory::Messages(const std::string& key, int64_t since, size_t limit, const MessageNames& names) const { nlohmann::json out = nlohmann::json::array(); const auto it = m_Servers.find(key); if (it == m_Servers.end()) return out; std::unordered_map merged; for (const auto& [time, messages] : it->second.messages) { if (time < since) continue; for (const auto& m : messages) { auto& sum = merged[m.key]; sum.key = m.key; sum.count += m.count; sum.totalUs += m.totalUs; sum.maxUs = std::max(sum.maxUs, m.maxUs); } } std::vector list; for (const auto& [_, m] : merged) list.push_back(m); std::sort(list.begin(), list.end(), [](const Profiler::MessageTime& a, const Profiler::MessageTime& b) { return a.totalUs != b.totalUs ? a.totalUs > b.totalUs : a.key < b.key; }); if (list.size() > limit) list.resize(limit); for (const auto& m : list) { nlohmann::json row = names ? names(m.key) : nlohmann::json::object(); row["count"] = m.count; row["total_ms"] = Ms(m.totalUs); row["avg_ms"] = m.count ? Ms(m.totalUs / m.count) : 0.0; row["max_ms"] = Ms(m.maxUs); out.push_back(std::move(row)); } return out; } nlohmann::json PerfHistory::SlowJson(const std::string& key, const Label& label, const ServerLabel& serverLabel) const { nlohmann::json out = nlohmann::json::array(); for (auto it = m_Slow.rbegin(); it != m_Slow.rend(); ++it) { if (!key.empty() && it->server != key) continue; auto frame = FrameJson(it->frame, label); frame["server"] = it->server; frame["server_label"] = serverLabel ? serverLabel(it->server) : it->server; out.push_back(std::move(frame)); } return out; } nlohmann::json PerfHistory::ProfileJson(const Profiler::Profile& profile, const Label& label) { nlohmann::json nodes = nlohmann::json::array(); for (size_t i = 0; i < profile.nodes.size(); i++) { const auto& node = profile.nodes[i]; uint64_t children = 0; for (size_t j = i + 1; j < profile.nodes.size() && profile.nodes[j].depth > node.depth; j++) { if (profile.nodes[j].depth == node.depth + 1) children += profile.nodes[j].totalUs; } nodes.push_back({ {"label", LabelOf(node, label)}, {"depth", node.depth}, {"count", node.count}, {"total_us", node.totalUs}, {"self_us", node.totalUs > children ? node.totalUs - children : 0} }); } return { {"duration_ms", profile.durationMs}, {"frames", profile.frames}, {"total_ms", Ms(profile.totalUs)}, {"truncated", profile.truncated}, {"nodes", nodes}, {"folded", Profiler::Folded(profile.nodes, [&label](const Profiler::Node& node) { return LabelOf(node, label); })} }; }