#pragma once #include #include #include #include #include #include #include #include #include #include /** * Traffic diagnostics every server keeps about itself: packets and bytes in and out per second, the packet and game * message types they were, and (servers with a web server) HTTP requests with their latency. Counting is a few * additions under an uncontended lock; every few seconds dServer takes a Report and ships it to the dashboard through * master (see docs/Dashboard.md, "Traffic diagnostics"). * * Seconds are Unix seconds, so reports from different servers line up. */ namespace TrafficStats { /** * Latency histogram with fixed, geometric buckets (three per doubling, from 100 microseconds), so histograms from * different seconds and servers add up and percentiles of a minute or an hour come out right. */ class Histogram { public: static constexpr size_t BUCKETS = 58; // the last one holds everything over ~41 s // Largest value (microseconds) bucket i holds; the last bucket has no limit (UINT64_MAX) static uint64_t UpperBound(size_t bucket); static size_t BucketFor(uint64_t microseconds); void Add(uint64_t microseconds, uint32_t count = 1); void Merge(const Histogram& other); void AddBucket(size_t bucket, uint32_t count); uint64_t Count() const { return m_Count; } uint64_t Sum() const { return m_Sum; } // microseconds, all values together void SetSum(uint64_t sum) { m_Sum = sum; } bool Empty() const { return m_Count == 0; } uint32_t At(size_t bucket) const { return m_Counts[bucket]; } // The value below which `fraction` (0..1) of the values are, interpolated within its bucket; 0 when empty uint64_t Percentile(double fraction) const; // Non-empty buckets as (bucket, count), for the wire and for keeping many histograms small std::vector> Sparse() const; static Histogram FromSparse(const std::vector>& sparse, uint64_t sum); private: std::array m_Counts{}; uint64_t m_Count{}; uint64_t m_Sum{}; }; // The header a server puts on HTTP requests it makes to another server's web server, naming itself constexpr const char* SERVER_HEADER = "X-Darkflame-Server"; // HTTP status classes 1xx..5xx as index 0..4 (anything else counts as 5xx) size_t StatusClass(uint16_t status); // What a packet was: its LU service and packet ID, and for game messages the game message ID struct MessageKey { bool outbound{}; uint16_t service{}; // ServiceType; RAKNET for RakNet's own messages (no LU header) uint32_t packet{}; // LU packet ID, or the RakNet message ID uint16_t gameMessage{}; // game messages only static constexpr uint16_t RAKNET = 0xFFFF; uint64_t Packed() const; static MessageKey Unpack(uint64_t packed); bool operator==(const MessageKey&) const = default; }; // Reads the key from a packet's first bytes (never reads past `length`) MessageKey KeyOf(const uint8_t* data, size_t length, bool outbound); /** * Which side of a server a packet went to or came from, so the dashboard can draw each link: its own connections * (players' game clients on auth and worlds), its link to master, or other servers (every server on master, the * worlds on chat, chat on a world). */ enum class Peer : uint8_t { CLIENTS = 0, MASTER = 1, SERVERS = 2 }; constexpr size_t PEER_CLASSES = 3; struct PeerCounts { uint64_t packetsIn{}; uint64_t packetsOut{}; uint64_t bytesIn{}; uint64_t bytesOut{}; void Merge(const PeerCounts& other); bool Empty() const { return packetsIn == 0 && packetsOut == 0; } bool operator==(const PeerCounts&) const = default; }; struct MessageCount { MessageKey key; uint64_t count{}; uint64_t bytes{}; }; // One second of traffic struct Second { int64_t time{}; uint64_t packetsIn{}; uint64_t packetsOut{}; uint64_t bytesIn{}; uint64_t bytesOut{}; uint64_t httpRequests{}; std::array httpStatus{}; // by StatusClass uint64_t httpBytesOut{}; Histogram httpLatency; std::array peers{}; // the packets above split by Peer uint64_t httpFromServers{}; // of httpRequests, those another server made (X-Darkflame-Server) uint64_t httpFromServersBytesOut{}; // their response bodies uint64_t httpOutRequests{}; // HTTP requests this server made to another server's web server uint64_t httpOutBytesIn{}; // the bodies those answered with void Merge(const Second& other); // adds the counts (keeps this one's time) bool Idle() const { return packetsIn == 0 && packetsOut == 0 && httpRequests == 0; } }; struct RouteStats { std::string route; // "GET /api/players/:id" uint64_t count{}; std::array status{}; uint64_t bytesOut{}; Histogram latency; void Merge(const RouteStats& other); }; // RakNet's view of the connections (all datagrams, acknowledgements and resends included), over the report struct Link { uint32_t connections{}; uint64_t datagramsSent{}; uint64_t datagramsReceived{}; uint64_t bytesSent{}; uint64_t bytesReceived{}; uint64_t resends{}; uint32_t resendQueue{}; // messages waiting to be resent now uint32_t averagePingMs{}; // over the connections }; /** * One remote end of a server over a report: a RakNet connection (RakNet's own counts: datagrams, with * acknowledgements and resends) or an HTTP client address (requests and body bytes). Personal data: the dashboard * keeps these in memory only and shows addresses only with network_ips. */ struct Connection { std::string address; // "203.0.113.5" or an IPv6 address; "" for the sum of the ones left out uint16_t port{}; // 0 for HTTP clients (one entry per address) Peer peer{}; bool http{}; uint64_t packetsIn{}, packetsOut{}; // datagrams, or HTTP requests and responses uint64_t bytesIn{}, bytesOut{}; uint32_t resends{}; uint32_t pingMs{}; uint32_t accountId{}; // a logged-in player's (world servers) or a signed-in dashboard account's (HTTP), else 0 uint64_t characterId{}; std::string account, character; uint64_t Bytes() const { return bytesIn + bytesOut; } void Merge(const Connection& other); // adds the counts }; struct Report { std::vector seconds; // oldest first, one per second without gaps std::vector messages; // the busiest types per direction std::vector routes; Link link; std::vector> gauges; // e.g. workers_busy bool peerSplit{}; // whether the seconds' `peers` are filled (reports of older servers don't have them) std::vector connections; // the busiest remote ends, busiest first Connection otherConnections; // the rest summed uint32_t otherConnectionCount{}; bool hasConnections{}; // false in reports of older servers }; // Keeps the `limit` busiest connections of the report, sums the rest into otherConnections void TrimConnections(Report& report, size_t limit); class Recorder { public: static constexpr size_t TOP_MESSAGES = 24; // per direction, per report static constexpr size_t MAX_ROUTES = 64; // distinct routes per report; more count as "other" static constexpr int64_t MAX_GAP = 120; // seconds of silence a report fills in at most static constexpr size_t MAX_HTTP_CLIENTS = 1024; // addresses per report; more count as one static constexpr size_t TOP_CONNECTIONS = 32; // connections a report names; the rest are summed // `fanout`: how many connections a broadcast went to; `peer`: which side it went to or came from void Packet(int64_t now, const MessageKey& key, uint64_t bytes, uint32_t fanout = 1, Peer peer = Peer::CLIENTS); // `fromServer`: the request came from another server (it sent SERVER_HEADER), not a browser or game client void Http(int64_t now, const std::string& route, uint16_t status, uint64_t microseconds, uint64_t bytesOut, bool fromServer = false); // A request this server made to another server's web server (the dashboard to the UGC server); any thread void HttpOut(int64_t now, uint64_t bytesIn); // An HTTP client's request by its address (bytes of the request and of the answer's body). `accountId` and `user`: // the account the request was signed in as (the dashboard's session or API key), 0 and "" when none; each // signed-in account on an address is counted apart void HttpClient(const std::string& address, bool fromServer, uint64_t bytesIn, uint64_t bytesOut, uint32_t accountId = 0, const std::string& user = {}, const std::string& apiKeyName = {}); // Evaluated when a report is taken (on the thread that takes it) void SetGauge(const std::string& name, std::function source); /** * The seconds before `now` not reported yet (silent ones as zeros, at most MAX_GAP of them), the busiest * message types and the routes since the last report. The current second stays for the next one. */ Report Take(int64_t now); // Whether `interval` seconds passed since the last Take (the first call only starts the clock) bool Due(int64_t now, int64_t interval); private: Second& SecondAt(int64_t now); // under m_Mutex std::mutex m_Mutex; std::map m_Seconds; Second* m_Current{}; // m_Seconds[m_CurrentTime] (map entries stay put) int64_t m_CurrentTime{}; int64_t m_LastReported{}; // last second a report covered std::unordered_map m_Messages; std::map m_Routes; std::unordered_map m_HttpClients; std::vector>> m_Gauges; int64_t m_LastTake{}; }; // The busiest `limit` message types of each direction, busiest first std::vector Top(const std::vector& counts, size_t limit); // This process's recorder (dServer counts packets into it, the web server requests) Recorder& Local(); int64_t Now(); // Unix seconds }