mirror of
https://github.com/DarkflameUniverse/DarkflameServer.git
synced 2026-10-02 02:43:44 +00:00
feat(capture): record whole packets of an account, a character or everything on every server
Staff arm a packet capture on the dashboard; master passes MESSAGE_CAPTURE_CONTROL ARM to every world, auth and chat and arms its own. Each server's PacketCapture tap (dServer receive, and a send hook in RakPeer::Send so replica constructions are seen too) records into one preallocated chunk per server and ships sealed chunks through master on the main loop when capture_flush_bytes or capture_flush_interval_ms is reached; past capture_buffer_max_mb the oldest chunks are dropped and the dashboard records a gap. Nothing is armed: one flag check. - targets: an account (from its login; packets before the login are kept per connection and added once auth or the world knows whose they are), a character (from when it is picked), or everything; up to 8 at once (a bit each in the record mask) - worlds and auth record their clients' packets and the master link messages of a captured player (session keys by name, zone transfers by request, player added/removed, migration); chat finds the player in each packet; master records server traffic for everything - secrets are never recorded: structs that carry them (login request, login response user key, world validation session key, session key messages between servers) are read, blanked and written again before recording; auth keeps only the handshake and login - PacketDecoder: a registry by service and message id names every packet and decodes the registered structs; CaptureBundle is the file format (DLUBNDL1, metadata, records); CaptureTools orders records on one timeline, pulls movement out, makes bundles portable or anonymous and diffs replays - the dashboard keeps packet captures in message_capture_sessions (capture_kind 1) and their packets in a file under capture_dir, one write per batch; arming is audited - MESSAGE_CAPTURE_CONTROL/DATA only gain appended enum values and trailing fields Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
@@ -1,4 +1,5 @@
|
||||
#include "AuthPackets.h"
|
||||
#include "PacketCapture.h"
|
||||
#include "MasterPackets.h"
|
||||
|
||||
#include <ctime>
|
||||
@@ -144,6 +145,8 @@ void AuthPackets::LoginRequest::Handle() {
|
||||
stamps.Add(eStamps::PASSPORT_AUTH_DB_SELECT_START);
|
||||
auto accountInfo = Database::Get()->GetAccountInfo(username);
|
||||
stamps.Add(eStamps::PASSPORT_AUTH_DB_SELECT_FINISH, accountInfo ? 1 : 0);
|
||||
// The dashboard's packet capture: this connection is this account's (failed logins included)
|
||||
PacketCapture::Bind(sysAddr, accountInfo ? accountInfo->id : 0, username);
|
||||
|
||||
if (!accountInfo) {
|
||||
LOG("No user by name %s found!", username.c_str());
|
||||
|
||||
@@ -6,6 +6,9 @@ set(DNET_SOURCES "AuthPackets.cpp"
|
||||
"dServer.cpp"
|
||||
"MailInfo.cpp"
|
||||
"MasterPackets.cpp"
|
||||
"PacketCapture.cpp"
|
||||
"PacketDecoder.cpp"
|
||||
"CaptureTools.cpp"
|
||||
"Stamps.cpp"
|
||||
"WorldPackets.cpp"
|
||||
"WorldRoutePacket.cpp"
|
||||
|
||||
104
dNet/CaptureBundle.h
Normal file
104
dNet/CaptureBundle.h
Normal file
@@ -0,0 +1,104 @@
|
||||
#ifndef __CAPTUREBUNDLE__H__
|
||||
#define __CAPTUREBUNDLE__H__
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include "json.hpp"
|
||||
#include "PacketRecord.h"
|
||||
|
||||
/**
|
||||
* The packet bundle file format (docs/CaptureReplay.md), shared by the dashboard's capture files, exported bundles,
|
||||
* bundles converted from other captures, and the capture tool:
|
||||
*
|
||||
* "DLUBNDL1" 8 bytes, the format and its version
|
||||
* u32 metadata length little endian
|
||||
* metadata UTF-8 JSON (see docs/CaptureReplay.md: origin, server version, zones and checksums,
|
||||
* the setup section, how ids are written)
|
||||
* records PacketRecordHeader + bytes, one after another to the end of the file
|
||||
*
|
||||
* The dashboard appends records to a capture's file while it runs (one write per batch), so the metadata is written
|
||||
* once, when the file is made.
|
||||
*/
|
||||
namespace CaptureBundle {
|
||||
constexpr std::string_view MAGIC{ "DLUBNDL1", 8 };
|
||||
constexpr uint32_t FORMAT_VERSION = 1;
|
||||
constexpr uint32_t MAX_METADATA = 64 * 1024 * 1024;
|
||||
|
||||
struct Record {
|
||||
PacketRecordHeader header;
|
||||
std::string bytes;
|
||||
};
|
||||
|
||||
struct Bundle {
|
||||
nlohmann::json meta = nlohmann::json::object();
|
||||
std::vector<Record> records;
|
||||
};
|
||||
|
||||
inline std::string Header(const nlohmann::json& meta) {
|
||||
const auto text = meta.dump();
|
||||
std::string out(MAGIC);
|
||||
const auto length = static_cast<uint32_t>(text.size());
|
||||
out.append(reinterpret_cast<const char*>(&length), sizeof(length));
|
||||
out += text;
|
||||
return out;
|
||||
}
|
||||
|
||||
inline void AppendRecord(std::string& out, const Record& record) {
|
||||
auto header = record.header;
|
||||
header.length = static_cast<uint32_t>(record.bytes.size());
|
||||
PacketRecord::Append(out, header, record.bytes.data());
|
||||
}
|
||||
|
||||
// Parses a whole file's contents. A file cut short in its last record (a capture still being written) keeps the
|
||||
// records before it; `error` is set for anything else.
|
||||
inline bool Parse(std::string_view data, Bundle& bundle, std::string& error, bool* truncated = nullptr) {
|
||||
if (data.size() < MAGIC.size() + 4 || data.substr(0, MAGIC.size()) != MAGIC) {
|
||||
error = "Not a packet bundle (DLUBNDL1)";
|
||||
return false;
|
||||
}
|
||||
uint32_t length{};
|
||||
std::memcpy(&length, data.data() + MAGIC.size(), sizeof(length));
|
||||
const size_t start = MAGIC.size() + 4;
|
||||
if (length > MAX_METADATA || data.size() - start < length) {
|
||||
error = "The bundle's metadata is cut short";
|
||||
return false;
|
||||
}
|
||||
bundle.meta = nlohmann::json::parse(data.substr(start, length), nullptr, false);
|
||||
if (bundle.meta.is_discarded() || !bundle.meta.is_object()) {
|
||||
error = "The bundle's metadata isn't JSON";
|
||||
return false;
|
||||
}
|
||||
bundle.records.clear();
|
||||
const bool whole = PacketRecord::ForEach(data.substr(start + length), [&](const PacketRecordHeader& header, std::string_view bytes) {
|
||||
bundle.records.push_back({ header, std::string(bytes) });
|
||||
});
|
||||
if (truncated) *truncated = !whole;
|
||||
return true;
|
||||
}
|
||||
|
||||
inline bool Load(const std::filesystem::path& path, Bundle& bundle, std::string& error, bool* truncated = nullptr) {
|
||||
std::ifstream file(path, std::ios::binary);
|
||||
if (!file) {
|
||||
error = "Can't open " + path.string();
|
||||
return false;
|
||||
}
|
||||
std::string data((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
|
||||
return Parse(data, bundle, error, truncated);
|
||||
}
|
||||
|
||||
inline bool Save(const std::filesystem::path& path, const Bundle& bundle) {
|
||||
std::string out = Header(bundle.meta);
|
||||
for (const auto& record : bundle.records) AppendRecord(out, record);
|
||||
std::ofstream file(path, std::ios::binary | std::ios::trunc);
|
||||
file.write(out.data(), static_cast<std::streamsize>(out.size()));
|
||||
return static_cast<bool>(file);
|
||||
}
|
||||
}
|
||||
|
||||
#endif //!__CAPTUREBUNDLE__H__
|
||||
253
dNet/CaptureTools.cpp
Normal file
253
dNet/CaptureTools.cpp
Normal file
@@ -0,0 +1,253 @@
|
||||
#include "CaptureTools.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstring>
|
||||
#include <set>
|
||||
|
||||
#include "PacketDecoder.h"
|
||||
#include "ServiceType.h"
|
||||
|
||||
namespace {
|
||||
using json = nlohmann::json;
|
||||
using Record = CaptureBundle::Record;
|
||||
|
||||
const char* SourceName(uint8_t source) {
|
||||
switch (static_cast<eCaptureSource>(source)) {
|
||||
case eCaptureSource::AUTH: return "auth";
|
||||
case eCaptureSource::CHAT: return "chat";
|
||||
case eCaptureSource::WORLD: return "world";
|
||||
case eCaptureSource::MASTER: return "master";
|
||||
default: return "?";
|
||||
}
|
||||
}
|
||||
|
||||
// Who is on each end, as the viewer shows it
|
||||
std::pair<std::string, std::string> Ends(const PacketRecordHeader& h) {
|
||||
std::string server = SourceName(h.source);
|
||||
if (h.source == static_cast<uint8_t>(eCaptureSource::WORLD) && h.zoneId) server += " " + std::to_string(h.zoneId) + ":" + std::to_string(h.instanceId);
|
||||
std::string other;
|
||||
if (h.flags & PacketRecordFlags::MASTER_LINK) other = "master";
|
||||
else if (h.source == static_cast<uint8_t>(eCaptureSource::CHAT) || h.source == static_cast<uint8_t>(eCaptureSource::MASTER)) other = "server";
|
||||
else other = "client";
|
||||
if (h.flags & PacketRecordFlags::BROADCAST) other = "everyone";
|
||||
return h.direction == static_cast<uint8_t>(ePacketDirection::RECEIVED) ? std::pair{ other, server } : std::pair{ server, other };
|
||||
}
|
||||
|
||||
void ReplaceAll(std::string& bytes, int64_t from, int64_t to) {
|
||||
if (from == 0 || from == to) return;
|
||||
char a[8], b[8];
|
||||
std::memcpy(a, &from, 8);
|
||||
std::memcpy(b, &to, 8);
|
||||
const std::string_view needle(a, 8);
|
||||
for (size_t at = bytes.find(needle); at != std::string::npos; at = bytes.find(needle, at + 8)) std::memcpy(bytes.data() + at, b, 8);
|
||||
}
|
||||
|
||||
// Remove volatile fields from decoded JSON, recursively
|
||||
void Strip(json& value) {
|
||||
if (value.is_object()) {
|
||||
for (auto it = value.begin(); it != value.end();) {
|
||||
if (CaptureTools::IsVolatileField(it.key())) it = value.erase(it);
|
||||
else {
|
||||
Strip(it.value());
|
||||
++it;
|
||||
}
|
||||
}
|
||||
} else if (value.is_array()) {
|
||||
for (auto& item : value) Strip(item);
|
||||
}
|
||||
}
|
||||
|
||||
std::string NameOf(const Record& record) {
|
||||
return PacketDecoder::Decode(record.bytes, CaptureTools::FromClient(record.header)).name;
|
||||
}
|
||||
}
|
||||
|
||||
namespace CaptureTools {
|
||||
void SortTimeline(std::vector<Record>& records) {
|
||||
std::stable_sort(records.begin(), records.end(), [](const Record& a, const Record& b) {
|
||||
if (a.header.timeUs != b.header.timeUs) return a.header.timeUs < b.header.timeUs;
|
||||
if (a.header.source != b.header.source) return a.header.source < b.header.source;
|
||||
return a.header.seq < b.header.seq;
|
||||
});
|
||||
}
|
||||
|
||||
bool FromClient(const PacketRecordHeader& h) {
|
||||
const auto source = static_cast<eCaptureSource>(h.source);
|
||||
return h.direction == static_cast<uint8_t>(ePacketDirection::RECEIVED) && !(h.flags & PacketRecordFlags::MASTER_LINK) &&
|
||||
(source == eCaptureSource::AUTH || source == eCaptureSource::WORLD);
|
||||
}
|
||||
|
||||
bool IsVolatileField(const std::string& name) {
|
||||
static const std::set<std::string> fields{
|
||||
// Made by the server each run
|
||||
"objectID", "objectId", "lootID", "lootOwnerID", "requestID", "i64LocalID", "uiSkillHandle", "uiBehaviorHandle",
|
||||
// Time and where things run
|
||||
"timestamp", "stamps", "instanceID", "instanceId", "zoneInstance", "cloneID", "zoneClone", "serverIP", "serverPort",
|
||||
"worldServerIP", "worldServerPort", "processID", "port",
|
||||
// Per account on each server
|
||||
"playerID", "targetID", "senderID", "username",
|
||||
};
|
||||
return fields.contains(name);
|
||||
}
|
||||
|
||||
json RecordJson(const Record& record, size_t index, int64_t startUs, bool fields) {
|
||||
const auto& h = record.header;
|
||||
json out{
|
||||
{"i", index},
|
||||
{"t", static_cast<double>(h.timeUs - startUs) / 1000.0},
|
||||
{"time", h.timeUs / 1000},
|
||||
{"source", SourceName(h.source)},
|
||||
{"seq", h.seq},
|
||||
{"zone", h.zoneId}, {"instance", h.instanceId}, {"clone", h.cloneId},
|
||||
{"account", h.accountId},
|
||||
{"character", std::to_string(h.characterId)},
|
||||
{"peer", PacketRecord::PeerText(h.peer)},
|
||||
{"bits", h.bits},
|
||||
{"bytes", record.bytes.size()},
|
||||
{"cut", (h.flags & PacketRecordFlags::CUT) != 0},
|
||||
};
|
||||
if (h.flags & PacketRecordFlags::GAP) {
|
||||
out["gap"] = h.bits;
|
||||
out["name"] = "(" + std::to_string(h.bits) + " packets lost)";
|
||||
return out;
|
||||
}
|
||||
const auto [from, to] = Ends(h);
|
||||
out["from"] = from;
|
||||
out["to"] = to;
|
||||
out["toServer"] = FromClient(h);
|
||||
const auto decoded = PacketDecoder::Decode(record.bytes, FromClient(h));
|
||||
out["service"] = decoded.service;
|
||||
out["name"] = decoded.name;
|
||||
if (decoded.gameMessageId >= 0) {
|
||||
out["gameMessage"] = decoded.gameMessageId;
|
||||
out["object"] = std::to_string(decoded.objectId);
|
||||
}
|
||||
if (decoded.failed) out["unreadable"] = true;
|
||||
if (fields && decoded.fields) out["fields"] = *decoded.fields;
|
||||
return out;
|
||||
}
|
||||
|
||||
std::vector<Track> Tracks(const std::vector<Record>& records, int64_t startUs) {
|
||||
std::vector<Track> tracks;
|
||||
for (const auto& record : records) {
|
||||
if (!FromClient(record.header) || record.header.source != static_cast<uint8_t>(eCaptureSource::WORLD)) continue;
|
||||
const auto position = PacketDecoder::Position(record.bytes);
|
||||
if (!position) continue;
|
||||
const auto& h = record.header;
|
||||
auto it = std::ranges::find_if(tracks, [&](const Track& t) { return t.characterId == h.characterId && t.zoneId == h.zoneId && t.instanceId == h.instanceId; });
|
||||
if (it == tracks.end()) {
|
||||
tracks.push_back({ h.characterId, h.zoneId, h.instanceId, {} });
|
||||
it = tracks.end() - 1;
|
||||
}
|
||||
it->samples.insert(it->samples.end(), { static_cast<float>(h.timeUs - startUs) / 1e6f, position->x, position->y, position->z });
|
||||
}
|
||||
return tracks;
|
||||
}
|
||||
|
||||
std::map<std::string, LWOOBJID> MakePortable(CaptureBundle::Bundle& bundle) {
|
||||
std::map<LWOOBJID, std::string> characters;
|
||||
std::map<uint32_t, uint32_t> accounts;
|
||||
for (const auto& record : bundle.records) {
|
||||
if (record.header.characterId && !characters.contains(record.header.characterId)) {
|
||||
characters[record.header.characterId] = "char#" + std::to_string(characters.size() + 1);
|
||||
}
|
||||
if (record.header.accountId && !accounts.contains(record.header.accountId)) accounts[record.header.accountId] = static_cast<uint32_t>(accounts.size() + 1);
|
||||
}
|
||||
json ids = json::object();
|
||||
std::map<std::string, LWOOBJID> found;
|
||||
std::map<LWOOBJID, LWOOBJID> placeholders;
|
||||
size_t n = 0;
|
||||
for (const auto& [id, symbol] : characters) {
|
||||
const auto placeholder = PLACEHOLDER_BASE + static_cast<int64_t>(++n);
|
||||
placeholders[id] = placeholder;
|
||||
ids[symbol] = { {"kind", "character"}, {"placeholder", std::to_string(placeholder)} };
|
||||
found[symbol] = id;
|
||||
}
|
||||
for (const auto& [id, number] : accounts) ids["account#" + std::to_string(number)] = { {"kind", "account"}, {"placeholder", number} };
|
||||
for (auto& record : bundle.records) {
|
||||
for (const auto& [id, placeholder] : placeholders) ReplaceAll(record.bytes, id, placeholder);
|
||||
if (record.header.characterId) record.header.characterId = placeholders[record.header.characterId];
|
||||
if (record.header.accountId) record.header.accountId = accounts[record.header.accountId];
|
||||
record.header.peer = 0;
|
||||
PacketDecoder::Scrub(record.bytes, false);
|
||||
record.header.length = static_cast<uint32_t>(record.bytes.size());
|
||||
}
|
||||
bundle.meta["ids"] = ids;
|
||||
bundle.meta["portable"] = true;
|
||||
return found;
|
||||
}
|
||||
|
||||
size_t Anonymise(CaptureBundle::Bundle& bundle) {
|
||||
size_t changed = 0;
|
||||
for (auto& record : bundle.records) {
|
||||
if (PacketDecoder::Scrub(record.bytes, true)) changed++;
|
||||
record.header.length = static_cast<uint32_t>(record.bytes.size());
|
||||
record.header.peer = 0;
|
||||
}
|
||||
bundle.meta["anonymised"] = true;
|
||||
return changed;
|
||||
}
|
||||
|
||||
json DiffReport::ToJson() const {
|
||||
return { {"expected", expected}, {"matched", matched}, {"differing", differing}, {"missing", missing}, {"extra", extra},
|
||||
{"differingByName", differingByName}, {"missingByName", missingByName}, {"extraByName", extraByName}, {"examples", examples} };
|
||||
}
|
||||
|
||||
DiffReport Diff(const std::vector<Record>& expected, const std::vector<Record>& actual) {
|
||||
DiffReport report;
|
||||
// Server -> client packets only (what the server answered)
|
||||
const auto answers = [](const std::vector<Record>& records) {
|
||||
std::vector<const Record*> out;
|
||||
for (const auto& r : records) {
|
||||
if (r.header.flags & (PacketRecordFlags::GAP | PacketRecordFlags::MASTER_LINK)) continue;
|
||||
const auto source = static_cast<eCaptureSource>(r.header.source);
|
||||
if ((source == eCaptureSource::AUTH || source == eCaptureSource::WORLD) && r.header.direction == static_cast<uint8_t>(ePacketDirection::SENT)) out.push_back(&r);
|
||||
}
|
||||
return out;
|
||||
};
|
||||
const auto want = answers(expected), got = answers(actual);
|
||||
report.expected = want.size();
|
||||
std::vector<bool> used(got.size());
|
||||
size_t from = 0;
|
||||
for (const auto* w : want) {
|
||||
const auto name = NameOf(*w);
|
||||
// The next unused answer with the same name, looking a little ahead so one missing packet doesn't shift everything
|
||||
size_t found = got.size();
|
||||
for (size_t i = from; i < got.size() && i < from + 200; i++) {
|
||||
if (!used[i] && NameOf(*got[i]) == name) {
|
||||
found = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (found == got.size()) {
|
||||
report.missing++;
|
||||
report.missingByName[name]++;
|
||||
continue;
|
||||
}
|
||||
used[found] = true;
|
||||
while (from < used.size() && used[from]) from++;
|
||||
auto a = PacketDecoder::Decode(w->bytes, false).fields.value_or(json());
|
||||
auto b = PacketDecoder::Decode(got[found]->bytes, false).fields.value_or(json());
|
||||
Strip(a);
|
||||
Strip(b);
|
||||
// Packets without decoded fields compare by size
|
||||
const bool same = a.is_null() && b.is_null() ? w->bytes.size() == got[found]->bytes.size() : a == b;
|
||||
if (same) {
|
||||
report.matched++;
|
||||
} else {
|
||||
report.differing++;
|
||||
report.differingByName[name]++;
|
||||
if (report.examples.size() < 20) {
|
||||
report.examples.push_back(name + ": recorded " + (a.is_null() ? std::to_string(w->bytes.size()) + " bytes" : a.dump()).substr(0, 300) +
|
||||
" / replayed " + (b.is_null() ? std::to_string(got[found]->bytes.size()) + " bytes" : b.dump()).substr(0, 300));
|
||||
}
|
||||
}
|
||||
}
|
||||
for (size_t i = 0; i < got.size(); i++) {
|
||||
if (used[i]) continue;
|
||||
report.extra++;
|
||||
report.extraByName[NameOf(*got[i])]++;
|
||||
}
|
||||
return report;
|
||||
}
|
||||
}
|
||||
74
dNet/CaptureTools.h
Normal file
74
dNet/CaptureTools.h
Normal file
@@ -0,0 +1,74 @@
|
||||
#ifndef __CAPTURETOOLS__H__
|
||||
#define __CAPTURETOOLS__H__
|
||||
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "CaptureBundle.h"
|
||||
#include "dCommonVars.h"
|
||||
#include "json.hpp"
|
||||
|
||||
/**
|
||||
* What the dashboard's capture viewer and the capture tool do with recorded packets (docs/CaptureReplay.md):
|
||||
* describe them, order them on one timeline, pull the player's movement out, make a bundle portable or anonymous,
|
||||
* and compare a replay's answers with the recorded ones. Pure functions over records, so they are unit tested.
|
||||
*/
|
||||
namespace CaptureTools {
|
||||
// The records of all servers on one timeline: by time, then by server and its sequence
|
||||
void SortTimeline(std::vector<CaptureBundle::Record>& records);
|
||||
|
||||
// Whether a record went from a game client to a server
|
||||
bool FromClient(const PacketRecordHeader& header);
|
||||
|
||||
// One record for the viewer: where it went, its name and, when `fields`, its decoded fields
|
||||
nlohmann::json RecordJson(const CaptureBundle::Record& record, size_t index, int64_t startUs, bool fields);
|
||||
|
||||
struct Track {
|
||||
LWOOBJID characterId{};
|
||||
uint32_t zoneId{};
|
||||
uint32_t instanceId{};
|
||||
std::vector<float> samples; // t (seconds from the capture's start), x, y, z, ...
|
||||
};
|
||||
// Where each captured character moved (their POSITION_UPDATEs), per zone and instance
|
||||
std::vector<Track> Tracks(const std::vector<CaptureBundle::Record>& records, int64_t startUs);
|
||||
|
||||
/**
|
||||
* Makes a bundle portable: the source server's character and account IDs are replaced by placeholders
|
||||
* (PLACEHOLDER_BASE + n, written in the records' bytes and headers), listed in meta.ids as "char#n" / "account#n";
|
||||
* account names and session fields are already blank (they are never recorded). Returns the characters found, by
|
||||
* symbol, with their source ID, for the setup section.
|
||||
*/
|
||||
constexpr int64_t PLACEHOLDER_BASE = 0x1FEDC00000000000LL;
|
||||
std::map<std::string, LWOOBJID> MakePortable(CaptureBundle::Bundle& bundle);
|
||||
|
||||
// Blanks what players typed and names (chat text, character and account names) in every packet whose struct is
|
||||
// known, so a bundle can be kept as a test fixture. Returns how many packets were changed.
|
||||
size_t Anonymise(CaptureBundle::Bundle& bundle);
|
||||
|
||||
/**
|
||||
* A replay's answers against the recorded ones. Server->client packets are paired in order by name; paired
|
||||
* packets are compared by their decoded fields, leaving out what legitimately differs between runs (object IDs
|
||||
* the server makes, timestamps, session keys, instance and clone IDs, server addresses).
|
||||
*/
|
||||
struct DiffReport {
|
||||
size_t expected{};
|
||||
size_t matched{}; // same fields
|
||||
size_t differing{}; // same packet, different fields
|
||||
size_t missing{}; // recorded, not answered in the replay
|
||||
size_t extra{}; // answered in the replay, not recorded
|
||||
std::map<std::string, size_t> differingByName;
|
||||
std::map<std::string, size_t> missingByName;
|
||||
std::map<std::string, size_t> extraByName;
|
||||
std::vector<std::string> examples; // the first few differences, readable
|
||||
nlohmann::json ToJson() const;
|
||||
};
|
||||
DiffReport Diff(const std::vector<CaptureBundle::Record>& expected, const std::vector<CaptureBundle::Record>& actual);
|
||||
|
||||
// Fields left out of comparisons (by name, in any packet)
|
||||
bool IsVolatileField(const std::string& name);
|
||||
}
|
||||
|
||||
#endif //!__CAPTURETOOLS__H__
|
||||
613
dNet/PacketCapture.cpp
Normal file
613
dNet/PacketCapture.cpp
Normal file
@@ -0,0 +1,613 @@
|
||||
#include "PacketCapture.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <deque>
|
||||
#include <map>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "BitStream.h"
|
||||
#include "BitStreamUtils.h"
|
||||
#include "dConfig.h"
|
||||
#include "Game.h"
|
||||
#include "GeneralUtils.h"
|
||||
#include "Logger.h"
|
||||
#include "MasterPackets.h"
|
||||
#include "PacketDecoder.h"
|
||||
#include "master/InstanceMigration.h"
|
||||
#include "master/MessageCapture.h"
|
||||
#include "MessageIdentifiers.h"
|
||||
#include "MessageType/Master.h"
|
||||
#include "MessageType/Auth.h"
|
||||
#include "MessageType/Client.h"
|
||||
#include "RakPeer.h"
|
||||
#include "RakPeerInterface.h"
|
||||
#include "ServiceType.h"
|
||||
#include "dServer.h"
|
||||
|
||||
// Servers always have a logger; tests may not
|
||||
#define CAPTURE_LOG(...) do { if (Game::logger) LOG(__VA_ARGS__); } while (0)
|
||||
|
||||
namespace {
|
||||
using Clock = std::chrono::steady_clock;
|
||||
using Master = MessageType::Master;
|
||||
|
||||
// Packets kept per connection before it is known whose it is (a login, a world's session check), for account
|
||||
// captures only: when the account turns out to be captured they are added in front
|
||||
constexpr size_t PENDING_RECORDS = 32;
|
||||
constexpr size_t PENDING_BYTES = 64 * 1024;
|
||||
constexpr auto PENDING_FOR = std::chrono::seconds(60);
|
||||
// Zone transfer requests remembered to tie their answer to the player
|
||||
constexpr size_t MAX_REQUESTS = 256;
|
||||
|
||||
struct Slot {
|
||||
bool armed{};
|
||||
uint32_t captureId{};
|
||||
eCaptureTarget target{};
|
||||
uint32_t accountId{};
|
||||
std::string accountName;
|
||||
std::vector<LWOOBJID> characterIds;
|
||||
Clock::time_point until{};
|
||||
};
|
||||
|
||||
struct Binding {
|
||||
uint32_t accountId{};
|
||||
std::string accountName;
|
||||
LWOOBJID characterId{};
|
||||
uint8_t mask{}; // account and character slots this connection belongs to
|
||||
};
|
||||
|
||||
struct Pending {
|
||||
std::string records;
|
||||
size_t count{};
|
||||
Clock::time_point since{};
|
||||
};
|
||||
|
||||
struct Chunk {
|
||||
std::string records;
|
||||
std::array<uint32_t, MessageCapture::MAX_SLOTS> slots{};
|
||||
uint32_t count{};
|
||||
};
|
||||
|
||||
ServiceType g_ServerType{};
|
||||
eCaptureSource g_Source{};
|
||||
RakPeerInterface* g_Peer{};
|
||||
RakPeerInterface* g_MasterLink{};
|
||||
uint16_t g_Zone{}, g_Instance{};
|
||||
uint32_t g_Clone{};
|
||||
std::vector<uint64_t> g_Ignored;
|
||||
|
||||
std::array<Slot, MessageCapture::MAX_SLOTS> g_Slots;
|
||||
uint8_t g_EverythingMask{}; // slots capturing everything
|
||||
uint8_t g_SubjectMask{}; // slots capturing an account or a character
|
||||
uint8_t g_AccountMask{}; // slots capturing an account
|
||||
|
||||
std::unordered_map<uint64_t, Binding> g_Bindings;
|
||||
std::vector<uint64_t> g_Unbind; // connections that closed; forgotten at the next receive
|
||||
std::unordered_map<uint64_t, Pending> g_Pending;
|
||||
std::map<uint64_t, uint8_t> g_Requests; // zone transfer request -> mask
|
||||
uint64_t g_Scope{}; // connection whose packet is being handled
|
||||
|
||||
PacketCapture::Settings g_Settings;
|
||||
Chunk g_Chunk;
|
||||
std::deque<Chunk> g_Sealed;
|
||||
uint64_t g_SealedBytes{};
|
||||
uint32_t g_Dropped{}; // since the last batch
|
||||
uint32_t g_Seq{};
|
||||
Clock::time_point g_ChunkStarted{};
|
||||
PacketCapture::Sink g_Sink;
|
||||
PacketCapture::Stats g_Stats;
|
||||
|
||||
uint64_t Key(const SystemAddress& address) {
|
||||
return (static_cast<uint64_t>(address.binaryAddress) << 16) | address.port;
|
||||
}
|
||||
|
||||
int64_t NowUs() {
|
||||
return std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
|
||||
}
|
||||
|
||||
bool SameName(const std::string& a, const std::string& b) {
|
||||
return !a.empty() && GeneralUtils::CaseInsensitiveStringCompare(a, b);
|
||||
}
|
||||
|
||||
// The account and character slots a connection's owner belongs to
|
||||
uint8_t MaskFor(uint32_t accountId, const std::string& accountName, LWOOBJID characterId) {
|
||||
uint8_t mask = 0;
|
||||
for (uint8_t i = 0; i < g_Slots.size(); i++) {
|
||||
const auto& slot = g_Slots[i];
|
||||
if (!slot.armed || slot.target == eCaptureTarget::EVERYTHING) continue;
|
||||
const bool character = characterId != 0 && std::ranges::find(slot.characterIds, characterId) != slot.characterIds.end();
|
||||
const bool account = (accountId != 0 && accountId == slot.accountId) || SameName(accountName, slot.accountName);
|
||||
if (slot.target == eCaptureTarget::CHARACTER ? character : (account || character)) mask |= 1 << i;
|
||||
}
|
||||
return mask;
|
||||
}
|
||||
|
||||
void Remask() {
|
||||
g_EverythingMask = g_SubjectMask = g_AccountMask = 0;
|
||||
for (uint8_t i = 0; i < g_Slots.size(); i++) {
|
||||
const auto& slot = g_Slots[i];
|
||||
if (!slot.armed) continue;
|
||||
if (slot.target == eCaptureTarget::EVERYTHING) g_EverythingMask |= 1 << i;
|
||||
else g_SubjectMask |= 1 << i;
|
||||
if (slot.target == eCaptureTarget::ACCOUNT) g_AccountMask |= 1 << i;
|
||||
}
|
||||
for (auto& [key, binding] : g_Bindings) binding.mask = MaskFor(binding.accountId, binding.accountName, binding.characterId);
|
||||
}
|
||||
|
||||
void Seal() {
|
||||
if (g_Chunk.count == 0) return;
|
||||
for (size_t i = 0; i < g_Slots.size(); i++) g_Chunk.slots[i] = g_Slots[i].armed ? g_Slots[i].captureId : 0;
|
||||
g_SealedBytes += g_Chunk.records.size();
|
||||
g_Sealed.push_back(std::move(g_Chunk));
|
||||
g_Chunk = Chunk{};
|
||||
g_Chunk.records.reserve(g_Settings.flushBytes + 4096);
|
||||
g_ChunkStarted = Clock::now();
|
||||
// Over the cap: the oldest go first, and the dashboard is told how many
|
||||
while (g_SealedBytes > g_Settings.maxBufferBytes && g_Sealed.size() > 1) {
|
||||
g_SealedBytes -= g_Sealed.front().records.size();
|
||||
g_Dropped += g_Sealed.front().count;
|
||||
g_Stats.dropped += g_Sealed.front().count;
|
||||
g_Sealed.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
void Append(const PacketRecordHeader& header, const unsigned char* data) {
|
||||
PacketRecord::Append(g_Chunk.records, header, data);
|
||||
g_Chunk.count++;
|
||||
g_Stats.recorded++;
|
||||
g_Stats.recordedBytes += sizeof(header) + header.length;
|
||||
if (g_Chunk.records.size() >= g_Settings.flushBytes) Seal();
|
||||
}
|
||||
|
||||
PacketRecordHeader Header(ePacketDirection direction, uint8_t flags, uint64_t peer, uint32_t bits) {
|
||||
PacketRecordHeader header;
|
||||
header.timeUs = NowUs();
|
||||
header.seq = ++g_Seq;
|
||||
header.source = static_cast<uint8_t>(g_Source);
|
||||
header.direction = static_cast<uint8_t>(direction);
|
||||
header.flags = flags;
|
||||
header.peer = peer;
|
||||
header.zoneId = g_Zone;
|
||||
header.instanceId = g_Instance;
|
||||
header.cloneId = g_Clone;
|
||||
header.bits = bits;
|
||||
const auto bytes = (bits + 7) / 8;
|
||||
header.length = std::min<uint32_t>(bytes, PacketRecord::MAX_BYTES);
|
||||
if (header.length < bytes) header.flags |= PacketRecordFlags::CUT;
|
||||
return header;
|
||||
}
|
||||
|
||||
// An LU packet's service and message ID, if it is one
|
||||
bool LuHeader(const unsigned char* data, uint32_t bits, ServiceType& service, uint32_t& id) {
|
||||
if (bits < 64 || data[0] != ID_USER_PACKET_ENUM) return false;
|
||||
uint16_t rawService;
|
||||
std::memcpy(&rawService, data + 1, sizeof(rawService));
|
||||
std::memcpy(&id, data + 3, sizeof(id));
|
||||
service = static_cast<ServiceType>(rawService);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Never record the capture's own traffic
|
||||
bool IsCaptureTraffic(const unsigned char* data, uint32_t bits) {
|
||||
ServiceType service;
|
||||
uint32_t id;
|
||||
return LuHeader(data, bits, service, id) && service == ServiceType::MASTER &&
|
||||
(id == static_cast<uint32_t>(Master::MESSAGE_CAPTURE_CONTROL) || id == static_cast<uint32_t>(Master::MESSAGE_CAPTURE_DATA));
|
||||
}
|
||||
|
||||
// Chat: every chat packet between worlds and chat starts with the player's object ID
|
||||
LWOOBJID ChatSubject(const unsigned char* data, uint32_t bits) {
|
||||
ServiceType service;
|
||||
uint32_t id;
|
||||
if (!LuHeader(data, bits, service, id) || bits < 128) return 0;
|
||||
LWOOBJID subject;
|
||||
std::memcpy(&subject, data + 8, sizeof(subject));
|
||||
return subject;
|
||||
}
|
||||
|
||||
/**
|
||||
* Secrets never reach a record (PacketDecoder::Redact): packets that carry them are rewritten with them blanked,
|
||||
* into `scratch`, and dropped if they don't read. Auth keeps only the handshake and the login request and
|
||||
* response (both redacted). False: don't record it.
|
||||
*/
|
||||
bool Prepare(const unsigned char*& data, uint32_t& bits, std::string& scratch) {
|
||||
ServiceType service;
|
||||
uint32_t id;
|
||||
if (!LuHeader(data, bits, service, id)) return true;
|
||||
if (g_Source == eCaptureSource::AUTH && service != ServiceType::COMMON &&
|
||||
!(service == ServiceType::AUTH && id == static_cast<uint32_t>(MessageType::Auth::LOGIN_REQUEST)) &&
|
||||
!(service == ServiceType::CLIENT && id == static_cast<uint32_t>(MessageType::Client::LOGIN_RESPONSE))) return false;
|
||||
if (!PacketDecoder::HasSecrets(service, id)) return true;
|
||||
scratch.assign(reinterpret_cast<const char*>(data), (bits + 7) / 8);
|
||||
if (!PacketDecoder::Redact(scratch)) return false;
|
||||
data = reinterpret_cast<const unsigned char*>(scratch.data());
|
||||
bits = static_cast<uint32_t>(scratch.size() * 8);
|
||||
return true;
|
||||
}
|
||||
|
||||
// A packet on the listening peer: who it belongs to, and so which captures keep it
|
||||
void RecordMain(const SystemAddress& address, ePacketDirection direction, bool broadcast, const unsigned char* data, uint32_t bits) {
|
||||
if (bits < 8 || IsCaptureTraffic(data, bits)) return;
|
||||
std::string scratch;
|
||||
if (!Prepare(data, bits, scratch)) return;
|
||||
const auto key = Key(address);
|
||||
if (!g_Ignored.empty() && std::ranges::find(g_Ignored, key) != g_Ignored.end()) return;
|
||||
|
||||
// Master: server-to-server traffic belongs to EVERYTHING captures only (the other servers record what belongs to a player)
|
||||
if (g_Source == eCaptureSource::MASTER) {
|
||||
if (!g_EverythingMask) return;
|
||||
auto header = Header(direction, broadcast ? PacketRecordFlags::BROADCAST : 0, key, bits);
|
||||
header.mask = g_EverythingMask;
|
||||
Append(header, data);
|
||||
return;
|
||||
}
|
||||
|
||||
uint8_t mask = g_EverythingMask;
|
||||
uint32_t accountId = 0;
|
||||
LWOOBJID characterId = 0;
|
||||
if (g_SubjectMask) {
|
||||
if (g_Source == eCaptureSource::CHAT) {
|
||||
characterId = ChatSubject(data, bits);
|
||||
if (characterId) mask |= MaskFor(0, "", characterId);
|
||||
} else if (broadcast) {
|
||||
// Everyone but `address`: the captured players it reaches
|
||||
for (const auto& [bound, binding] : g_Bindings) {
|
||||
if (binding.mask && bound != key) {
|
||||
mask |= binding.mask;
|
||||
accountId = binding.accountId;
|
||||
characterId = binding.characterId;
|
||||
}
|
||||
}
|
||||
} else if (const auto it = g_Bindings.find(key); it != g_Bindings.end()) {
|
||||
mask |= it->second.mask;
|
||||
accountId = it->second.accountId;
|
||||
characterId = it->second.characterId;
|
||||
} else if (g_AccountMask && g_Source != eCaptureSource::UNKNOWN) {
|
||||
// Not known yet whose it is: keep a few until a login says
|
||||
auto& pending = g_Pending[key];
|
||||
if (pending.count == 0) pending.since = Clock::now();
|
||||
auto header = Header(direction, 0, key, bits);
|
||||
if (pending.count < PENDING_RECORDS && pending.records.size() + sizeof(header) + header.length <= PENDING_BYTES) {
|
||||
PacketRecord::Append(pending.records, header, data);
|
||||
pending.count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!mask) return;
|
||||
auto header = Header(direction, broadcast ? PacketRecordFlags::BROADCAST : 0, key, bits);
|
||||
header.mask = mask;
|
||||
header.accountId = accountId;
|
||||
header.characterId = characterId;
|
||||
Append(header, data);
|
||||
}
|
||||
|
||||
// A master link message: which captured players it belongs to
|
||||
uint8_t MasterLinkMask(ePacketDirection direction, const unsigned char* data, uint32_t bits, uint32_t& accountId, LWOOBJID& characterId) {
|
||||
ServiceType service;
|
||||
uint32_t id;
|
||||
if (!LuHeader(data, bits, service, id) || service != ServiceType::MASTER) return 0;
|
||||
RakNet::BitStream stream(const_cast<unsigned char*>(data), (bits + 7) / 8, false);
|
||||
LUBitStream header;
|
||||
if (!header.ReadHeader(stream)) return 0;
|
||||
|
||||
const auto scope = g_Bindings.find(g_Scope);
|
||||
const auto* scoped = g_Scope && scope != g_Bindings.end() && scope->second.mask ? &scope->second : nullptr;
|
||||
const auto byName = [&](const std::string& name) { return MaskFor(0, name, 0); };
|
||||
uint8_t everyone = 0;
|
||||
for (const auto& [key, binding] : g_Bindings) everyone |= binding.mask;
|
||||
|
||||
switch (static_cast<Master>(id)) {
|
||||
case Master::MESSAGE_CAPTURE_CONTROL:
|
||||
case Master::MESSAGE_CAPTURE_DATA:
|
||||
case Master::PLAYER_POSITIONS:
|
||||
case Master::DATA_CHANGED:
|
||||
case Master::PLAYER_ACTION_RESULT:
|
||||
return 0;
|
||||
case Master::REQUEST_SESSION_KEY: {
|
||||
MasterPackets::RequestSessionKey msg;
|
||||
return msg.Deserialize(stream) ? byName(msg.username.GetAsString()) : 0;
|
||||
}
|
||||
case Master::SESSION_KEY_RESPONSE: {
|
||||
MasterPackets::SessionKeyResponse msg;
|
||||
return msg.Deserialize(stream) ? byName(msg.username.GetAsString()) : 0;
|
||||
}
|
||||
case Master::SET_SESSION_KEY: {
|
||||
MasterPackets::SetSessionKey msg;
|
||||
return msg.Deserialize(stream) ? byName(msg.username.string) : 0;
|
||||
}
|
||||
case Master::NEW_SESSION_ALERT: {
|
||||
MasterPackets::NewSessionAlert msg;
|
||||
return msg.Deserialize(stream) ? byName(msg.username.string) : 0;
|
||||
}
|
||||
case Master::REQUEST_ZONE_TRANSFER: {
|
||||
MasterPackets::RequestZoneTransfer msg;
|
||||
if (!scoped || !msg.Deserialize(stream)) return 0;
|
||||
if (g_Requests.size() >= MAX_REQUESTS) g_Requests.erase(g_Requests.begin());
|
||||
g_Requests[msg.requestID] = scoped->mask;
|
||||
accountId = scoped->accountId;
|
||||
characterId = scoped->characterId;
|
||||
return scoped->mask;
|
||||
}
|
||||
case Master::REQUEST_ZONE_TRANSFER_RESPONSE: {
|
||||
MasterPackets::RequestZoneTransferResponse msg;
|
||||
if (!msg.Deserialize(stream)) return 0;
|
||||
const auto it = g_Requests.find(msg.requestID);
|
||||
if (it == g_Requests.end()) return 0;
|
||||
const auto mask = it->second & (g_SubjectMask);
|
||||
g_Requests.erase(it);
|
||||
return mask;
|
||||
}
|
||||
// Instance-wide: they concern every captured player in this world
|
||||
case Master::INSTANCE_MIGRATE:
|
||||
case Master::MIGRATE_PLAYERS:
|
||||
case Master::MIGRATE_STATUS:
|
||||
case Master::MIGRATE_PLAYER_STATE:
|
||||
case Master::SHUTDOWN:
|
||||
case Master::AFFIRM_TRANSFER_REQUEST:
|
||||
case Master::AFFIRM_TRANSFER_RESPONSE:
|
||||
return everyone;
|
||||
default:
|
||||
// Anything else sent while handling a captured player's packet (player added or removed, ...)
|
||||
if (direction == ePacketDirection::SENT && scoped) {
|
||||
accountId = scoped->accountId;
|
||||
characterId = scoped->characterId;
|
||||
return scoped->mask;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
void RecordMasterLink(ePacketDirection direction, const unsigned char* data, uint32_t bits) {
|
||||
if (!g_SubjectMask || bits < 64) return;
|
||||
uint32_t accountId = 0;
|
||||
LWOOBJID characterId = 0;
|
||||
const auto mask = MasterLinkMask(direction, data, bits, accountId, characterId);
|
||||
if (!mask) return;
|
||||
std::string scratch;
|
||||
if (!Prepare(data, bits, scratch)) return;
|
||||
auto header = Header(direction, PacketRecordFlags::MASTER_LINK, 0, bits);
|
||||
header.mask = mask;
|
||||
header.accountId = accountId;
|
||||
header.characterId = characterId;
|
||||
Append(header, data);
|
||||
}
|
||||
|
||||
void OnSend(RakPeerInterface* peer, const unsigned char* data, BitSize_t bits, SystemAddress address, bool broadcast) {
|
||||
if (!PacketCapture::g_Armed || !data) return;
|
||||
if (peer == g_Peer) RecordMain(address, ePacketDirection::SENT, broadcast, data, static_cast<uint32_t>(bits));
|
||||
else if (peer == g_MasterLink && g_MasterLink) RecordMasterLink(ePacketDirection::SENT, data, static_cast<uint32_t>(bits));
|
||||
}
|
||||
|
||||
void SetArmed(bool armed) {
|
||||
PacketCapture::g_Armed = armed;
|
||||
g_RakPeerSendHook = armed ? &OnSend : nullptr;
|
||||
if (!armed) {
|
||||
g_Pending.clear();
|
||||
g_Requests.clear();
|
||||
}
|
||||
}
|
||||
|
||||
void ReadSettings() {
|
||||
if (!Game::config) return;
|
||||
PacketCapture::Settings settings;
|
||||
settings.flushIntervalMs = std::clamp<uint32_t>(GeneralUtils::TryParse<uint32_t>(Game::config->GetValue("capture_flush_interval_ms")).value_or(1000), 50, 60000);
|
||||
settings.flushBytes = std::clamp<uint32_t>(GeneralUtils::TryParse<uint32_t>(Game::config->GetValue("capture_flush_bytes")).value_or(256 * 1024), 4096, 4 * 1024 * 1024);
|
||||
settings.maxBufferBytes = std::clamp<uint64_t>(GeneralUtils::TryParse<uint64_t>(Game::config->GetValue("capture_buffer_max_mb")).value_or(16), 1, 1024) * 1024 * 1024;
|
||||
PacketCapture::SetSettings(settings);
|
||||
}
|
||||
|
||||
bool SendToMaster(MessageCaptureData& data) {
|
||||
if (!Game::server || !Game::server->GetIsConnectedToMaster()) return false;
|
||||
MasterPackets::SendToMaster(data);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
namespace PacketCapture {
|
||||
bool g_Armed = false;
|
||||
bool g_Tracking = false;
|
||||
|
||||
void Attach(ServiceType serverType, RakPeerInterface* peer, RakPeerInterface* masterLink, uint32_t zoneId, uint32_t instanceId) {
|
||||
g_ServerType = serverType;
|
||||
switch (serverType) {
|
||||
case ServiceType::AUTH: g_Source = eCaptureSource::AUTH; break;
|
||||
case ServiceType::CHAT: g_Source = eCaptureSource::CHAT; break;
|
||||
case ServiceType::WORLD: g_Source = eCaptureSource::WORLD; break;
|
||||
case ServiceType::MASTER: g_Source = eCaptureSource::MASTER; break;
|
||||
default: g_Source = eCaptureSource::UNKNOWN; break;
|
||||
}
|
||||
g_Peer = peer;
|
||||
g_MasterLink = serverType == ServiceType::MASTER ? nullptr : masterLink;
|
||||
g_Zone = static_cast<uint16_t>(zoneId);
|
||||
g_Instance = static_cast<uint16_t>(instanceId);
|
||||
}
|
||||
|
||||
void Detach() {
|
||||
Reset();
|
||||
g_Peer = g_MasterLink = nullptr;
|
||||
}
|
||||
|
||||
void SetClone(uint32_t cloneId) { g_Clone = cloneId; }
|
||||
|
||||
void IgnorePeer(const SystemAddress& address) {
|
||||
const auto key = Key(address);
|
||||
if (std::ranges::find(g_Ignored, key) == g_Ignored.end()) g_Ignored.push_back(key);
|
||||
}
|
||||
|
||||
void SetSink(Sink sink) { g_Sink = std::move(sink); }
|
||||
|
||||
void SetSettings(const Settings& settings) {
|
||||
g_Settings = settings;
|
||||
if (g_Chunk.records.capacity() < g_Settings.flushBytes) g_Chunk.records.reserve(g_Settings.flushBytes + 4096);
|
||||
}
|
||||
|
||||
const Stats& GetStats() { return g_Stats; }
|
||||
|
||||
void Control(const MessageCaptureControl& control) {
|
||||
if (control.action != eMessageCaptureControl::ARM && control.action != eMessageCaptureControl::DISARM) return;
|
||||
if (control.slot >= g_Slots.size()) return;
|
||||
auto& slot = g_Slots[control.slot];
|
||||
const bool sameCapture = slot.armed && slot.captureId == control.captureId;
|
||||
if (control.action == eMessageCaptureControl::DISARM) {
|
||||
if (!sameCapture) return;
|
||||
Seal();
|
||||
slot = Slot{};
|
||||
CAPTURE_LOG("Packet capture %u ended here", control.captureId);
|
||||
} else {
|
||||
if (control.seconds == 0) return;
|
||||
// The slot changes hands: what was recorded so far goes out under the old capture
|
||||
if (!sameCapture) Seal();
|
||||
if (!sameCapture) CAPTURE_LOG("Packet capture %u armed here (%s) for %u seconds", control.captureId,
|
||||
control.target == eCaptureTarget::EVERYTHING ? "everything" : control.target == eCaptureTarget::ACCOUNT ? "an account" : "a character", control.seconds);
|
||||
slot.armed = true;
|
||||
slot.captureId = control.captureId;
|
||||
slot.target = control.target;
|
||||
slot.accountId = control.accountId;
|
||||
slot.accountName = control.accountName;
|
||||
slot.characterIds = control.characterIds;
|
||||
slot.until = Clock::now() + std::chrono::seconds(std::min(control.seconds, MessageCapture::MAX_SECONDS));
|
||||
ReadSettings();
|
||||
if (g_ChunkStarted == Clock::time_point{}) g_ChunkStarted = Clock::now();
|
||||
}
|
||||
Remask();
|
||||
// Stays armed while chunks wait to be sent, so Update keeps sending them
|
||||
SetArmed(std::ranges::any_of(g_Slots, &Slot::armed) || !g_Sealed.empty() || g_Chunk.count);
|
||||
}
|
||||
|
||||
void OnReceiveImpl(const Packet* packet) {
|
||||
// Closed connections are forgotten once their disconnect was handled (a world still names the player then)
|
||||
if (!g_Unbind.empty()) {
|
||||
for (const auto key : g_Unbind) {
|
||||
g_Bindings.erase(key);
|
||||
g_Pending.erase(key);
|
||||
}
|
||||
g_Unbind.clear();
|
||||
g_Tracking = !g_Bindings.empty();
|
||||
}
|
||||
if (!packet) {
|
||||
g_Scope = 0;
|
||||
return;
|
||||
}
|
||||
g_Scope = Key(packet->systemAddress);
|
||||
if (packet->length >= 1 && (packet->data[0] == ID_DISCONNECTION_NOTIFICATION || packet->data[0] == ID_CONNECTION_LOST)) g_Unbind.push_back(g_Scope);
|
||||
if (!g_Armed) return;
|
||||
RecordMain(packet->systemAddress, ePacketDirection::RECEIVED, false, packet->data, static_cast<uint32_t>(packet->bitSize));
|
||||
}
|
||||
|
||||
void OnReceiveFromMasterImpl(const Packet* packet) {
|
||||
if (!packet || !g_MasterLink) return;
|
||||
RecordMasterLink(ePacketDirection::RECEIVED, packet->data, static_cast<uint32_t>(packet->bitSize));
|
||||
}
|
||||
|
||||
void Bind(const SystemAddress& address, uint32_t accountId, const std::string& accountName) {
|
||||
const auto key = Key(address);
|
||||
auto& binding = g_Bindings[key];
|
||||
g_Tracking = true;
|
||||
binding.accountId = accountId;
|
||||
binding.accountName = accountName;
|
||||
binding.mask = MaskFor(accountId, accountName, binding.characterId);
|
||||
|
||||
// What came before the login, now that it is known to be a captured account's
|
||||
const auto pending = g_Pending.find(key);
|
||||
if (pending == g_Pending.end()) return;
|
||||
const auto mask = binding.mask & g_AccountMask;
|
||||
if (mask) {
|
||||
PacketRecord::ForEach(pending->second.records, [&](PacketRecordHeader header, std::string_view bytes) {
|
||||
header.mask = mask;
|
||||
header.accountId = accountId;
|
||||
Append(header, reinterpret_cast<const unsigned char*>(bytes.data()));
|
||||
});
|
||||
}
|
||||
g_Pending.erase(pending);
|
||||
}
|
||||
|
||||
void BindCharacter(const SystemAddress& address, LWOOBJID characterId) {
|
||||
auto& binding = g_Bindings[Key(address)];
|
||||
g_Tracking = true;
|
||||
binding.characterId = characterId;
|
||||
binding.mask = MaskFor(binding.accountId, binding.accountName, characterId);
|
||||
}
|
||||
|
||||
void Update() {
|
||||
if (!g_Armed) return;
|
||||
const auto now = Clock::now();
|
||||
|
||||
// Captures past their time end here on their own
|
||||
bool changed = false;
|
||||
for (auto& slot : g_Slots) {
|
||||
if (slot.armed && now >= slot.until) {
|
||||
Seal();
|
||||
CAPTURE_LOG("Packet capture %u reached its time limit here", slot.captureId);
|
||||
slot = Slot{};
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
if (changed) Remask();
|
||||
|
||||
std::erase_if(g_Pending, [now](const auto& entry) { return now - entry.second.since > PENDING_FOR; });
|
||||
|
||||
if (g_Chunk.count && (g_Chunk.records.size() >= g_Settings.flushBytes || now - g_ChunkStarted >= std::chrono::milliseconds(g_Settings.flushIntervalMs))) Seal();
|
||||
if (g_Chunk.count == 0) g_ChunkStarted = now;
|
||||
|
||||
while (!g_Sealed.empty() || g_Dropped) {
|
||||
MessageCaptureData data;
|
||||
data.status = eMessageCaptureStatus::PACKETS;
|
||||
data.source = static_cast<uint8_t>(g_Source);
|
||||
data.zoneId = g_Zone;
|
||||
data.instanceId = g_Instance;
|
||||
data.cloneId = g_Clone;
|
||||
data.packetsDropped = g_Dropped;
|
||||
if (!g_Sealed.empty()) {
|
||||
auto& chunk = g_Sealed.front();
|
||||
data.slots = chunk.slots;
|
||||
data.packetCount = chunk.count;
|
||||
data.packets = std::move(chunk.records);
|
||||
} else {
|
||||
for (size_t i = 0; i < g_Slots.size(); i++) data.slots[i] = g_Slots[i].armed ? g_Slots[i].captureId : 0;
|
||||
}
|
||||
const bool sent = g_Sink ? g_Sink(data) : SendToMaster(data);
|
||||
if (!sent) {
|
||||
// Put it back and try again next time
|
||||
if (!g_Sealed.empty()) g_Sealed.front().records = std::move(data.packets);
|
||||
break;
|
||||
}
|
||||
g_Stats.batches++;
|
||||
g_Stats.sentBytes += data.packets.size();
|
||||
g_Dropped = 0;
|
||||
if (!g_Sealed.empty()) {
|
||||
g_SealedBytes -= std::min<uint64_t>(g_SealedBytes, data.packets.size());
|
||||
g_Sealed.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
if (!std::ranges::any_of(g_Slots, &Slot::armed) && g_Sealed.empty() && g_Chunk.count == 0) SetArmed(false);
|
||||
}
|
||||
|
||||
void Reset() {
|
||||
g_Slots = {};
|
||||
g_Bindings.clear();
|
||||
g_Tracking = false;
|
||||
g_Unbind.clear();
|
||||
g_Pending.clear();
|
||||
g_Requests.clear();
|
||||
g_Ignored.clear();
|
||||
g_Scope = 0;
|
||||
g_Chunk = Chunk{};
|
||||
g_Sealed.clear();
|
||||
g_SealedBytes = 0;
|
||||
g_Dropped = 0;
|
||||
g_Seq = 0;
|
||||
g_Stats = {};
|
||||
g_Sink = nullptr;
|
||||
g_Settings = {};
|
||||
Remask();
|
||||
SetArmed(false);
|
||||
}
|
||||
|
||||
void RecordForTest(const SystemAddress& address, bool sent, bool broadcast, const unsigned char* data, uint32_t bits) {
|
||||
if (!g_Armed) return;
|
||||
RecordMain(address, sent ? ePacketDirection::SENT : ePacketDirection::RECEIVED, broadcast, data, bits);
|
||||
}
|
||||
}
|
||||
104
dNet/PacketCapture.h
Normal file
104
dNet/PacketCapture.h
Normal file
@@ -0,0 +1,104 @@
|
||||
#ifndef __PACKETCAPTURE__H__
|
||||
#define __PACKETCAPTURE__H__
|
||||
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <string>
|
||||
|
||||
#include "dCommonVars.h"
|
||||
#include "PacketRecord.h"
|
||||
|
||||
class RakPeerInterface;
|
||||
struct MessageCaptureControl;
|
||||
struct MessageCaptureData;
|
||||
struct Packet;
|
||||
struct SystemAddress;
|
||||
enum class ServiceType : uint16_t;
|
||||
|
||||
/**
|
||||
* This server's side of the dashboard's packet capture (docs/CaptureReplay.md): records whole packets, as they went
|
||||
* over RakNet, for up to MessageCapture::MAX_SLOTS armed captures at once, and ships them to the dashboard in batches.
|
||||
*
|
||||
* Every server runs one (auth, chat, world and master): the dashboard arms a capture with MESSAGE_CAPTURE_CONTROL
|
||||
* ARM, master passes it to every server, and each records its part:
|
||||
* - auth and world: what their clients send and receive (world: everything, replica constructions included, since
|
||||
* the tap sits in RakPeer::Send), plus their own master link messages that belong to a captured player
|
||||
* (session keys, zone transfers, player added/removed, instance migration);
|
||||
* - chat: what worlds route to and from it (the player is the object ID each chat packet starts with);
|
||||
* - master: for an EVERYTHING capture, all server-to-server traffic except the dashboard's.
|
||||
*
|
||||
* Nothing is written to disk here and nothing is sent per packet: records are appended to one preallocated chunk,
|
||||
* which is sealed when it reaches capture_flush_bytes or capture_flush_interval_ms has passed, and sent from the
|
||||
* main loop (Update). Sealed chunks waiting to be sent (master unreachable) are capped at capture_buffer_max_mb; the
|
||||
* oldest are dropped past that and counted, so the dashboard shows a gap. With nothing armed the cost is one flag
|
||||
* check per received packet and one null check per sent one.
|
||||
*
|
||||
* RakNet isn't thread safe: everything here runs on the server's main thread (sends from the main thread, receives
|
||||
* where the main loop reads them).
|
||||
*/
|
||||
namespace PacketCapture {
|
||||
extern bool g_Armed;
|
||||
extern bool g_Tracking; // connections are bound (they are forgotten when they close, armed or not)
|
||||
|
||||
// dServer: which peer is the listening one and which the link to master (nullptr on master itself)
|
||||
void Attach(ServiceType serverType, RakPeerInterface* peer, RakPeerInterface* masterLink, uint32_t zoneId, uint32_t instanceId);
|
||||
void Detach();
|
||||
|
||||
// Worlds: their clone (a property's owner), once known
|
||||
void SetClone(uint32_t cloneId);
|
||||
|
||||
// Master: packets to and from the dashboard aren't recorded
|
||||
void IgnorePeer(const SystemAddress& address);
|
||||
|
||||
// ARM or DISARM from the dashboard (START/STOP are the game message inspector's)
|
||||
void Control(const MessageCaptureControl& control);
|
||||
|
||||
// A packet the listening peer received / nullptr when there are no more this frame (dServer::Receive)
|
||||
void OnReceiveImpl(const Packet* packet);
|
||||
inline void OnReceive(const Packet* packet) { if (g_Armed || g_Tracking) OnReceiveImpl(packet); }
|
||||
|
||||
// A packet from master (dServer::ReceiveFromMaster)
|
||||
void OnReceiveFromMasterImpl(const Packet* packet);
|
||||
inline void OnReceiveFromMaster(const Packet* packet) { if (g_Armed) OnReceiveFromMasterImpl(packet); }
|
||||
|
||||
// Who is on the other end of a connection: auth when an account logs in (the account may not exist: 0), worlds
|
||||
// when a session is validated, and the character when one is picked
|
||||
void Bind(const SystemAddress& address, uint32_t accountId, const std::string& accountName);
|
||||
void BindCharacter(const SystemAddress& address, LWOOBJID characterId);
|
||||
|
||||
// Main loop: seals the chunk when it is due, sends sealed chunks, ends captures past their time
|
||||
void Update();
|
||||
|
||||
/**
|
||||
* Where sealed chunks go; returns false when they can't be sent right now (they are kept, up to the cap). The
|
||||
* default sends them to master; master sends them straight to the dashboard.
|
||||
*/
|
||||
using Sink = std::function<bool(MessageCaptureData& data)>;
|
||||
void SetSink(Sink sink);
|
||||
|
||||
struct Settings {
|
||||
uint32_t flushIntervalMs{ 1000 };
|
||||
uint32_t flushBytes{ 256 * 1024 };
|
||||
uint64_t maxBufferBytes{ 16ull * 1024 * 1024 };
|
||||
};
|
||||
// Read from capture_flush_interval_ms, capture_flush_bytes and capture_buffer_max_mb when a capture is armed
|
||||
void SetSettings(const Settings& settings);
|
||||
|
||||
struct Stats {
|
||||
uint64_t recorded{}; // records kept
|
||||
uint64_t recordedBytes{}; // their size, headers included
|
||||
uint64_t dropped{}; // records dropped over the buffer cap
|
||||
uint64_t batches{}; // chunks sent
|
||||
uint64_t sentBytes{};
|
||||
};
|
||||
const Stats& GetStats();
|
||||
|
||||
// Tests: drop every capture, binding and buffer (the attachment stays)
|
||||
void Reset();
|
||||
|
||||
// Tests and tools: record as if the listening peer had sent (broadcast: to all but `address`) or received this
|
||||
void RecordForTest(const SystemAddress& address, bool sent, bool broadcast, const unsigned char* data, uint32_t bits);
|
||||
}
|
||||
|
||||
#endif //!__PACKETCAPTURE__H__
|
||||
419
dNet/PacketDecoder.cpp
Normal file
419
dNet/PacketDecoder.cpp
Normal file
@@ -0,0 +1,419 @@
|
||||
#include "PacketDecoder.h"
|
||||
|
||||
#include <cstring>
|
||||
#include <map>
|
||||
#include <utility>
|
||||
|
||||
#include "AuthPackets.h"
|
||||
#include "BitStreamUtils.h"
|
||||
#include "ChatPackets.h"
|
||||
#include "ClientPackets.h"
|
||||
#include "CommonPackets.h"
|
||||
#include "MasterPackets.h"
|
||||
#include "MessageIdentifiers.h"
|
||||
#include "MessageType/Auth.h"
|
||||
#include "MessageType/Chat.h"
|
||||
#include "MessageType/Client.h"
|
||||
#include "MessageType/Master.h"
|
||||
#include "MessageType/Server.h"
|
||||
#include "MessageType/World.h"
|
||||
#include "ServiceType.h"
|
||||
#include "WorldPackets.h"
|
||||
#include "WorldRoutePacket.h"
|
||||
#include "magic_enum.hpp"
|
||||
|
||||
namespace {
|
||||
using json = nlohmann::json;
|
||||
using Fields = std::function<std::optional<json>(RakNet::BitStream&, bool fromClient)>;
|
||||
using Redactor = std::function<bool(std::string&)>;
|
||||
|
||||
using Scrubber = std::function<bool(std::string&, bool anonymise)>;
|
||||
|
||||
struct Entry {
|
||||
Fields fields;
|
||||
Redactor redact; // set for structs with secret fields
|
||||
Scrubber scrub; // set for structs with account names, character names or typed text
|
||||
};
|
||||
|
||||
void X(std::u16string& text) { for (auto& c : text) c = u'x'; }
|
||||
void X(std::string& text) { for (auto& c : text) c = 'x'; }
|
||||
void X(LUWString& text) { X(text.string); }
|
||||
void X(LUString& text) { X(text.string); }
|
||||
|
||||
// Like Blank, for Scrub: `names(packet)` replaces account names, `anonymous(packet)` the rest
|
||||
template<typename T>
|
||||
Scrubber Scrub(std::function<void(T&)> names, std::function<void(T&)> anonymous = nullptr) {
|
||||
return [names, anonymous](std::string& bytes, bool anonymise) {
|
||||
RakNet::BitStream in(reinterpret_cast<unsigned char*>(bytes.data()), static_cast<unsigned int>(bytes.size()), false);
|
||||
T packet;
|
||||
if (!packet.ReadHeader(in) || !packet.Deserialize(in)) return false;
|
||||
if (names) names(packet);
|
||||
if (anonymise && anonymous) anonymous(packet);
|
||||
RakNet::BitStream out;
|
||||
packet.WritePacket(out);
|
||||
std::string written(reinterpret_cast<const char*>(out.GetData()), out.GetNumberOfBytesUsed());
|
||||
if (written == bytes) return false;
|
||||
bytes = std::move(written);
|
||||
return true;
|
||||
};
|
||||
}
|
||||
|
||||
PacketDecoder::GameMessageFields g_GameMessages;
|
||||
|
||||
using Rewriter = std::function<std::optional<std::string>(std::string_view)>;
|
||||
template<typename T>
|
||||
Rewriter Rewrite() {
|
||||
return [](std::string_view bytes) -> std::optional<std::string> {
|
||||
RakNet::BitStream in(reinterpret_cast<unsigned char*>(const_cast<char*>(bytes.data())), static_cast<unsigned int>(bytes.size()), false);
|
||||
T packet;
|
||||
if (!packet.ReadHeader(in) || !packet.Deserialize(in)) return std::nullopt;
|
||||
RakNet::BitStream out;
|
||||
packet.WritePacket(out);
|
||||
return std::string(reinterpret_cast<const char*>(out.GetData()), out.GetNumberOfBytesUsed());
|
||||
};
|
||||
}
|
||||
|
||||
std::string Id(LWOOBJID id) { return std::to_string(id); }
|
||||
json Point(const NiPoint3& p) { return json::array({ p.x, p.y, p.z }); }
|
||||
json Rotation(const NiQuaternion& q) { return json::array({ q.x, q.y, q.z, q.w }); }
|
||||
|
||||
template<typename T>
|
||||
Fields Make(std::function<void(const T&, json&)> fill) {
|
||||
return [fill](RakNet::BitStream& stream, bool) -> std::optional<json> {
|
||||
T packet;
|
||||
if (!packet.Deserialize(stream)) return std::nullopt;
|
||||
json out = json::object();
|
||||
fill(packet, out);
|
||||
return out;
|
||||
};
|
||||
}
|
||||
|
||||
// Reads T from a whole packet, lets `blank` clear its secrets, and writes it back in place
|
||||
template<typename T>
|
||||
Redactor Blank(std::function<void(T&)> blank) {
|
||||
return [blank](std::string& bytes) {
|
||||
RakNet::BitStream in(reinterpret_cast<unsigned char*>(bytes.data()), static_cast<unsigned int>(bytes.size()), false);
|
||||
T packet;
|
||||
if (!packet.ReadHeader(in) || !packet.Deserialize(in)) return false;
|
||||
blank(packet);
|
||||
RakNet::BitStream out;
|
||||
packet.WritePacket(out);
|
||||
bytes.assign(reinterpret_cast<const char*>(out.GetData()), out.GetNumberOfBytesUsed());
|
||||
return true;
|
||||
};
|
||||
}
|
||||
|
||||
using Key = std::pair<ServiceType, uint32_t>;
|
||||
template<typename E> Key K(ServiceType service, E id) { return { service, static_cast<uint32_t>(id) }; }
|
||||
|
||||
const std::map<Key, Entry>& Registry() {
|
||||
using S = ServiceType;
|
||||
static const std::map<Key, Entry> registry{
|
||||
// Handshake: one ID, a struct per direction
|
||||
{ K(S::COMMON, MessageType::Server::VERSION_CONFIRM), { [](RakNet::BitStream& stream, bool fromClient) -> std::optional<json> {
|
||||
if (fromClient) {
|
||||
CommonPackets::ClientVersionConfirm packet;
|
||||
if (!packet.Deserialize(stream)) return std::nullopt;
|
||||
return json{ {"netVersion", packet.netVersion}, {"serviceType", static_cast<int>(packet.serviceType)}, {"processID", packet.processID}, {"port", packet.port} };
|
||||
}
|
||||
CommonPackets::ServerVersionConfirm packet;
|
||||
if (!packet.Deserialize(stream)) return std::nullopt;
|
||||
return json{ {"netVersion", packet.netVersion}, {"serviceType", static_cast<int>(packet.serviceType)} };
|
||||
} } },
|
||||
|
||||
// Auth: the username and password are blanked when recorded
|
||||
{ K(S::AUTH, MessageType::Auth::LOGIN_REQUEST), {
|
||||
Make<AuthPackets::LoginRequest>([](const auto& p, json& j) {
|
||||
j = { {"username", p.username.GetAsString()}, {"localeID", static_cast<int>(p.localeID)}, {"clientOS", static_cast<int>(p.clientOS)},
|
||||
{"memoryStats", p.memoryStats.GetAsString()}, {"videoCard", p.videoCard.GetAsString()} };
|
||||
}),
|
||||
Blank<AuthPackets::LoginRequest>([](auto& p) { p.username.string.clear(); p.password.string.clear(); }),
|
||||
Scrub<AuthPackets::LoginRequest>([](auto& p) { X(p.username); }) } },
|
||||
{ K(S::CLIENT, MessageType::Client::LOGIN_RESPONSE), {
|
||||
Make<ClientPackets::LoginResponse>([](const auto& p, json& j) {
|
||||
json stamps = json::array();
|
||||
for (const auto& stamp : p.stamps.list) stamps.push_back({ {"type", static_cast<int>(stamp.type)}, {"value", stamp.value}, {"timestamp", stamp.timestamp} });
|
||||
j = { {"responseCode", static_cast<int>(p.responseCode)}, {"worldServerIP", p.worldServerIP.string}, {"worldServerPort", p.worldServerPort},
|
||||
{"errorMessage", p.errorMessage}, {"stamps", stamps} };
|
||||
}),
|
||||
Blank<ClientPackets::LoginResponse>([](auto& p) { p.userKey.string.clear(); p.cdnKey.string.clear(); }) } },
|
||||
|
||||
// World
|
||||
{ K(S::WORLD, MessageType::World::VALIDATION), {
|
||||
Make<WorldPackets::Validation>([](const auto& p, json& j) { j = { {"username", p.username.GetAsString()}, {"fdbChecksum", p.fdbChecksum.string} }; }),
|
||||
Blank<WorldPackets::Validation>([](auto& p) { p.sessionKey.string.clear(); }),
|
||||
Scrub<WorldPackets::Validation>([](auto& p) { X(p.username); }) } },
|
||||
{ K(S::WORLD, MessageType::World::CHARACTER_CREATE_REQUEST), { Make<WorldPackets::CharacterCreateRequest>([](const auto& p, json& j) {
|
||||
j = { {"name", p.name.GetAsString()}, {"firstNameIndex", p.firstNameIndex}, {"middleNameIndex", p.middleNameIndex}, {"lastNameIndex", p.lastNameIndex},
|
||||
{"shirtColor", p.shirtColor}, {"shirtStyle", p.shirtStyle}, {"pantsColor", p.pantsColor}, {"hairStyle", p.hairStyle}, {"hairColor", p.hairColor},
|
||||
{"eyebrows", p.eyebrows}, {"eyes", p.eyes}, {"mouth", p.mouth} };
|
||||
}),
|
||||
nullptr, Scrub<WorldPackets::CharacterCreateRequest>(nullptr, [](auto& p) { X(p.name); }) } },
|
||||
{ K(S::WORLD, MessageType::World::LOGIN_REQUEST), { Make<WorldPackets::CharacterLoginRequest>([](const auto& p, json& j) { j = { {"playerID", Id(p.playerID)} }; }) } },
|
||||
{ K(S::WORLD, MessageType::World::CHARACTER_DELETE_REQUEST), { Make<WorldPackets::CharacterDeleteRequest>([](const auto& p, json& j) { j = { {"objectID", Id(p.objectID)} }; }) } },
|
||||
{ K(S::WORLD, MessageType::World::CHARACTER_RENAME_REQUEST), { Make<WorldPackets::CharacterRenameRequest>([](const auto& p, json& j) {
|
||||
j = { {"objectID", Id(p.objectID)}, {"name", p.name.GetAsString()} };
|
||||
}),
|
||||
nullptr, Scrub<WorldPackets::CharacterRenameRequest>(nullptr, [](auto& p) { X(p.name); }) } },
|
||||
{ K(S::WORLD, MessageType::World::LEVEL_LOAD_COMPLETE), { Make<WorldPackets::LevelLoadComplete>([](const auto& p, json& j) {
|
||||
j = { {"mapID", p.mapID}, {"instanceID", p.instanceID}, {"cloneID", p.cloneID} };
|
||||
}) } },
|
||||
{ K(S::WORLD, MessageType::World::POSITION_UPDATE), { Make<WorldPackets::PositionUpdate>([](const auto& p, json& j) {
|
||||
j = { {"position", Point(p.update.position)}, {"rotation", Rotation(p.update.rotation)}, {"onGround", p.update.onGround}, {"onRail", p.update.onRail} };
|
||||
if (p.hasVelocity) j["velocity"] = Point(p.update.velocity);
|
||||
if (p.hasLocalSpaceInfo) j["platform"] = Id(p.update.localSpaceInfo.objectId);
|
||||
}) } },
|
||||
{ K(S::WORLD, MessageType::World::GENERAL_CHAT_MESSAGE), { Make<WorldPackets::GeneralChatMessage>([](const auto& p, json& j) {
|
||||
j = { {"chatChannel", p.chatChannel}, {"message", GeneralUtils::UTF16ToWTF8(p.message)} };
|
||||
}),
|
||||
nullptr, Scrub<WorldPackets::GeneralChatMessage>(nullptr, [](auto& p) { X(p.message); }) } },
|
||||
{ K(S::WORLD, MessageType::World::ROUTE_PACKET), { Make<WorldPackets::RoutePacket>([](const auto& p, json& j) {
|
||||
j = { {"routed", PacketDecoder::Name(p.routedService, p.routedMessageID)}, {"size", p.size} };
|
||||
}) } },
|
||||
|
||||
// To the client
|
||||
{ K(S::CLIENT, MessageType::Client::LOAD_STATIC_ZONE), { Make<ClientPackets::LoadStaticZone>([](const auto& p, json& j) {
|
||||
j = { {"mapID", p.mapID}, {"instanceID", p.instanceID}, {"cloneID", p.cloneID}, {"mapChecksum", p.mapChecksum}, {"playerPosition", Point(p.playerPosition)},
|
||||
{"instanceType", p.instanceType} };
|
||||
}) } },
|
||||
{ K(S::CLIENT, MessageType::Client::CHARACTER_LIST_RESPONSE), { Make<ClientPackets::CharacterListResponse>([](const auto& p, json& j) {
|
||||
json characters = json::array();
|
||||
for (const auto& c : p.characters) {
|
||||
characters.push_back({ {"objectID", Id(c.objectID)}, {"name", c.name.GetAsString()}, {"zoneID", c.zoneID}, {"equippedItems", c.equippedItems} });
|
||||
}
|
||||
j = { {"selectedCharacterIndex", p.selectedCharacterIndex}, {"characters", characters} };
|
||||
}),
|
||||
nullptr, Scrub<ClientPackets::CharacterListResponse>(nullptr, [](auto& p) { for (auto& c : p.characters) { X(c.name); X(c.unapprovedName); } }) } },
|
||||
{ K(S::CLIENT, MessageType::Client::CHARACTER_CREATE_RESPONSE), { Make<ClientPackets::CharacterCreateResponse>([](const auto& p, json& j) {
|
||||
j = { {"response", static_cast<int>(p.response)} };
|
||||
}) } },
|
||||
{ K(S::CLIENT, MessageType::Client::TRANSFER_TO_WORLD), { Make<ClientPackets::TransferToWorld>([](const auto& p, json& j) {
|
||||
j = { {"serverIP", p.serverIP.string}, {"serverPort", p.serverPort}, {"mythranShift", p.mythranShift} };
|
||||
}) } },
|
||||
|
||||
// Chat
|
||||
{ K(S::CHAT, MessageType::Chat::GENERAL_CHAT_MESSAGE), { Make<ChatPackets::GeneralChatMessage>([](const auto& p, json& j) {
|
||||
j = { {"playerID", Id(p.playerID)}, {"chatChannel", static_cast<int>(p.chatChannel)}, {"message", p.message.GetAsString()} };
|
||||
}),
|
||||
nullptr, Scrub<ChatPackets::GeneralChatMessage>(nullptr, [](auto& p) { X(p.senderName); X(p.message); }) } },
|
||||
{ K(S::CHAT, MessageType::Chat::PRIVATE_CHAT_MESSAGE), { Make<ChatPackets::PrivateChatMessage>([](const auto& p, json& j) {
|
||||
j = { {"playerID", Id(p.playerID)}, {"senderName", p.senderName.GetAsString()}, {"receiverName", p.receiverName.GetAsString()},
|
||||
{"responseCode", p.responseCode}, {"message", p.message.GetAsString()} };
|
||||
}),
|
||||
nullptr, Scrub<ChatPackets::PrivateChatMessage>(nullptr, [](auto& p) { X(p.senderName); X(p.receiverName); X(p.message); }) } },
|
||||
{ K(S::CHAT, MessageType::Chat::WORLD_ROUTE_PACKET), { Make<ChatPackets::WorldRoutePacket>([](const auto& p, json& j) {
|
||||
j = { {"targetID", Id(p.targetID)}, {"bytes", p.routedData.size()} };
|
||||
if (p.routedData.size() >= 8 && p.routedData[0] == ID_USER_PACKET_ENUM) {
|
||||
uint16_t service;
|
||||
uint32_t id;
|
||||
std::memcpy(&service, p.routedData.data() + 1, sizeof(service));
|
||||
std::memcpy(&id, p.routedData.data() + 3, sizeof(id));
|
||||
j["routed"] = PacketDecoder::Name(static_cast<ServiceType>(service), id);
|
||||
}
|
||||
}) } },
|
||||
|
||||
// Between servers: session keys are blanked when recorded
|
||||
{ K(S::MASTER, MessageType::Master::REQUEST_SESSION_KEY), { Make<MasterPackets::RequestSessionKey>([](const auto& p, json& j) { j = { {"username", p.username.GetAsString()} }; }),
|
||||
nullptr, Scrub<MasterPackets::RequestSessionKey>([](auto& p) { X(p.username); }) } },
|
||||
{ K(S::MASTER, MessageType::Master::SESSION_KEY_RESPONSE), {
|
||||
Make<MasterPackets::SessionKeyResponse>([](const auto& p, json& j) { j = { {"username", p.username.GetAsString()} }; }),
|
||||
Blank<MasterPackets::SessionKeyResponse>([](auto& p) { p.sessionKey = 0; }),
|
||||
Scrub<MasterPackets::SessionKeyResponse>([](auto& p) { X(p.username); }) } },
|
||||
{ K(S::MASTER, MessageType::Master::SET_SESSION_KEY), {
|
||||
Make<MasterPackets::SetSessionKey>([](const auto& p, json& j) { j = { {"username", p.username.string} }; }),
|
||||
Blank<MasterPackets::SetSessionKey>([](auto& p) { p.sessionKey = 0; }),
|
||||
Scrub<MasterPackets::SetSessionKey>([](auto& p) { X(p.username); }) } },
|
||||
{ K(S::MASTER, MessageType::Master::NEW_SESSION_ALERT), {
|
||||
Make<MasterPackets::NewSessionAlert>([](const auto& p, json& j) { j = { {"username", p.username.string} }; }),
|
||||
Blank<MasterPackets::NewSessionAlert>([](auto& p) { p.sessionKey = 0; }),
|
||||
Scrub<MasterPackets::NewSessionAlert>([](auto& p) { X(p.username); }) } },
|
||||
{ K(S::MASTER, MessageType::Master::REQUEST_ZONE_TRANSFER), { Make<MasterPackets::RequestZoneTransfer>([](const auto& p, json& j) {
|
||||
j = { {"requestID", Id(p.requestID)}, {"zoneID", p.zoneID}, {"cloneID", p.cloneID}, {"mythranShift", p.mythranShift}, {"stamps", p.stamps.size()} };
|
||||
}) } },
|
||||
{ K(S::MASTER, MessageType::Master::REQUEST_ZONE_TRANSFER_RESPONSE), { Make<MasterPackets::RequestZoneTransferResponse>([](const auto& p, json& j) {
|
||||
j = { {"requestID", Id(p.requestID)}, {"zoneID", p.zoneID}, {"zoneInstance", p.zoneInstance}, {"zoneClone", p.zoneClone}, {"serverPort", p.serverPort},
|
||||
{"stamps", p.stamps.size()} };
|
||||
}) } },
|
||||
{ K(S::MASTER, MessageType::Master::PLAYER_ADDED), { Make<MasterPackets::PlayerAdded>([](const auto& p, json& j) { j = { {"zoneID", p.zoneID}, {"instanceID", p.instanceID} }; }) } },
|
||||
{ K(S::MASTER, MessageType::Master::PLAYER_REMOVED), { Make<MasterPackets::PlayerRemoved>([](const auto& p, json& j) { j = { {"zoneID", p.zoneID}, {"instanceID", p.instanceID} }; }) } },
|
||||
};
|
||||
return registry;
|
||||
}
|
||||
|
||||
const std::map<Key, Rewriter>& Rewriters() {
|
||||
using S = ServiceType;
|
||||
static const std::map<Key, Rewriter> rewriters{
|
||||
{ K(S::AUTH, MessageType::Auth::LOGIN_REQUEST), Rewrite<AuthPackets::LoginRequest>() },
|
||||
{ K(S::CLIENT, MessageType::Client::LOGIN_RESPONSE), Rewrite<ClientPackets::LoginResponse>() },
|
||||
{ K(S::WORLD, MessageType::World::VALIDATION), Rewrite<WorldPackets::Validation>() },
|
||||
{ K(S::WORLD, MessageType::World::CHARACTER_CREATE_REQUEST), Rewrite<WorldPackets::CharacterCreateRequest>() },
|
||||
{ K(S::WORLD, MessageType::World::LOGIN_REQUEST), Rewrite<WorldPackets::CharacterLoginRequest>() },
|
||||
{ K(S::WORLD, MessageType::World::CHARACTER_DELETE_REQUEST), Rewrite<WorldPackets::CharacterDeleteRequest>() },
|
||||
{ K(S::WORLD, MessageType::World::CHARACTER_RENAME_REQUEST), Rewrite<WorldPackets::CharacterRenameRequest>() },
|
||||
{ K(S::WORLD, MessageType::World::LEVEL_LOAD_COMPLETE), Rewrite<WorldPackets::LevelLoadComplete>() },
|
||||
{ K(S::WORLD, MessageType::World::POSITION_UPDATE), Rewrite<WorldPackets::PositionUpdate>() },
|
||||
{ K(S::WORLD, MessageType::World::GENERAL_CHAT_MESSAGE), Rewrite<WorldPackets::GeneralChatMessage>() },
|
||||
{ K(S::WORLD, MessageType::World::ROUTE_PACKET), Rewrite<WorldPackets::RoutePacket>() },
|
||||
{ K(S::CLIENT, MessageType::Client::LOAD_STATIC_ZONE), Rewrite<ClientPackets::LoadStaticZone>() },
|
||||
{ K(S::CLIENT, MessageType::Client::CHARACTER_LIST_RESPONSE), Rewrite<ClientPackets::CharacterListResponse>() },
|
||||
{ K(S::CLIENT, MessageType::Client::CHARACTER_CREATE_RESPONSE), Rewrite<ClientPackets::CharacterCreateResponse>() },
|
||||
{ K(S::CLIENT, MessageType::Client::TRANSFER_TO_WORLD), Rewrite<ClientPackets::TransferToWorld>() },
|
||||
{ K(S::CHAT, MessageType::Chat::GENERAL_CHAT_MESSAGE), Rewrite<ChatPackets::GeneralChatMessage>() },
|
||||
{ K(S::CHAT, MessageType::Chat::PRIVATE_CHAT_MESSAGE), Rewrite<ChatPackets::PrivateChatMessage>() },
|
||||
{ K(S::CHAT, MessageType::Chat::WORLD_ROUTE_PACKET), Rewrite<ChatPackets::WorldRoutePacket>() },
|
||||
{ K(S::MASTER, MessageType::Master::REQUEST_SESSION_KEY), Rewrite<MasterPackets::RequestSessionKey>() },
|
||||
{ K(S::MASTER, MessageType::Master::SESSION_KEY_RESPONSE), Rewrite<MasterPackets::SessionKeyResponse>() },
|
||||
{ K(S::MASTER, MessageType::Master::SET_SESSION_KEY), Rewrite<MasterPackets::SetSessionKey>() },
|
||||
{ K(S::MASTER, MessageType::Master::NEW_SESSION_ALERT), Rewrite<MasterPackets::NewSessionAlert>() },
|
||||
{ K(S::MASTER, MessageType::Master::REQUEST_ZONE_TRANSFER), Rewrite<MasterPackets::RequestZoneTransfer>() },
|
||||
{ K(S::MASTER, MessageType::Master::REQUEST_ZONE_TRANSFER_RESPONSE), Rewrite<MasterPackets::RequestZoneTransferResponse>() },
|
||||
{ K(S::MASTER, MessageType::Master::PLAYER_ADDED), Rewrite<MasterPackets::PlayerAdded>() },
|
||||
{ K(S::MASTER, MessageType::Master::PLAYER_REMOVED), Rewrite<MasterPackets::PlayerRemoved>() },
|
||||
};
|
||||
return rewriters;
|
||||
}
|
||||
|
||||
template<typename E>
|
||||
std::string EnumName(uint32_t id) {
|
||||
const auto name = magic_enum::enum_name(static_cast<E>(id));
|
||||
return name.empty() ? std::to_string(id) : std::string(name);
|
||||
}
|
||||
|
||||
std::string RakNetName(uint8_t id) {
|
||||
switch (id) {
|
||||
case ID_CONNECTION_REQUEST_ACCEPTED: return "ID_CONNECTION_REQUEST_ACCEPTED";
|
||||
case ID_NEW_INCOMING_CONNECTION: return "ID_NEW_INCOMING_CONNECTION";
|
||||
case ID_DISCONNECTION_NOTIFICATION: return "ID_DISCONNECTION_NOTIFICATION";
|
||||
case ID_CONNECTION_LOST: return "ID_CONNECTION_LOST";
|
||||
case ID_TIMESTAMP: return "ID_TIMESTAMP";
|
||||
case ID_REPLICA_MANAGER_CONSTRUCTION: return "ID_REPLICA_MANAGER_CONSTRUCTION";
|
||||
case ID_REPLICA_MANAGER_SCOPE_CHANGE: return "ID_REPLICA_MANAGER_SCOPE_CHANGE";
|
||||
case ID_REPLICA_MANAGER_SERIALIZE: return "ID_REPLICA_MANAGER_SERIALIZE";
|
||||
case ID_REPLICA_MANAGER_DESTRUCTION: return "ID_REPLICA_MANAGER_DESTRUCTION";
|
||||
case ID_REPLICA_MANAGER_DOWNLOAD_STARTED: return "ID_REPLICA_MANAGER_DOWNLOAD_STARTED";
|
||||
case ID_REPLICA_MANAGER_DOWNLOAD_COMPLETE: return "ID_REPLICA_MANAGER_DOWNLOAD_COMPLETE";
|
||||
default: return "RAKNET_" + std::to_string(id);
|
||||
}
|
||||
}
|
||||
|
||||
bool ReadLuHeader(std::string_view bytes, ServiceType& service, uint32_t& id) {
|
||||
if (bytes.size() < 8 || static_cast<uint8_t>(bytes[0]) != ID_USER_PACKET_ENUM) return false;
|
||||
uint16_t raw;
|
||||
std::memcpy(&raw, bytes.data() + 1, sizeof(raw));
|
||||
std::memcpy(&id, bytes.data() + 3, sizeof(id));
|
||||
service = static_cast<ServiceType>(raw);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
namespace PacketDecoder {
|
||||
std::string Name(ServiceType service, uint32_t messageId) {
|
||||
switch (service) {
|
||||
case ServiceType::COMMON: return EnumName<MessageType::Server>(messageId);
|
||||
case ServiceType::AUTH: return EnumName<MessageType::Auth>(messageId);
|
||||
case ServiceType::CHAT: return EnumName<MessageType::Chat>(messageId);
|
||||
case ServiceType::WORLD: return EnumName<MessageType::World>(messageId);
|
||||
case ServiceType::CLIENT: return EnumName<MessageType::Client>(messageId);
|
||||
case ServiceType::MASTER: return EnumName<MessageType::Master>(messageId);
|
||||
default: return std::to_string(messageId);
|
||||
}
|
||||
}
|
||||
|
||||
Decoded Decode(std::string_view bytes, bool fromClient) {
|
||||
Decoded out;
|
||||
if (bytes.empty()) return out;
|
||||
ServiceType service{};
|
||||
uint32_t id{};
|
||||
if (!ReadLuHeader(bytes, service, id)) {
|
||||
out.service = "RAKNET";
|
||||
out.messageId = static_cast<uint8_t>(bytes[0]);
|
||||
out.name = RakNetName(static_cast<uint8_t>(bytes[0]));
|
||||
return out;
|
||||
}
|
||||
out.lu = true;
|
||||
out.serviceId = static_cast<uint16_t>(service);
|
||||
out.messageId = id;
|
||||
out.service = std::string(magic_enum::enum_name(service));
|
||||
if (out.service.empty()) out.service = std::to_string(out.serviceId);
|
||||
out.name = Name(service, id);
|
||||
|
||||
RakNet::BitStream stream(reinterpret_cast<unsigned char*>(const_cast<char*>(bytes.data())), static_cast<unsigned int>(bytes.size()), false);
|
||||
stream.IgnoreBytes(8);
|
||||
|
||||
// Game messages: the object, the message ID, then its fields
|
||||
const bool gameMessage = (service == ServiceType::WORLD && id == static_cast<uint32_t>(MessageType::World::GAME_MSG)) ||
|
||||
(service == ServiceType::CLIENT && id == static_cast<uint32_t>(MessageType::Client::GAME_MSG));
|
||||
if (gameMessage) {
|
||||
uint16_t messageId{};
|
||||
if (!stream.Read(out.objectId) || !stream.Read(messageId)) {
|
||||
out.failed = true;
|
||||
return out;
|
||||
}
|
||||
out.gameMessageId = messageId;
|
||||
const auto name = magic_enum::enum_name(static_cast<MessageType::Game>(messageId));
|
||||
out.name += " " + (name.empty() ? std::to_string(messageId) : std::string(name));
|
||||
if (g_GameMessages) {
|
||||
// What follows the header, as a stream of its own
|
||||
const auto offset = stream.GetReadOffset() / 8;
|
||||
RakNet::BitStream payload(reinterpret_cast<unsigned char*>(const_cast<char*>(bytes.data())) + offset, static_cast<unsigned int>(bytes.size() - offset), false);
|
||||
out.fields = g_GameMessages(static_cast<MessageType::Game>(messageId), service == ServiceType::WORLD, payload);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const auto it = Registry().find({ service, id });
|
||||
if (it == Registry().end()) return out;
|
||||
out.fields = it->second.fields(stream, fromClient);
|
||||
out.failed = !out.fields.has_value();
|
||||
return out;
|
||||
}
|
||||
|
||||
std::optional<NiPoint3> Position(std::string_view bytes) {
|
||||
ServiceType service{};
|
||||
uint32_t id{};
|
||||
if (!ReadLuHeader(bytes, service, id) || service != ServiceType::WORLD || id != static_cast<uint32_t>(MessageType::World::POSITION_UPDATE)) return std::nullopt;
|
||||
RakNet::BitStream stream(reinterpret_cast<unsigned char*>(const_cast<char*>(bytes.data())), static_cast<unsigned int>(bytes.size()), false);
|
||||
stream.IgnoreBytes(8);
|
||||
WorldPackets::PositionUpdate update;
|
||||
if (!update.Deserialize(stream)) return std::nullopt;
|
||||
return update.update.position;
|
||||
}
|
||||
|
||||
void SetGameMessageDecoder(GameMessageFields decoder) { g_GameMessages = std::move(decoder); }
|
||||
|
||||
bool Redact(std::string& bytes) {
|
||||
ServiceType service{};
|
||||
uint32_t id{};
|
||||
if (!ReadLuHeader(bytes, service, id)) return true;
|
||||
const auto it = Registry().find({ service, id });
|
||||
if (it == Registry().end() || !it->second.redact) return true;
|
||||
return it->second.redact(bytes);
|
||||
}
|
||||
|
||||
bool Scrub(std::string& bytes, bool anonymise) {
|
||||
ServiceType service{};
|
||||
uint32_t id{};
|
||||
if (!ReadLuHeader(bytes, service, id)) return false;
|
||||
const auto it = Registry().find({ service, id });
|
||||
if (it == Registry().end() || !it->second.scrub) return false;
|
||||
return it->second.scrub(bytes, anonymise);
|
||||
}
|
||||
|
||||
std::optional<bool> RoundTrip(std::string_view bytes) {
|
||||
ServiceType service{};
|
||||
uint32_t id{};
|
||||
if (!ReadLuHeader(bytes, service, id)) return std::nullopt;
|
||||
const auto it = Rewriters().find({ service, id });
|
||||
if (it == Rewriters().end()) return std::nullopt;
|
||||
const auto written = it->second(bytes);
|
||||
if (!written) return std::nullopt;
|
||||
return *written == bytes;
|
||||
}
|
||||
|
||||
bool HasSecrets(ServiceType service, uint32_t messageId) {
|
||||
const auto it = Registry().find({ service, messageId });
|
||||
return it != Registry().end() && it->second.redact;
|
||||
}
|
||||
|
||||
size_t RegisteredCount() { return Registry().size() + 1; }
|
||||
}
|
||||
79
dNet/PacketDecoder.h
Normal file
79
dNet/PacketDecoder.h
Normal file
@@ -0,0 +1,79 @@
|
||||
#ifndef __PACKETDECODER__H__
|
||||
#define __PACKETDECODER__H__
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
#include "BitStream.h"
|
||||
#include "dCommonVars.h"
|
||||
#include "json.hpp"
|
||||
#include "MessageType/Game.h"
|
||||
#include "NiPoint3.h"
|
||||
|
||||
enum class ServiceType : uint16_t;
|
||||
|
||||
/**
|
||||
* Reads recorded packets (PacketRecord.h) for the dashboard's capture viewer and the capture tool, with the
|
||||
* server's own packet structs and their Deserialize: a registry keyed by (service, message ID). Every packet gets a
|
||||
* name (RakNet's own messages, LU packets by their MessageType enum); the ones registered here also get their fields.
|
||||
* Game messages (WORLD and CLIENT GAME_MSG) are named by MessageType::Game; their fields come from the decoder a
|
||||
* program that links the game sets with SetGameMessageDecoder (the capture tool does; the dashboard doesn't link
|
||||
* the game, so it shows their bytes).
|
||||
*/
|
||||
namespace PacketDecoder {
|
||||
struct Decoded {
|
||||
std::string service; // "WORLD", "CLIENT", ... or "RAKNET" for RakNet's own messages
|
||||
std::string name; // e.g. "POSITION_UPDATE", "GAME_MSG REQUEST_USE", "ID_REPLICA_MANAGER_CONSTRUCTION"
|
||||
uint16_t serviceId{};
|
||||
uint32_t messageId{}; // the LU message ID, or the RakNet message ID
|
||||
bool lu{}; // an LU packet (8 byte header)
|
||||
int32_t gameMessageId{ -1 };
|
||||
LWOOBJID objectId{}; // game messages: the object it is for
|
||||
std::optional<nlohmann::json> fields; // set when a registered struct read it cleanly
|
||||
bool failed{}; // a registered struct did not read it
|
||||
};
|
||||
|
||||
// `fromClient`: the packet came from a game client (worlds and auth receiving on their listening peer)
|
||||
Decoded Decode(std::string_view bytes, bool fromClient);
|
||||
|
||||
// The name alone
|
||||
std::string Name(ServiceType service, uint32_t messageId);
|
||||
|
||||
// Where a client's POSITION_UPDATE says the player is; nullopt for any other packet
|
||||
std::optional<NiPoint3> Position(std::string_view bytes);
|
||||
|
||||
using GameMessageFields = std::function<std::optional<nlohmann::json>(MessageType::Game messageId, bool toServer, RakNet::BitStream& payload)>;
|
||||
void SetGameMessageDecoder(GameMessageFields decoder);
|
||||
|
||||
/**
|
||||
* Secrets never reach a capture: packets whose struct declares secret fields (passwords, session and user keys)
|
||||
* are read, those fields blanked (strings emptied, keys 0), and written again before they are recorded. Returns
|
||||
* false when the packet declares secrets but doesn't read cleanly: then it must not be recorded at all.
|
||||
* Packets without secrets are left as they are.
|
||||
*/
|
||||
bool Redact(std::string& bytes);
|
||||
|
||||
/**
|
||||
* For bundles that leave the server: account names are always replaced (portable bundles), and with `anonymise`
|
||||
* also character names and what players typed (test fixtures). Text is replaced with as many 'x' as it had, so
|
||||
* the packet keeps its size. Returns true if the packet changed.
|
||||
*/
|
||||
bool Scrub(std::string& bytes, bool anonymise);
|
||||
|
||||
// Whether a packet's struct declares secret fields (they are blanked when recorded)
|
||||
bool HasSecrets(ServiceType service, uint32_t messageId);
|
||||
|
||||
/**
|
||||
* Reads a packet with its struct and writes it again. nullopt: no struct for it (or it doesn't read); otherwise
|
||||
* whether the bytes came back the same (the local fixture tests' byte-equality check).
|
||||
*/
|
||||
std::optional<bool> RoundTrip(std::string_view bytes);
|
||||
|
||||
// How many (service, message) pairs have typed decoders, for tests
|
||||
size_t RegisteredCount();
|
||||
}
|
||||
|
||||
#endif //!__PACKETDECODER__H__
|
||||
96
dNet/PacketRecord.h
Normal file
96
dNet/PacketRecord.h
Normal file
@@ -0,0 +1,96 @@
|
||||
#ifndef __PACKETRECORD__H__
|
||||
#define __PACKETRECORD__H__
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
/**
|
||||
* One recorded packet of a packet capture (PacketCapture.h), as the servers pack them into batches and as the
|
||||
* dashboard appends them to a capture file: a fixed little-endian header, then the packet's bytes exactly as they
|
||||
* went over RakNet (the RakNet message ID first; for LU packets the 8 byte header follows).
|
||||
*
|
||||
* Files of records (a capture on the dashboard's disk, an exported bundle) are described in CaptureBundle.h.
|
||||
*/
|
||||
enum class eCaptureSource : uint8_t {
|
||||
UNKNOWN,
|
||||
AUTH,
|
||||
CHAT,
|
||||
WORLD,
|
||||
MASTER,
|
||||
};
|
||||
|
||||
enum class ePacketDirection : uint8_t {
|
||||
RECEIVED, // by the server that recorded it
|
||||
SENT, // by the server that recorded it
|
||||
};
|
||||
|
||||
namespace PacketRecordFlags {
|
||||
constexpr uint8_t MASTER_LINK = 1 << 0; // on a server's own link to master (not its listening socket)
|
||||
constexpr uint8_t BROADCAST = 1 << 1; // sent to everyone connected (peer is who it skipped)
|
||||
constexpr uint8_t CUT = 1 << 2; // longer than the server keeps; `bits` is the full size
|
||||
constexpr uint8_t GAP = 1 << 3; // not a packet: `bits` packets were lost here (a server's buffer was full)
|
||||
}
|
||||
|
||||
#pragma pack(push, 1)
|
||||
struct PacketRecordHeader {
|
||||
int64_t timeUs{}; // Unix time in microseconds
|
||||
uint32_t seq{}; // per server, from 1 when it first armed a capture; a jump means records were dropped
|
||||
uint8_t mask{}; // the capture slots this record belongs to (one bit each)
|
||||
uint8_t source{}; // eCaptureSource
|
||||
uint8_t direction{}; // ePacketDirection
|
||||
uint8_t flags{}; // PacketRecordFlags
|
||||
uint64_t peer{}; // the other end: IPv4 address << 16 | port (0: none)
|
||||
uint32_t accountId{}; // whose it is, when known
|
||||
int64_t characterId{};
|
||||
uint16_t zoneId{}; // the recording server's zone (worlds), else 0
|
||||
uint16_t instanceId{};
|
||||
uint32_t cloneId{};
|
||||
uint32_t bits{}; // the packet's full size in bits
|
||||
uint32_t length{}; // bytes of the packet stored after this header
|
||||
};
|
||||
#pragma pack(pop)
|
||||
static_assert(sizeof(PacketRecordHeader) == 52, "The record header is part of the capture file format");
|
||||
|
||||
namespace PacketRecord {
|
||||
// Bytes of one packet kept; longer ones are cut (large replica constructions and character data fit)
|
||||
constexpr uint32_t MAX_BYTES = 256 * 1024;
|
||||
|
||||
inline void Append(std::string& out, const PacketRecordHeader& header, const void* data) {
|
||||
const auto at = out.size();
|
||||
out.resize(at + sizeof(header) + header.length);
|
||||
std::memcpy(out.data() + at, &header, sizeof(header));
|
||||
if (header.length) std::memcpy(out.data() + at + sizeof(header), data, header.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calls fn(header, bytes) for every record in `blob`, in order. Returns false if the blob ends in the middle of a
|
||||
* record or a record is larger than records can be (what came before was still passed to fn).
|
||||
*/
|
||||
template<typename Fn>
|
||||
bool ForEach(std::string_view blob, Fn&& fn) {
|
||||
size_t at = 0;
|
||||
while (at < blob.size()) {
|
||||
if (blob.size() - at < sizeof(PacketRecordHeader)) return false;
|
||||
PacketRecordHeader header;
|
||||
std::memcpy(&header, blob.data() + at, sizeof(header));
|
||||
at += sizeof(header);
|
||||
if (header.length > MAX_BYTES || blob.size() - at < header.length) return false;
|
||||
fn(header, blob.substr(at, header.length));
|
||||
at += header.length;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// "a.b.c.d:port" of a record's peer
|
||||
inline std::string PeerText(uint64_t peer) {
|
||||
if (peer == 0) return "";
|
||||
const auto address = static_cast<uint32_t>(peer >> 16);
|
||||
// RakNet keeps IPv4 addresses in network order
|
||||
return std::to_string(address & 0xff) + "." + std::to_string((address >> 8) & 0xff) + "." + std::to_string((address >> 16) & 0xff) + "." +
|
||||
std::to_string(address >> 24) + ":" + std::to_string(peer & 0xffff);
|
||||
}
|
||||
}
|
||||
|
||||
#endif //!__PACKETRECORD__H__
|
||||
@@ -15,6 +15,8 @@
|
||||
#include "CommonPackets.h"
|
||||
#include "MasterPackets.h"
|
||||
#include "ZoneInstanceManager.h"
|
||||
#include "PacketCapture.h"
|
||||
#include "master/MessageCapture.h"
|
||||
#include "StringifiedEnum.h"
|
||||
#include "GeneralUtils.h"
|
||||
#include "TrafficStats.h"
|
||||
@@ -134,6 +136,9 @@ dServer::dServer(
|
||||
mPeer->AttachPlugin(mReplicaManager);
|
||||
mPeer->SetNetworkIDManager(mNetIDManager);
|
||||
}
|
||||
|
||||
// The dashboard's packet capture records what goes through the listening peer and, for a captured player, the master link
|
||||
PacketCapture::Attach(serverType, mPeer, mMasterPeer, zoneID, static_cast<uint32_t>(instanceID));
|
||||
}
|
||||
|
||||
dServer::~dServer() {
|
||||
@@ -148,6 +153,7 @@ Packet* dServer::ReceiveFromMaster() {
|
||||
CountTraffic(packet, TrafficStats::Peer::MASTER);
|
||||
if (packet) {
|
||||
if (packet->length < 1) { mMasterPeer->DeallocatePacket(packet); return nullptr; }
|
||||
PacketCapture::OnReceiveFromMaster(packet);
|
||||
|
||||
switch (packet->data[0]) {
|
||||
case ID_DISCONNECTION_NOTIFICATION:
|
||||
@@ -201,6 +207,15 @@ Packet* dServer::ReceiveFromMaster() {
|
||||
break;
|
||||
}
|
||||
|
||||
case MessageType::Master::MESSAGE_CAPTURE_CONTROL: {
|
||||
// Packet captures run on every server; the game message inspector's start and stop go to the world's own handler
|
||||
MessageCaptureControl control;
|
||||
if (!control.Deserialize(inStream)) break;
|
||||
if (control.action != eMessageCaptureControl::ARM && control.action != eMessageCaptureControl::DISARM) return packet;
|
||||
PacketCapture::Control(control);
|
||||
break;
|
||||
}
|
||||
|
||||
// When we handle these packets in World instead dServer, we just return the packet's pointer.
|
||||
default:
|
||||
return packet;
|
||||
@@ -219,6 +234,7 @@ Packet* dServer::ReceiveFromMaster() {
|
||||
Packet* dServer::Receive() {
|
||||
Packet* packet = mPeer->Receive();
|
||||
CountTraffic(packet, PeerOfConnections());
|
||||
PacketCapture::OnReceive(packet);
|
||||
return packet;
|
||||
}
|
||||
|
||||
@@ -297,6 +313,7 @@ void dServer::UpdateBandwidthLimit() {
|
||||
}
|
||||
|
||||
void dServer::Shutdown() {
|
||||
PacketCapture::Detach();
|
||||
if (mPeer) {
|
||||
mPeer->Shutdown(1000);
|
||||
RakNetworkFactory::DestroyRakPeerInterface(mPeer);
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
#define __MESSAGECAPTURE__H__
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <deque>
|
||||
@@ -30,12 +31,22 @@ enum class eMessageDirection : uint8_t {
|
||||
enum class eMessageCaptureControl : uint8_t {
|
||||
START,
|
||||
STOP,
|
||||
ARM, // packet capture (PacketCapture.h): every server records the target's packets into `slot`
|
||||
DISARM, // packet capture: stop recording into `slot`
|
||||
};
|
||||
|
||||
// What a packet capture (ARM) records
|
||||
enum class eCaptureTarget : uint8_t {
|
||||
CHARACTER, // one character, from the moment it is picked in a world
|
||||
ACCOUNT, // everything of one account, from its login (every character)
|
||||
EVERYTHING, // all traffic on all servers
|
||||
};
|
||||
|
||||
enum class eMessageCaptureStatus : uint8_t {
|
||||
STARTED, // the world holding the character started capturing
|
||||
ENTRIES, // captured messages (also sent empty every few seconds while capturing)
|
||||
ENDED, // the world stopped capturing; `reason` says why
|
||||
PACKETS, // a batch of recorded packets (PacketCapture.h) from any server: `packets`, `slots`, `source`
|
||||
};
|
||||
|
||||
enum class eMessageCaptureEnd : uint8_t {
|
||||
@@ -53,6 +64,12 @@ namespace MessageCapture {
|
||||
constexpr uint16_t MAX_DECODED = 8192;
|
||||
// Message IDs in a filter list
|
||||
constexpr uint16_t MAX_FILTER = 256;
|
||||
// Packet captures armed at once (one bit each in a packet record's mask)
|
||||
constexpr uint8_t MAX_SLOTS = 8;
|
||||
// Characters of an account a packet capture follows
|
||||
constexpr uint16_t MAX_CHARACTERS = 64;
|
||||
// One batch of packet records
|
||||
constexpr uint32_t MAX_PACKET_BATCH = 8 * 1024 * 1024;
|
||||
|
||||
// Lowercase hex, two digits per byte
|
||||
inline std::string ToHex(std::string_view bytes) {
|
||||
@@ -106,6 +123,12 @@ struct MessageCaptureControl : public LUBitStream {
|
||||
bool toClient{ true }; // START: capture what the client receives
|
||||
std::vector<uint16_t> only; // START: capture only these message IDs (empty: all)
|
||||
std::vector<uint16_t> skip; // START: never capture these message IDs
|
||||
// ARM / DISARM (packet capture); `seconds` is its time limit and captureId its id
|
||||
uint8_t slot{}; // 0 to MAX_SLOTS - 1: the bit this capture has in packet records
|
||||
eCaptureTarget target{};
|
||||
uint32_t accountId{}; // ACCOUNT, CHARACTER: the account
|
||||
std::string accountName; // its name, for packets that name the account instead (logins, session keys)
|
||||
std::vector<LWOOBJID> characterIds; // ACCOUNT: its characters; CHARACTER: the one
|
||||
|
||||
void Serialize(RakNet::BitStream& stream) const override {
|
||||
stream.Write(captureId);
|
||||
@@ -116,6 +139,13 @@ struct MessageCaptureControl : public LUBitStream {
|
||||
stream.Write<uint8_t>(toClient);
|
||||
MessageCapture::WriteIds(stream, only);
|
||||
MessageCapture::WriteIds(stream, skip);
|
||||
stream.Write(slot);
|
||||
stream.Write(target);
|
||||
stream.Write(accountId);
|
||||
MessageCapture::WriteString(stream, accountName, 64);
|
||||
const auto count = static_cast<uint16_t>(std::min<size_t>(characterIds.size(), MessageCapture::MAX_CHARACTERS));
|
||||
stream.Write(count);
|
||||
for (uint16_t i = 0; i < count; i++) stream.Write(characterIds[i]);
|
||||
}
|
||||
|
||||
bool Deserialize(RakNet::BitStream& stream) override {
|
||||
@@ -125,7 +155,13 @@ struct MessageCaptureControl : public LUBitStream {
|
||||
seconds = std::min(seconds, MessageCapture::MAX_SECONDS);
|
||||
toServer = server != 0;
|
||||
toClient = client != 0;
|
||||
return MessageCapture::ReadIds(stream, only) && MessageCapture::ReadIds(stream, skip);
|
||||
if (!MessageCapture::ReadIds(stream, only) || !MessageCapture::ReadIds(stream, skip)) return false;
|
||||
uint16_t count{};
|
||||
if (!stream.Read(slot) || slot >= MessageCapture::MAX_SLOTS || !stream.Read(target) || target > eCaptureTarget::EVERYTHING ||
|
||||
!stream.Read(accountId) || !MessageCapture::ReadString(stream, accountName, 64) || !stream.Read(count) || count > MessageCapture::MAX_CHARACTERS) return false;
|
||||
characterIds.resize(count);
|
||||
for (auto& id : characterIds) if (!stream.Read(id)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Whether a message passes this capture's filters
|
||||
@@ -184,6 +220,13 @@ struct MessageCaptureData : public LUBitStream {
|
||||
uint32_t dropped{}; // messages left out since the last batch (over the rate or buffer limit)
|
||||
std::vector<MessageCaptureEntry> entries;
|
||||
uint32_t cloneId{}; // the world's clone (a property's owner), 0 elsewhere
|
||||
// PACKETS: records (PacketRecord.h) packed one after another, the capture id of each mask bit when they were
|
||||
// recorded, which server recorded them, how many there are and how many that server left out since its last batch
|
||||
uint8_t source{}; // eCaptureSource
|
||||
std::array<uint32_t, MessageCapture::MAX_SLOTS> slots{};
|
||||
uint32_t packetCount{};
|
||||
uint32_t packetsDropped{};
|
||||
std::string packets;
|
||||
|
||||
void Serialize(RakNet::BitStream& stream) const override {
|
||||
stream.Write(captureId);
|
||||
@@ -197,6 +240,13 @@ struct MessageCaptureData : public LUBitStream {
|
||||
stream.Write(count);
|
||||
for (uint16_t i = 0; i < count; i++) entries[i].Serialize(stream);
|
||||
stream.Write(cloneId);
|
||||
stream.Write(source);
|
||||
for (const auto id : slots) stream.Write(id);
|
||||
stream.Write(packetCount);
|
||||
stream.Write(packetsDropped);
|
||||
const auto length = static_cast<uint32_t>(std::min<size_t>(packets.size(), MessageCapture::MAX_PACKET_BATCH));
|
||||
stream.Write(length);
|
||||
if (length) stream.Write(packets.data(), length);
|
||||
}
|
||||
|
||||
bool Deserialize(RakNet::BitStream& stream) override {
|
||||
@@ -205,7 +255,12 @@ struct MessageCaptureData : public LUBitStream {
|
||||
!stream.Read(instanceId) || !stream.Read(reason) || !stream.Read(dropped) || !stream.Read(count) || count > MAX_ENTRIES) return false;
|
||||
entries.resize(count);
|
||||
for (auto& entry : entries) if (!entry.Deserialize(stream)) return false;
|
||||
return stream.Read(cloneId);
|
||||
uint32_t length{};
|
||||
if (!stream.Read(cloneId) || !stream.Read(source)) return false;
|
||||
for (auto& id : slots) if (!stream.Read(id)) return false;
|
||||
if (!stream.Read(packetCount) || !stream.Read(packetsDropped) || !stream.Read(length) || length > MessageCapture::MAX_PACKET_BATCH) return false;
|
||||
packets.resize(length);
|
||||
return length == 0 || stream.Read(packets.data(), length);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user