From 3a8838d4634ca03a30d879cf42e0b5c04e9891bf Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Sun, 27 Sep 2026 09:25:37 -0500 Subject: [PATCH] feat(capture): replay bundles against a sandbox stack with a headless client CaptureTool (built next to the servers) replays packet bundles and compares the answers: - replay: per bundle a fresh sandbox folder with copied server binaries, rewritten settings (replay_sandbox=1, a new SQLite file inside the folder, ports from --port on, no dashboard), read back before anything starts; sandbox-setup runs inside it to apply migrations and make the replay account and the bundle's characters (setup mode); master is started, the stack is stopped as one process group and the folder deleted unless kept - with replay_sandbox=1 every server refuses a database that isn't SQLite, isn't inside its own folder, or is replay_live_sqlite_path (Database::Connect); replay-target against a running server needs --i-know-this-is-not-a-sandbox - the fake client splits the recording into connections, logs in and picks the character itself when the recording doesn't, fills in the target's account, session key and IDs, learns server-made object IDs from replica constructions by LOT, follows the recorded timing and waits for the answers a client waits for; the diff pairs answers by name (and constructions by LOT) and ignores fields that differ between runs - import-live converts the 2014 live captures (folders of *_traffic.zip; pcaps and encrypted captures are left alone) into bundles, with secrets removed and CREATE_CHARACTER as setup - anonymise makes local fixtures; docs/CaptureReplay.md describes capture, the bundle format, portability rules, the sandbox and the replay Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 1 + dCaptureTool/CMakeLists.txt | 5 + dCaptureTool/CaptureTool.cpp | 313 +++++++++++++++++++++ dCaptureTool/FakeClient.cpp | 95 +++++++ dCaptureTool/FakeClient.h | 55 ++++ dCaptureTool/LiveImport.cpp | 287 +++++++++++++++++++ dCaptureTool/LiveImport.h | 30 ++ dCaptureTool/Replayer.cpp | 413 ++++++++++++++++++++++++++++ dCaptureTool/Replayer.h | 51 ++++ dCaptureTool/Sandbox.cpp | 271 ++++++++++++++++++ dCaptureTool/Sandbox.h | 57 ++++ dDatabase/GameDatabase/Database.cpp | 27 ++ dNet/CaptureTools.cpp | 33 ++- dNet/PacketDecoder.cpp | 19 ++ docs/CaptureReplay.md | 213 ++++++++++++++ docs/Dashboard.md | 27 ++ docs/PacketArchitecture.md | 1 + 17 files changed, 1885 insertions(+), 13 deletions(-) create mode 100644 dCaptureTool/CMakeLists.txt create mode 100644 dCaptureTool/CaptureTool.cpp create mode 100644 dCaptureTool/FakeClient.cpp create mode 100644 dCaptureTool/FakeClient.h create mode 100644 dCaptureTool/LiveImport.cpp create mode 100644 dCaptureTool/LiveImport.h create mode 100644 dCaptureTool/Replayer.cpp create mode 100644 dCaptureTool/Replayer.h create mode 100644 dCaptureTool/Sandbox.cpp create mode 100644 dCaptureTool/Sandbox.h create mode 100644 docs/CaptureReplay.md diff --git a/CMakeLists.txt b/CMakeLists.txt index d526cb686..14b7c1eb5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -331,6 +331,7 @@ add_subdirectory(dAuthServer) add_subdirectory(dChatServer) add_subdirectory(dDashboardServer) add_subdirectory(dUgcServer) +add_subdirectory(dCaptureTool) add_subdirectory(dMasterServer) # Add MasterServer last so it can rely on the other binaries target_precompile_headers( diff --git a/dCaptureTool/CMakeLists.txt b/dCaptureTool/CMakeLists.txt new file mode 100644 index 000000000..c1c202918 --- /dev/null +++ b/dCaptureTool/CMakeLists.txt @@ -0,0 +1,5 @@ +# The capture tool (docs/CaptureReplay.md): bundles, live capture import, and replays against a sandbox stack +add_executable(CaptureTool "CaptureTool.cpp" "FakeClient.cpp" "LiveImport.cpp" "Replayer.cpp" "Sandbox.cpp") +target_include_directories(CaptureTool PRIVATE ${PROJECT_SOURCE_DIR}/dServer) +target_compile_definitions(CaptureTool PRIVATE PROJECT_VERSION="\"${PROJECT_VERSION}\"") +target_link_libraries(CaptureTool ${COMMON_LIBRARIES} bcrypt dServer) diff --git a/dCaptureTool/CaptureTool.cpp b/dCaptureTool/CaptureTool.cpp new file mode 100644 index 000000000..fc0458527 --- /dev/null +++ b/dCaptureTool/CaptureTool.cpp @@ -0,0 +1,313 @@ +/** + * The capture tool (docs/CaptureReplay.md): looks into packet bundles, converts the 2014 live captures into bundles, + * makes anonymous test fixtures, and replays bundles against a throwaway sandbox server stack, comparing the + * server's answers with the recorded ones. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "BinaryPathFinder.h" +#include "CaptureBundle.h" +#include "CaptureTools.h" +#include "LiveImport.h" +#include "PacketDecoder.h" +#include "Replayer.h" +#include "Sandbox.h" + +#include + +#include "dConfig.h" +#include "dServer.h" +#include "Game.h" +#include "Logger.h" + +namespace fs = std::filesystem; +using json = nlohmann::json; + +// What every program linking the game's libraries defines (the tool runs no server of its own) +namespace Game { + Logger* logger = nullptr; + dServer* server = nullptr; + dConfig* config = nullptr; + Game::signal_t lastSignal = 0; + std::mt19937 randomEngine; +} + +namespace { + void Usage() { + std::cout << + "CaptureTool: packet bundles (docs/CaptureReplay.md)\n" + " info what is in a bundle\n" + " decode [--fields] [--limit N] its packets on one timeline\n" + " anonymise a test fixture: names and chat replaced, IDs placeholders\n" + " import-live convert live captures (every folder of *_traffic.zip under )\n" + " replay ... [options] replay against a fresh sandbox stack per bundle and compare\n" + " --client the game client's files (default: client_location of the servers' sharedconfig.ini)\n" + " --server-dir the built servers (default: this tool's folder)\n" + " --cdserver CDServer.sqlite to copy (default: /resServer/CDServer.sqlite)\n" + " --sandbox-root where sandboxes are made (default: the system's temporary folder)\n" + " --port first of the sandbox's ports (default 41000; uses n to n+200)\n" + " --mode setup|as-is setup (default): make the bundle's characters first; as-is: only an account\n" + " --speed how much faster than recorded (default 4)\n" + " --keep | --keep-on-failure keep the sandbox folder\n" + " --report write the results as JSON\n" + " replay-target --host --auth-port

--username --password

--i-know-this-is-not-a-sandbox\n" + " replay against a running server (it must not be a live one)\n"; + } + + std::string Arg(const std::vector& args, const std::string& name, const std::string& fallback = "") { + for (size_t i = 0; i + 1 < args.size(); i++) if (args[i] == name) return args[i + 1]; + return fallback; + } + bool Flag(const std::vector& args, const std::string& name) { + for (const auto& arg : args) if (arg == name) return true; + return false; + } + // Arguments that aren't options (nor option values) + std::vector Positional(const std::vector& args, size_t from) { + static const std::vector withValue{ "--client", "--server-dir", "--cdserver", "--sandbox-root", "--port", "--mode", "--speed", "--report", + "--host", "--auth-port", "--username", "--password", "--limit" }; + std::vector out; + for (size_t i = from; i < args.size(); i++) { + if (std::find(withValue.begin(), withValue.end(), args[i]) != withValue.end()) i++; + else if (!args[i].starts_with("--")) out.push_back(args[i]); + } + return out; + } + + bool LoadBundle(const std::string& path, CaptureBundle::Bundle& bundle) { + std::string error; + bool truncated = false; + if (!CaptureBundle::Load(path, bundle, error, &truncated)) { + std::cerr << path << ": " << error << "\n"; + return false; + } + if (truncated) std::cerr << path << ": the last record is cut short (a capture still being written?)\n"; + return true; + } + + std::string ConfigValue(const fs::path& file, const std::string& key) { + std::ifstream in(file); + std::string line, value; + while (std::getline(in, line)) { + if (line.starts_with(key + "=")) value = line.substr(key.size() + 1); + } + return value; + } + + int Info(const std::string& path) { + CaptureBundle::Bundle bundle; + if (!LoadBundle(path, bundle)) return 1; + std::map names; + for (const auto& record : bundle.records) names[PacketDecoder::Decode(record.bytes, CaptureTools::FromClient(record.header)).name]++; + auto meta = bundle.meta; + if (meta.contains("setup")) for (auto& character : meta["setup"]["characters"]) character["xml"] = std::to_string(character.value("xml", std::string()).size()) + " bytes"; + std::cout << meta.dump(2) << "\n" << bundle.records.size() << " packets\n"; + for (const auto& [name, count] : names) std::cout << " " << count << "\t" << name << "\n"; + return 0; + } + + int Decode(const std::string& path, bool fields, size_t limit) { + CaptureBundle::Bundle bundle; + if (!LoadBundle(path, bundle)) return 1; + CaptureTools::SortTimeline(bundle.records); + const auto start = bundle.records.empty() ? 0 : bundle.records.front().header.timeUs; + for (size_t i = 0; i < bundle.records.size() && i < limit; i++) { + const auto j = CaptureTools::RecordJson(bundle.records[i], i, start, fields); + std::printf("%7zu %10.3f %-12s %-18s %s%s\n", i, j["t"].get() / 1000.0, (j.value("from", std::string()) + ">" + j.value("to", std::string())).c_str(), + j.value("source", std::string()).c_str(), j.value("name", std::string()).c_str(), fields && j.contains("fields") ? (" " + j["fields"].dump()).c_str() : ""); + } + return 0; + } + + int Anonymise(const std::string& in, const std::string& out) { + CaptureBundle::Bundle bundle; + if (!LoadBundle(in, bundle)) return 1; + if (!bundle.meta.value("portable", false)) CaptureTools::MakePortable(bundle); + const auto changed = CaptureTools::Anonymise(bundle); + // The setup section's names too + if (bundle.meta.contains("setup")) for (auto& character : bundle.meta["setup"]["characters"]) character["name"] = character.value("symbol", std::string("replay")); + if (!CaptureBundle::Save(out, bundle)) { + std::cerr << "Can't write " << out << "\n"; + return 1; + } + std::cout << "Wrote " << out << " (" << changed << " packets changed). Keep it local: tests/fixtures-local is never committed.\n"; + return 0; + } + + int ImportLive(const fs::path& root, const fs::path& outDir) { + std::error_code ec; + fs::create_directories(outDir, ec); + auto scenarios = LiveImport::FindScenarios(root); + if (scenarios.empty() && fs::is_regular_file(root)) scenarios.push_back(root); + size_t written = 0; + for (const auto& scenario : scenarios) { + auto result = LiveImport::Import(scenario); + if (!result.error.empty()) { + std::cout << "skipped " << scenario << ": " << result.error << "\n"; + continue; + } + // A file name from the folders it came from + auto name = fs::relative(scenario, fs::is_directory(root) ? root : root.parent_path(), ec).string(); + for (auto& c : name) if (!std::isalnum(static_cast(c)) && c != '-' && c != '_') c = '_'; + const auto out = outDir / (name + ".bundle"); + if (!CaptureBundle::Save(out, result.bundle)) { + std::cout << "can't write " << out << "\n"; + continue; + } + written++; + std::cout << out.filename().string() << ": " << result.zips << " zip(s), " << result.packets << " packets, " << result.skipped << " skipped, " + << result.bundle.meta["setup"]["characters"].size() << " character(s)\n"; + } + std::cout << written << " bundle(s) in " << outDir << "\n"; + return written ? 0 : 1; + } + + void PrintResult(const std::string& name, const Replayer::Result& r) { + std::cout << "== " << name << "\n" + << " connections " << r.connectionsReached << "/" << r.connections << ", sent " << r.sent << ", received " << r.received + << (r.stoppedAt.empty() ? "" : ", stopped: " + r.stoppedAt) << "\n" + << " answers: " << r.diff.expected << " recorded, " << r.diff.matched << " same, " << r.diff.differing << " different, " << r.diff.missing + << " missing, " << r.diff.extra << " extra\n"; + const auto top = [](const std::map& counts, const char* what) { + std::vector> sorted; + for (const auto& [n, c] : counts) sorted.push_back({ c, n }); + std::sort(sorted.rbegin(), sorted.rend()); + if (sorted.empty()) return; + std::cout << " " << what << ":"; + for (size_t i = 0; i < sorted.size() && i < 6; i++) std::cout << " " << sorted[i].second << " (" << sorted[i].first << ")"; + std::cout << "\n"; + }; + top(r.diff.differingByName, "different"); + top(r.diff.missingByName, "missing"); + top(r.diff.extraByName, "extra"); + for (const auto& note : r.notes) std::cout << " note: " << note << "\n"; + } + + int Replay(const std::vector& args) { + const auto bundles = Positional(args, 2); + if (bundles.empty()) { + Usage(); + return 1; + } + Sandbox::Options options; + options.serverDir = fs::absolute(Arg(args, "--server-dir", BinaryPathFinder::GetBinaryDir().string())); + options.root = Arg(args, "--sandbox-root", fs::temp_directory_path().string()); + options.cdServer = Arg(args, "--cdserver", (options.serverDir / "resServer" / "CDServer.sqlite").string()); + options.clientLocation = Arg(args, "--client", ConfigValue(options.serverDir / "sharedconfig.ini", "client_location")); + options.basePort = static_cast(std::stoi(Arg(args, "--port", "41000"))); + options.keep = Flag(args, "--keep"); + // The servers' own database, which a sandbox must never be + const auto live = ConfigValue(options.serverDir / "sharedconfig.ini", "sqlite_database_path"); + if (!live.empty()) options.liveDatabase = fs::absolute(options.serverDir / live); + if (options.clientLocation.empty()) { + std::cerr << "Where is the game client? Pass --client \n"; + return 1; + } + const bool setup = Arg(args, "--mode", "setup") != "as-is"; + json report = json::array(); + int failures = 0; + for (const auto& path : bundles) { + CaptureBundle::Bundle bundle; + if (!LoadBundle(path, bundle)) { + failures++; + continue; + } + std::string error; + auto stack = Sandbox::Stack::Create(options, error); + if (!stack) { + std::cerr << error << "\n"; + return 1; + } + const std::string username = "replay", password = "replay-sandbox"; + json ids; + Replayer::Result result; + if (!stack->Setup(path, username, password, setup, ids, error) || !stack->Start(error)) { + result.stoppedAt = error; + } else { + Replayer::Options replay; + replay.authPort = stack->AuthPort(); + replay.username = username; + replay.password = password; + replay.speed = std::stod(Arg(args, "--speed", "4")); + for (const auto& [symbol, character] : ids.items()) { + const auto placeholder = std::stoll(character["placeholder"].get()); + const auto id = std::stoll(character["id"].get()); + replay.ids[placeholder] = id; + if (!replay.character) replay.character = id; + } + result = Replayer::Replay(bundle, replay); + stack->Stop(); + } + // Where the bundle came from and what it ran on, so differences in data aren't read as server bugs + auto entry = result.ToJson(); + entry["bundle"] = path; + entry["origin"] = bundle.meta.value("origin", ""); + entry["recordedOn"] = bundle.meta.value("server", json::object()); + entry["replayedOn"] = { {"version", PROJECT_VERSION} }; + entry["sandbox"] = stack->Dir().string(); + report.push_back(entry); + PrintResult(fs::path(path).filename().string(), result); + const bool failed = !result.stoppedAt.empty(); + if (failed) failures++; + if (options.keep || (failed && Flag(args, "--keep-on-failure"))) { + stack->Keep(); + std::cout << " sandbox kept: " << stack->Dir() << "\n"; + } + } + const auto reportPath = Arg(args, "--report"); + if (!reportPath.empty()) std::ofstream(reportPath) << report.dump(1); + return failures ? 2 : 0; + } + + int ReplayTarget(const std::vector& args) { + if (!Flag(args, "--i-know-this-is-not-a-sandbox")) { + std::cerr << "replay-target sends a recording's packets to a running server as the account you give. Never point it at a live\n" + "server. If this really is a test server, add --i-know-this-is-not-a-sandbox. Otherwise use: replay (a fresh sandbox).\n"; + return 1; + } + const auto bundles = Positional(args, 2); + if (bundles.size() != 1) { + Usage(); + return 1; + } + CaptureBundle::Bundle bundle; + if (!LoadBundle(bundles[0], bundle)) return 1; + Replayer::Options replay; + replay.host = Arg(args, "--host", "127.0.0.1"); + replay.authPort = static_cast(std::stoi(Arg(args, "--auth-port", "1001"))); + replay.username = Arg(args, "--username"); + replay.password = Arg(args, "--password"); + replay.speed = std::stod(Arg(args, "--speed", "4")); + std::cout << "!! Replaying against " << replay.host << ":" << replay.authPort << ", which is NOT a sandbox.\n"; + const auto result = Replayer::Replay(bundle, replay); + PrintResult(bundles[0], result); + const auto reportPath = Arg(args, "--report"); + if (!reportPath.empty()) std::ofstream(reportPath) << result.ToJson().dump(1); + return result.stoppedAt.empty() ? 0 : 2; + } +} + +int main(int argc, char** argv) { + const std::vector args(argv, argv + argc); + if (args.size() < 2) { + Usage(); + return 1; + } + const auto& command = args[1]; + if (command == "info" && args.size() >= 3) return Info(args[2]); + if (command == "decode" && args.size() >= 3) return Decode(args[2], Flag(args, "--fields"), std::stoul(Arg(args, "--limit", "1000000"))); + if (command == "anonymise" && args.size() >= 4) return Anonymise(args[2], args[3]); + if (command == "import-live" && args.size() >= 4) return ImportLive(args[2], args[3]); + if (command == "replay") return Replay(args); + if (command == "replay-target") return ReplayTarget(args); + if (command == "sandbox-setup" && args.size() >= 7) return Sandbox::SetupCommand(args[2], args[3], args[4], args[5] == "1", args[6]); + Usage(); + return 1; +} diff --git a/dCaptureTool/FakeClient.cpp b/dCaptureTool/FakeClient.cpp new file mode 100644 index 000000000..3d1be8b4f --- /dev/null +++ b/dCaptureTool/FakeClient.cpp @@ -0,0 +1,95 @@ +#include "FakeClient.h" + +#include +#include + +#include "dNetCommon.h" +#include "MessageIdentifiers.h" +#include "RakNetworkFactory.h" +#include "RakPeerInterface.h" +#include "PacketPriority.h" + +namespace { + int64_t NowUs() { + return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + } +} + +FakeClient::FakeClient() = default; + +FakeClient::~FakeClient() { Disconnect(); } + +bool FakeClient::Connect(const std::string& host, uint16_t port, std::chrono::milliseconds timeout) { + Disconnect(); + m_Peer = RakNetworkFactory::GetRakPeerInterface(); + SocketDescriptor socket(0, nullptr); + if (!m_Peer->Startup(1, 10, &socket, 1)) return false; + m_LocalPort = m_Peer->GetInternalID().port; + if (!m_Peer->Connect(host.c_str(), port, NET_PASSWORD_EXTERNAL, static_cast(strnlen(NET_PASSWORD_EXTERNAL, sizeof(NET_PASSWORD_EXTERNAL))))) return false; + const auto until = std::chrono::steady_clock::now() + timeout; + while (std::chrono::steady_clock::now() < until) { + for (Packet* packet = m_Peer->Receive(); packet; packet = m_Peer->Receive()) { + const auto id = packet->length ? packet->data[0] : 0; + if (id == ID_CONNECTION_REQUEST_ACCEPTED) { + m_Server = packet->systemAddress; + m_Connected = true; + } + m_Peer->DeallocatePacket(packet); + if (id == ID_CONNECTION_ATTEMPT_FAILED || id == ID_NO_FREE_INCOMING_CONNECTIONS || id == ID_INVALID_PASSWORD) return false; + if (m_Connected) return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + return false; +} + +void FakeClient::Disconnect() { + if (!m_Peer) return; + if (m_Connected) m_Peer->CloseConnection(m_Server, true); + m_Peer->Shutdown(100); + RakNetworkFactory::DestroyRakPeerInterface(m_Peer); + m_Peer = nullptr; + m_Connected = false; +} + +void FakeClient::Send(const std::string& bytes) { + if (!m_Connected || bytes.empty()) return; + m_Peer->Send(bytes.data(), static_cast(bytes.size()), SYSTEM_PRIORITY, RELIABLE_ORDERED, 0, m_Server, false); +} + +bool FakeClient::Pump() { + if (!m_Peer) return false; + for (Packet* packet = m_Peer->Receive(); packet; packet = m_Peer->Receive()) { + if (packet->length) { + const auto id = packet->data[0]; + if (id == ID_DISCONNECTION_NOTIFICATION || id == ID_CONNECTION_LOST) m_Connected = false; + // RakNet's own connection messages aren't the server's answers + if (id == ID_USER_PACKET_ENUM || (id >= ID_REPLICA_MANAGER_CONSTRUCTION && id <= ID_REPLICA_MANAGER_DOWNLOAD_COMPLETE)) { + m_Received.push_back({ std::string(reinterpret_cast(packet->data), packet->length), NowUs() }); + } + } + m_Peer->DeallocatePacket(packet); + } + return m_Connected; +} + +int64_t FakeClient::WaitFor(const std::function& until, size_t from, std::chrono::milliseconds timeout) { + const auto end = std::chrono::steady_clock::now() + timeout; + size_t checked = from; + while (true) { + const bool open = Pump(); + for (; checked < m_Received.size(); checked++) { + if (until(m_Received[checked])) return static_cast(checked); + } + if (!open || std::chrono::steady_clock::now() >= end) return -1; + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + } +} + +void FakeClient::Idle(std::chrono::milliseconds time) { + const auto end = std::chrono::steady_clock::now() + time; + while (std::chrono::steady_clock::now() < end) { + if (!Pump()) return; + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + } +} diff --git a/dCaptureTool/FakeClient.h b/dCaptureTool/FakeClient.h new file mode 100644 index 000000000..c1f6aaab8 --- /dev/null +++ b/dCaptureTool/FakeClient.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "RakNetTypes.h" + +class RakPeerInterface; + +/** + * A headless game client for the capture tool's replay: one RakNet connection to an auth or world server that sends + * packets as given and keeps every packet it receives, with when it came. It speaks only RakNet and the LU packet + * header; what to send comes from the bundle being replayed. + */ +class FakeClient { +public: + struct Received { + std::string bytes; + int64_t timeUs{}; + }; + + FakeClient(); + ~FakeClient(); + FakeClient(const FakeClient&) = delete; + FakeClient& operator=(const FakeClient&) = delete; + + bool Connect(const std::string& host, uint16_t port, std::chrono::milliseconds timeout); + void Disconnect(); + bool IsConnected() const { return m_Connected; } + + void Send(const std::string& bytes); + + // Reads what arrived; false once the server closed the connection + bool Pump(); + + // Pumps until `until` returns true for a received packet (checked from `from` on) or the time is up. Returns + // the index of that packet, or -1. + int64_t WaitFor(const std::function& until, size_t from, std::chrono::milliseconds timeout); + + // Pumps for this long + void Idle(std::chrono::milliseconds time); + + const std::vector& GetReceived() const { return m_Received; } + uint16_t GetLocalPort() const { return m_LocalPort; } + +private: + RakPeerInterface* m_Peer{}; + SystemAddress m_Server{}; + bool m_Connected{}; + uint16_t m_LocalPort{}; + std::vector m_Received; +}; diff --git a/dCaptureTool/LiveImport.cpp b/dCaptureTool/LiveImport.cpp new file mode 100644 index 000000000..c4066d880 --- /dev/null +++ b/dCaptureTool/LiveImport.cpp @@ -0,0 +1,287 @@ +#include "LiveImport.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "ClientPackets.h" +#include "CaptureTools.h" +#include "MessageIdentifiers.h" +#include "MessageType/Client.h" +#include "PacketDecoder.h" +#include "ServiceType.h" +#include "ZCompression.h" + +namespace fs = std::filesystem; +using json = nlohmann::json; + +namespace { + constexpr uint16_t AUTH_PORT = 1001; + + struct Entry { + std::string name; + std::string data; + int64_t timeUs{}; + }; + + template + T Get(const std::string& data, size_t at) { + T value{}; + if (at + sizeof(T) <= data.size()) std::memcpy(&value, data.data() + at, sizeof(T)); + return value; + } + + int64_t DosTimeUs(uint16_t time, uint16_t date) { + std::tm tm{}; + tm.tm_year = ((date >> 9) & 0x7f) + 80; + tm.tm_mon = ((date >> 5) & 0x0f) - 1; + tm.tm_mday = date & 0x1f; + tm.tm_hour = (time >> 11) & 0x1f; + tm.tm_min = (time >> 5) & 0x3f; + tm.tm_sec = (time & 0x1f) * 2; + return static_cast(timegm(&tm)) * 1000000; + } + + // The entries of a zip file (stored or deflated; no zip64, no encryption) + bool ReadZip(const fs::path& path, std::vector& entries, std::string& error) { + std::ifstream file(path, std::ios::binary); + const std::string zip((std::istreambuf_iterator(file)), std::istreambuf_iterator()); + if (zip.size() < 22) { + error = "not a zip file"; + return false; + } + size_t end = std::string::npos; + for (size_t at = zip.size() - 22 + 1; at-- > 0 && zip.size() - at < 65557;) { + if (Get(zip, at) == 0x06054b50) { + end = at; + break; + } + } + if (end == std::string::npos) { + error = "no zip directory"; + return false; + } + const auto count = Get(zip, end + 10); + size_t at = Get(zip, end + 16); + for (uint16_t i = 0; i < count; i++) { + if (Get(zip, at) != 0x02014b50) { + error = "damaged zip directory"; + return false; + } + const auto flags = Get(zip, at + 8); + const auto method = Get(zip, at + 10); + const auto time = Get(zip, at + 12), date = Get(zip, at + 14); + const auto compressed = Get(zip, at + 20), size = Get(zip, at + 24); + const auto nameLength = Get(zip, at + 28), extraLength = Get(zip, at + 30), commentLength = Get(zip, at + 32); + const auto local = Get(zip, at + 42); + Entry entry; + entry.name = zip.substr(at + 46, nameLength); + entry.timeUs = DosTimeUs(time, date); + at += 46 + nameLength + extraLength + commentLength; + if (flags & 1) continue; // encrypted: left alone + if (Get(zip, local) != 0x04034b50) continue; + const size_t data = local + 30 + Get(zip, local + 26) + Get(zip, local + 28); + if (data + compressed > zip.size()) continue; + const std::string_view raw(zip.data() + data, compressed); + if (method == 0) entry.data = std::string(raw); + else if (method == 8) { + auto inflated = ZCompression::InflateRaw(raw, size); + if (!inflated) continue; + entry.data = std::move(*inflated); + } else continue; + entries.push_back(std::move(entry)); + } + return true; + } + + /** + * The live servers' CREATE_CHARACTER: a compressed LDF list with more types than DLU writes (so read here, not with + * ClientPackets::CreateCharacter): the object ID, name and character XML. + */ + bool ReadCreateCharacter(const std::string& bytes, LWOOBJID& id, std::string& name, std::string& xml) { + if (bytes.size() < 8 + 13) return false; + const auto compressedSize = Get(bytes, 8 + 9), size = Get(bytes, 8 + 5); + if (bytes[8 + 4] != 1 || 8 + 13 + static_cast(compressedSize) > bytes.size() || size > 64 * 1024 * 1024) return false; + std::vector out(size); + int32_t error{}; + if (ZCompression::Decompress(reinterpret_cast(bytes.data()) + 21, compressedSize, out.data(), size, error) != static_cast(size)) return false; + const std::string data(out.begin(), out.end()); + size_t at = 4; + for (uint32_t i = 0, count = Get(data, 0); i < count && at < data.size(); i++) { + const uint8_t keyBytes = static_cast(data[at++]); + std::u16string key(keyBytes / 2, u'\0'); + std::memcpy(key.data(), data.data() + at, keyBytes); + at += keyBytes; + const uint8_t type = static_cast(data[at++]); + switch (type) { + case 0: { // UTF-16 + const auto length = Get(data, at); + std::u16string value(length, u'\0'); + std::memcpy(value.data(), data.data() + at + 4, std::min(length * 2, data.size() - at - 4)); + if (key == u"name") name = GeneralUtils::UTF16ToWTF8(value); + at += 4 + length * 2; + break; + } + case 13: { // UTF-8 + const auto length = Get(data, at); + if (key == u"xmlData") xml = data.substr(at + 4, length); + at += 4 + length; + break; + } + case 1: case 3: case 5: at += 4; break; + case 7: at += 1; break; + case 4: case 8: at += 8; break; + case 9: + if (key == u"objid") id = Get(data, at); + at += 8; + break; + default: return id != 0; + } + } + return id != 0; + } + + // auth_traffic, char_traffic, world_traffic, world1_traffic, world2_traffic, ...: the order they were played in + int ZipOrder(const std::string& name) { + if (name.starts_with("auth")) return 0; + if (name.starts_with("char")) return 1; + static const std::regex world(R"(world(\d*)_traffic)"); + std::smatch match; + if (std::regex_search(name, match, world)) return 2 + (match[1].length() ? std::stoi(match[1]) : 0); + return 1000; + } +} + +namespace LiveImport { + std::vector FindScenarios(const fs::path& root) { + std::set folders; + std::error_code ec; + for (auto it = fs::recursive_directory_iterator(root, fs::directory_options::skip_permission_denied, ec); it != fs::recursive_directory_iterator(); it.increment(ec)) { + if (ec) break; + const auto name = it->path().filename().string(); + if (it->is_regular_file(ec) && name.ends_with("_traffic.zip")) folders.insert(it->path().parent_path()); + } + return { folders.begin(), folders.end() }; + } + + Result Import(const fs::path& scenario) { + Result result; + std::vector zips; + std::error_code ec; + if (fs::is_directory(scenario, ec)) { + for (const auto& entry : fs::directory_iterator(scenario, ec)) { + const auto name = entry.path().filename().string(); + if (entry.is_regular_file() && name.ends_with("_traffic.zip")) zips.push_back(entry.path()); + } + } else { + zips.push_back(scenario); + } + std::ranges::sort(zips, [](const fs::path& a, const fs::path& b) { return ZipOrder(a.filename().string()) < ZipOrder(b.filename().string()); }); + if (zips.empty()) { + result.error = "No *_traffic.zip files"; + return result; + } + + // Split packets come as their parts ("(1of81)", left out) and joined ("__joined_[...]") + static const std::regex packetName(R"(^(\d+)_(\d+)-(\d+)(?:_(\d+|joined))?_\[)"); + std::map> characters; // id -> name, xml + int64_t last = 0; + uint32_t seq = 0; + for (const auto& zip : zips) { + std::vector entries; + std::string error; + if (!ReadZip(zip, entries, error)) { + result.error = zip.filename().string() + ": " + error; + return result; + } + result.zips++; + struct Numbered { uint64_t index; uint32_t part; Entry* entry; uint16_t from; uint16_t to; }; + std::vector packets; + for (auto& entry : entries) { + std::smatch match; + const auto name = fs::path(entry.name).filename().string(); + if (entry.data.empty() || !std::regex_search(name, match, packetName)) { + result.skipped++; + continue; + } + const auto part = match[4].str(); + packets.push_back({ std::stoull(match[1]), part.empty() || part == "joined" ? 1u : static_cast(std::stoul(part)), &entry, + static_cast(std::stoul(match[2])), static_cast(std::stoul(match[3])) }); + } + std::ranges::sort(packets, [](const Numbered& a, const Numbered& b) { return a.index != b.index ? a.index < b.index : a.part < b.part; }); + + const bool auth = zip.filename().string().starts_with("auth"); + uint16_t zone = 0, instance = 0; + uint32_t clone = 0; + LWOOBJID character = 0; + for (const auto& p : packets) { + // The server is the end with the lower port (auth 1001, worlds 2000 and up; clients' ports are higher) + const uint16_t server = std::min(p.from, p.to), client = std::max(p.from, p.to); + CaptureBundle::Record record; + record.bytes = p.entry->data; + if (!PacketDecoder::Redact(record.bytes)) { + result.skipped++; + continue; + } + auto& h = record.header; + h.timeUs = last = std::max(last + 1000, p.entry->timeUs); + h.seq = ++seq; + h.source = static_cast(auth || server == AUTH_PORT ? eCaptureSource::AUTH : eCaptureSource::WORLD); + h.direction = static_cast(p.to == server ? ePacketDirection::RECEIVED : ePacketDirection::SENT); + h.peer = client; + h.bits = static_cast(record.bytes.size() * 8); + h.length = static_cast(record.bytes.size()); + + // Where this is, and whose: from the zone the server loads and the character it makes + const auto decoded = PacketDecoder::Decode(record.bytes, h.direction == static_cast(ePacketDirection::RECEIVED)); + if (decoded.name == "LOAD_STATIC_ZONE" && decoded.fields) { + zone = static_cast((*decoded.fields)["mapID"].get()); + instance = static_cast((*decoded.fields)["instanceID"].get()); + clone = (*decoded.fields)["cloneID"].get(); + } + if (decoded.name == "CREATE_CHARACTER") { + LWOOBJID id{}; + std::string name, xml; + if (ReadCreateCharacter(record.bytes, id, name, xml)) { + character = id; + characters[id] = { name, xml }; + } + } + h.zoneId = zone; + h.instanceId = instance; + h.cloneId = clone; + h.characterId = character; + result.bundle.records.push_back(std::move(record)); + result.packets++; + } + } + + auto& meta = result.bundle.meta; + meta["format"] = CaptureBundle::FORMAT_VERSION; + meta["origin"] = "live-2014"; + meta["scenario"] = scenario.filename().string(); + meta["server"] = { {"version", "LEGO Universe live servers"} }; + json zones = json::object(); + for (const auto& record : result.bundle.records) { + const auto decoded = PacketDecoder::Decode(record.bytes, CaptureTools::FromClient(record.header)); + if (decoded.name == "LOAD_STATIC_ZONE" && decoded.fields) zones[std::to_string((*decoded.fields)["mapID"].get())] = (*decoded.fields)["mapChecksum"]; + if (decoded.name == "VALIDATION" && decoded.fields) meta["fdbChecksum"] = (*decoded.fields)["fdbChecksum"]; + } + meta["zones"] = zones; + const auto symbols = CaptureTools::MakePortable(result.bundle); + json setup = json::array(); + static const std::regex account(R"( acct="[0-9]+")"); + for (const auto& [symbol, id] : symbols) { + const auto it = characters.find(id); + if (it == characters.end()) continue; + setup.push_back({ {"symbol", symbol}, {"placeholder", meta["ids"][symbol]["placeholder"]}, {"name", it->second.first}, + {"xml", std::regex_replace(it->second.second, account, "")} }); + } + meta["setup"] = { {"characters", setup} }; + return result; + } +} diff --git a/dCaptureTool/LiveImport.h b/dCaptureTool/LiveImport.h new file mode 100644 index 000000000..efad976e2 --- /dev/null +++ b/dCaptureTool/LiveImport.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include +#include + +#include "CaptureBundle.h" + +/** + * Converts the 2014 live captures (folders of _traffic.zip files, one packet per .bin entry named + * "_-[_]_[

]...bin") into packet bundles, so they replay like the server's + * own captures (docs/CaptureReplay.md). Only those zips are read: raw or encrypted captures (.pcap, key files) are + * left alone. Secrets are removed as when capturing (PacketDecoder::Redact), and the characters' own data + * (CREATE_CHARACTER) becomes the setup section. + */ +namespace LiveImport { + struct Result { + CaptureBundle::Bundle bundle; + size_t zips{}; + size_t packets{}; + size_t skipped{}; // entries that weren't packets, or secrets that didn't read + std::string error; + }; + + // One scenario: a folder of *_traffic.zip (auth, char, world, world1, world2, ... in that order), or one zip + Result Import(const std::filesystem::path& scenario); + + // Every folder under root that holds *_traffic.zip files + std::vector FindScenarios(const std::filesystem::path& root); +} diff --git a/dCaptureTool/Replayer.cpp b/dCaptureTool/Replayer.cpp new file mode 100644 index 000000000..4ff903414 --- /dev/null +++ b/dCaptureTool/Replayer.cpp @@ -0,0 +1,413 @@ +#include "Replayer.h" + +#include +#include +#include + +#include "AuthPackets.h" +#include "ClientPackets.h" +#include "CommonPackets.h" +#include "FakeClient.h" +#include "MessageIdentifiers.h" +#include "PacketDecoder.h" +#include "ServiceType.h" +#include "WorldPackets.h" +#include "eLoginResponse.h" + +using json = nlohmann::json; +using Record = CaptureBundle::Record; +using namespace std::chrono_literals; + +namespace { + struct Segment { + eCaptureSource source{}; + std::vector records; + bool characterSelect{}; + }; + + std::string NameOf(const Record& record) { return PacketDecoder::Decode(record.bytes, CaptureTools::FromClient(record.header)).name; } + std::string NameOf(const std::string& bytes) { return PacketDecoder::Decode(bytes, false).name; } + + template + std::string Bytes(const T& packet) { + RakNet::BitStream stream; + packet.WritePacket(stream); + return std::string(reinterpret_cast(stream.GetData()), stream.GetNumberOfBytesUsed()); + } + + template + bool Read(const std::string& bytes, T& packet) { + RakNet::BitStream stream(reinterpret_cast(const_cast(bytes.data())), static_cast(bytes.size()), false); + return packet.ReadHeader(stream) && packet.Deserialize(stream); + } + + void ReplaceAll(std::string& bytes, int64_t from, int64_t to) { + if (from == 0 || from == to) return; + char a[8], b[8]; + std::memcpy(a, &from, 8); + std::memcpy(b, &to, 8); + const std::string_view needle(a, 8); + for (size_t at = bytes.find(needle); at != std::string::npos; at = bytes.find(needle, at + 8)) std::memcpy(bytes.data() + at, b, 8); + } + + // A replica construction's object and LOT: [ID][bit][u16 network ID][i64 object][i32 LOT] + bool Construction(const std::string& bytes, int64_t& object, int32_t& lot) { + if (bytes.empty() || static_cast(bytes[0]) != ID_REPLICA_MANAGER_CONSTRUCTION) return false; + RakNet::BitStream stream(reinterpret_cast(const_cast(bytes.data())), static_cast(bytes.size()), false); + stream.IgnoreBytes(1); + bool flag{}; + uint16_t network{}; + return stream.Read(flag) && stream.Read(network) && stream.Read(object) && stream.Read(lot); + } + + std::vector Split(const std::vector& records) { + std::vector segments; + for (const auto& record : records) { + const auto source = static_cast(record.header.source); + if ((source != eCaptureSource::AUTH && source != eCaptureSource::WORLD) || (record.header.flags & (PacketRecordFlags::MASTER_LINK | PacketRecordFlags::GAP))) continue; + const bool handshake = CaptureTools::FromClient(record.header) && NameOf(record) == "VERSION_CONFIRM"; + if (segments.empty() || handshake || segments.back().source != source) segments.push_back({ source, {}, false }); + segments.back().records.push_back(&record); + const auto name = NameOf(record); + if (source == eCaptureSource::WORLD && (name == "CHARACTER_LIST_REQUEST" || name == "CHARACTER_LIST_RESPONSE" || name == "LOGIN_REQUEST")) segments.back().characterSelect = true; + } + return segments; + } + + class Run { + public: + Run(const CaptureBundle::Bundle& bundle, const Replayer::Options& options, Replayer::Result& result) : m_Bundle(bundle), m_Options(options), m_Result(result) {} + + void Go() { + auto records = m_Bundle.records; + CaptureTools::SortTimeline(records); + const auto segments = Split(records); + m_Result.connections = segments.size(); + if (segments.empty()) return Stop("The bundle has no client connections"); + + size_t first = 0; + if (segments.front().source == eCaptureSource::AUTH) { + if (!ReplaySegment(segments.front(), m_Options.host, m_Options.authPort)) return; + first = 1; + } else if (!Login()) { + return; + } + for (size_t i = first; i < segments.size(); i++) { + const auto& segment = segments[i]; + if (segment.source == eCaptureSource::AUTH) { + if (!ReplaySegment(segment, m_Options.host, m_Options.authPort)) return; + continue; + } + if (segment.characterSelect) { + if (!ReplaySegment(segment, m_WorldHost, m_WorldPort)) return; + continue; + } + // A zone: the target says where, after character select (done here if the recording didn't) + if (!m_TransferPort && !CharacterSelect()) return; + const auto port = m_TransferPort; + m_TransferPort = 0; + if (!ReplaySegment(segment, m_TransferHost, port)) return; + } + } + + private: + const CaptureBundle::Bundle& m_Bundle; + const Replayer::Options& m_Options; + Replayer::Result& m_Result; + std::vector m_Expected; + std::string m_UserKey; + std::string m_WorldHost, m_TransferHost; + uint16_t m_WorldPort{}, m_TransferPort{}; + std::map m_Objects; // recorded object -> the target's, learned from constructions + std::map> m_Recorded, m_Replayed; // constructions by LOT, in order + std::map m_Paired; + + void Stop(const std::string& why) { + if (m_Result.stoppedAt.empty()) m_Result.stoppedAt = why; + } + + std::string Host(const std::string& ip) const { + return ip.empty() || ip == "localhost" ? m_Options.host : ip; + } + + void Handshake(FakeClient& client) { + CommonPackets::ClientVersionConfirm version; + version.serviceType = ServiceType::CLIENT; + version.processID = 1; + version.port = client.GetLocalPort(); + client.Send(Bytes(version)); + client.WaitFor([](const auto& r) { return NameOf(r.bytes) == "VERSION_CONFIRM"; }, 0, 10s); + } + + bool ReadLoginResponse(const std::string& bytes) { + ClientPackets::LoginResponse response; + if (!Read(bytes, response)) return false; + if (response.responseCode != eLoginResponse::SUCCESS) { + Stop("The target refused the login (response " + std::to_string(static_cast(response.responseCode)) + ")"); + return false; + } + m_UserKey = response.userKey.GetAsString(); + m_WorldHost = Host(response.worldServerIP.string); + m_WorldPort = response.worldServerPort; + m_Result.loggedIn = true; + return true; + } + + // The recording has no login: log in on the target with the replay's account (not compared) + bool Login() { + FakeClient client; + if (!client.Connect(m_Options.host, m_Options.authPort, 10s)) { + Stop("Can't connect to auth at " + m_Options.host + ":" + std::to_string(m_Options.authPort)); + return false; + } + Handshake(client); + AuthPackets::LoginRequest login; + login.username = LUWString(m_Options.username); + login.password = LUWString(m_Options.password, 41); + client.Send(Bytes(login)); + const auto at = client.WaitFor([](const auto& r) { return NameOf(r.bytes) == "LOGIN_RESPONSE"; }, 0, 15s); + if (at < 0) { + Stop("No login response from the target"); + return false; + } + m_Result.notes.push_back("Logged in on the target first (the recording starts after the login)"); + return ReadLoginResponse(client.GetReceived()[at].bytes); + } + + // The recording starts in a zone: pick the character on the target (not compared) + bool CharacterSelect() { + FakeClient client; + if (!client.Connect(m_WorldHost, m_WorldPort, 10s)) { + Stop("Can't connect to character select at " + m_WorldHost + ":" + std::to_string(m_WorldPort)); + return false; + } + Handshake(client); + WorldPackets::Validation validation; + validation.username = LUWString(m_Options.username); + validation.sessionKey = LUWString(m_UserKey); + client.Send(Bytes(validation)); + client.Send(Bytes(WorldPackets::CharacterListRequest())); + auto at = client.WaitFor([](const auto& r) { return NameOf(r.bytes) == "CHARACTER_LIST_RESPONSE"; }, 0, 15s); + if (at < 0) { + Stop("No character list from the target"); + return false; + } + auto character = m_Options.character; + if (!character) { + // The bundle brought no character (it starts in the middle of a zone): make a plain one with the server's own code + ClientPackets::CharacterListResponse list; + if (Read(client.GetReceived()[at].bytes, list) && list.characters.empty()) { + WorldPackets::CharacterCreateRequest create; + create.name = LUWString(std::string("Replay")); + client.Send(Bytes(create)); + client.WaitFor([](const auto& r) { return NameOf(r.bytes) == "CHARACTER_CREATE_RESPONSE"; }, 0, 15s); + const auto from = client.GetReceived().size(); + client.Send(Bytes(WorldPackets::CharacterListRequest())); + at = client.WaitFor([](const auto& r) { return NameOf(r.bytes) == "CHARACTER_LIST_RESPONSE"; }, from, 15s); + if (at < 0 || !Read(client.GetReceived()[at].bytes, list)) { + Stop("Couldn't make a character on the target"); + return false; + } + m_Result.notes.push_back("The bundle has no character data: replayed with a new plain character"); + } + if (list.characters.empty()) { + Stop("No character to play on the target"); + return false; + } + character = list.characters.front().objectID; + } + WorldPackets::CharacterLoginRequest pick; + pick.playerID = character; + client.Send(Bytes(pick)); + at = client.WaitFor([](const auto& r) { return NameOf(r.bytes) == "TRANSFER_TO_WORLD"; }, 0, 180s); + if (at < 0) { + Stop("The target didn't send the character to a zone"); + return false; + } + ClientPackets::TransferToWorld transfer; + if (!Read(client.GetReceived()[at].bytes, transfer)) return false; + m_TransferHost = Host(transfer.serverIP.string); + m_TransferPort = transfer.serverPort; + m_Result.notes.push_back("Picked the character on the target first (the recording starts in a zone)"); + return true; + } + + // What goes out in place of a recorded packet: the target's account, session key and IDs + std::string Rewrite(const Record& record) { + auto bytes = record.bytes; + const auto name = NameOf(record); + if (name == "LOGIN_REQUEST" && record.header.source == static_cast(eCaptureSource::AUTH)) { + AuthPackets::LoginRequest login; + if (Read(bytes, login)) { + login.username = LUWString(m_Options.username); + login.password = LUWString(m_Options.password, 41); + bytes = Bytes(login); + } + } else if (name == "VALIDATION") { + WorldPackets::Validation validation; + if (Read(bytes, validation)) { + validation.username = LUWString(m_Options.username); + validation.sessionKey = LUWString(m_UserKey); + bytes = Bytes(validation); + } + } + for (const auto& [from, to] : m_Options.ids) ReplaceAll(bytes, from, to); + for (const auto& [from, to] : m_Objects) ReplaceAll(bytes, from, to); + return bytes; + } + + // Pairs the objects the target made with the recorded ones, by LOT and order + void Learn(const FakeClient& client, size_t& seen) { + const auto& received = client.GetReceived(); + for (; seen < received.size(); seen++) { + int64_t object{}; + int32_t lot{}; + if (Construction(received[seen].bytes, object, lot)) m_Replayed[lot].push_back(object); + } + for (auto& [lot, objects] : m_Recorded) { + auto& paired = m_Paired[lot]; + const auto& other = m_Replayed[lot]; + for (; paired < objects.size() && paired < other.size(); paired++) { + if (objects[paired] != other[paired]) m_Objects[objects[paired]] = other[paired]; + } + } + } + + bool ReplaySegment(const Segment& segment, const std::string& host, uint16_t port) { + FakeClient client; + if (port == 0 || !client.Connect(host, port, 15s)) { + Stop("Can't connect to " + host + ":" + std::to_string(port) + (segment.source == eCaptureSource::AUTH ? " (auth)" : " (world)")); + return false; + } + m_Result.connectionsReached++; + const auto zone = segment.records.front()->header.zoneId; + int64_t previous = segment.records.front()->header.timeUs; + size_t learned = 0; + bool ok = true; + // The answers recorded so far on this connection, by name, and the last one: a client packet goes out once the + // target has sent what the recorded server had sent before it (a real client reacts to those) + std::map recordedCounts; + std::string lastAnswer; + std::map slow; // answers the target didn't send in time once: not waited for long again + for (const auto* record : segment.records) { + if (!CaptureTools::FromClient(record->header)) { + lastAnswer = NameOf(*record); + recordedCounts[lastAnswer]++; + m_Expected.push_back(record); + int64_t object{}; + int32_t lot{}; + if (Construction(record->bytes, object, lot)) m_Recorded[lot].push_back(object); + continue; + } + const auto gap = std::min(static_cast((record->header.timeUs - previous) / 1000 / m_Options.speed), m_Options.maxGapMs); + previous = record->header.timeUs; + if (gap > 0) client.Idle(std::chrono::milliseconds(gap)); + // Wait for what a real client waits for before this + const auto name = NameOf(*record); + const auto waitFor = [&](const char* answer, std::chrono::seconds timeout) { + if (client.WaitFor([answer](const auto& r) { return NameOf(r.bytes) == answer; }, 0, timeout) < 0) { + m_Result.notes.push_back(std::string("No ") + answer + " from the target before " + name); + } + }; + if (name == "VALIDATION" || (name == "LOGIN_REQUEST" && segment.source == eCaptureSource::AUTH)) waitFor("VERSION_CONFIRM", 10s); + else if (name == "LOGIN_REQUEST") waitFor("CHARACTER_LIST_RESPONSE", 10s); + else if (name == "LEVEL_LOAD_COMPLETE") waitFor("LOAD_STATIC_ZONE", 30s); + if (!lastAnswer.empty()) { + const auto want = recordedCounts[lastAnswer]; + size_t have = 0; + const auto enough = [&](const auto& r) { return NameOf(r.bytes) == lastAnswer && ++have >= want; }; + if (client.WaitFor(enough, 0, slow[lastAnswer] ? 500ms : 10s) < 0 && !slow[lastAnswer]) { + slow[lastAnswer] = true; + m_Result.notes.push_back("Waited in vain for " + lastAnswer + " (" + std::to_string(want) + " recorded by then) before " + name); + } + } + Learn(client, learned); + if (!client.Pump()) { + Stop("The target closed the connection before " + name); + ok = false; + break; + } + client.Send(Rewrite(*record)); + m_Result.sent++; + } + // The last answers; and where to go next + if (ok) { + if (segment.source == eCaptureSource::AUTH) { + const auto at = client.WaitFor([](const auto& r) { return NameOf(r.bytes) == "LOGIN_RESPONSE"; }, 0, 15s); + if (at < 0 || !ReadLoginResponse(client.GetReceived()[at].bytes)) { + Stop(m_Result.stoppedAt.empty() ? "No login response from the target" : m_Result.stoppedAt); + ok = false; + } + } else { + // Until the target has been quiet a while (it may still be sending the zone) + for (size_t count = client.GetReceived().size(), rounds = 0; rounds < 30; rounds++) { + client.Idle(1s); + if (client.GetReceived().size() == count && rounds >= 3) break; + count = client.GetReceived().size(); + } + const auto at = client.WaitFor([](const auto& r) { return NameOf(r.bytes) == "TRANSFER_TO_WORLD"; }, 0, 1s); + if (at >= 0) { + ClientPackets::TransferToWorld transfer; + if (Read(client.GetReceived()[at].bytes, transfer)) { + m_TransferHost = Host(transfer.serverIP.string); + m_TransferPort = transfer.serverPort; + } + } + } + } + for (const auto& received : client.GetReceived()) { + Record record; + record.bytes = received.bytes; + record.header.timeUs = received.timeUs; + record.header.source = static_cast(segment.source); + record.header.direction = static_cast(ePacketDirection::SENT); + record.header.zoneId = zone; + record.header.length = static_cast(record.bytes.size()); + record.header.bits = record.header.length * 8; + m_Result.actual.push_back(std::move(record)); + } + m_Result.received += client.GetReceived().size(); + return ok; + } + + public: + std::vector Expected() const { + std::vector out; + for (const auto* record : m_Expected) out.push_back(*record); + return out; + } + }; +} + +namespace Replayer { + json Result::ToJson() const { + return { {"loggedIn", loggedIn}, {"connections", connections}, {"connectionsReached", connectionsReached}, {"sent", sent}, {"received", received}, + {"stoppedAt", stoppedAt}, {"notes", notes}, {"diff", diff.ToJson()} }; + } + + Result Replay(const CaptureBundle::Bundle& bundle, const Options& options) { + Result result; + Run run(bundle, options, result); + run.Go(); + const auto expected = run.Expected(); + result.diff = CaptureTools::Diff(expected, result.actual); + + // Zone data that differs is reported, not taken for a server difference + std::map recordedZones, replayedZones; + for (const auto& record : expected) { + const auto decoded = PacketDecoder::Decode(record.bytes, false); + if (decoded.name == "LOAD_STATIC_ZONE" && decoded.fields) recordedZones[(*decoded.fields)["mapID"].get()] = (*decoded.fields)["mapChecksum"]; + } + for (const auto& record : result.actual) { + const auto decoded = PacketDecoder::Decode(record.bytes, false); + if (decoded.name == "LOAD_STATIC_ZONE" && decoded.fields) replayedZones[(*decoded.fields)["mapID"].get()] = (*decoded.fields)["mapChecksum"]; + } + for (const auto& [zone, checksum] : recordedZones) { + const auto it = replayedZones.find(zone); + if (it != replayedZones.end() && it->second != checksum) { + result.notes.push_back("Zone " + std::to_string(zone) + " has other data on the target (checksum " + it->second.dump() + ", recorded " + checksum.dump() + ")"); + } + } + return result; + } +} diff --git a/dCaptureTool/Replayer.h b/dCaptureTool/Replayer.h new file mode 100644 index 000000000..2433e56d7 --- /dev/null +++ b/dCaptureTool/Replayer.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include +#include +#include + +#include "CaptureBundle.h" +#include "CaptureTools.h" +#include "json.hpp" + +/** + * Replays a bundle's client packets against a server with the fake client (docs/CaptureReplay.md) and compares the + * server's answers with the recorded ones. + * + * The recording is split into connections (each starts with the client's VERSION_CONFIRM). The replay logs in on + * the target's auth server itself when the recording has no login, and goes through character select itself when + * the recording starts in a zone. Before a packet goes out, what must differ on the target is filled in: the + * target account's name and password, the session key the target's auth gave, and the target's IDs for the + * characters (the bundle's placeholders). Timing follows the recording (sped up, and never more than a few seconds + * between packets), and the replay waits for the answers a real client waits for (the handshake, the character + * list, the zone to load, the world to transfer to). + */ +namespace Replayer { + struct Options { + std::string host{ "127.0.0.1" }; + uint16_t authPort{}; + std::string username; + std::string password; + double speed{ 4.0 }; + uint32_t maxGapMs{ 3000 }; + // Bundle placeholder -> the target's ID (the characters the setup made) + std::map ids; + int64_t character{}; // the target's ID of the bundle's first character (character select picks it) + }; + + struct Result { + bool loggedIn{}; + size_t connections{}; // recorded connections + size_t connectionsReached{}; // connections the replay got to + size_t sent{}; + size_t received{}; + std::string stoppedAt; // why it stopped early, if it did + std::vector notes; + std::vector actual; // what the target answered, as records + CaptureTools::DiffReport diff; + nlohmann::json ToJson() const; + }; + + Result Replay(const CaptureBundle::Bundle& bundle, const Options& options); +} diff --git a/dCaptureTool/Sandbox.cpp b/dCaptureTool/Sandbox.cpp new file mode 100644 index 000000000..3ef675182 --- /dev/null +++ b/dCaptureTool/Sandbox.cpp @@ -0,0 +1,271 @@ +#include "Sandbox.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "BinaryPathFinder.h" +#include "CaptureBundle.h" +#include "Database.h" +#include "dConfig.h" +#include "eGameMasterLevel.h" +#include "eObjectBits.h" +#include "FakeClient.h" +#include "Game.h" +#include "GeneralUtils.h" +#include "Logger.h" +#include "MigrationRunner.h" +#include "bcrypt/BCrypt.hpp" + +namespace fs = std::filesystem; +using json = nlohmann::json; + +namespace { + /** + * Settings every sandbox gets over the server's own files. The first value of a key counts, so each key's line + * is replaced where it is, and the ones the file doesn't have are added. + */ + void Append(const fs::path& file, const std::string& lines) { + std::map settings; + std::istringstream wanted(lines); + for (std::string line; std::getline(wanted, line);) { + const auto equals = line.find('='); + if (equals != std::string::npos) settings[line.substr(0, equals)] = line; + } + std::ifstream in(file); + std::string out; + for (std::string line; std::getline(in, line);) { + const auto equals = line.find('='); + const auto key = equals == std::string::npos || line.starts_with("#") ? "" : line.substr(0, equals); + const auto it = settings.find(key); + if (it != settings.end()) { + out += it->second + "\n"; + settings.erase(it); + } else { + out += line + "\n"; + } + } + in.close(); + out += "\n# Replay sandbox (docs/CaptureReplay.md)\n"; + for (const auto& [key, line] : settings) out += line + "\n"; + std::ofstream(file, std::ios::trunc) << out; + } + + pid_t Launch(const fs::path& dir, const fs::path& program, const fs::path& output, const std::vector& args = {}) { + const pid_t pid = fork(); + if (pid != 0) return pid; + // The child: its own process group, so the whole stack (master and what it starts) stops together + setpgid(0, 0); + // Settings can come from the environment too (dConfig): not in a sandbox + for (const auto* key : { "REPLAY_SANDBOX", "DATABASE_TYPE", "SQLITE_DATABASE_PATH", "MASTER_SERVER_PORT", "WORLD_PORT_START", "AUTH_SERVER_PORT", + "CHAT_SERVER_PORT", "MASTER_IP", "EXTERNAL_IP", "CLIENT_LOCATION", "MYSQL_HOST", "MYSQL_DATABASE", "DLU_CONFIG_DIR" }) unsetenv(key); + if (chdir(dir.c_str()) != 0) _exit(127); + const int fd = open(output.c_str(), O_WRONLY | O_CREAT | O_APPEND, 0644); + if (fd >= 0) { + dup2(fd, STDOUT_FILENO); + dup2(fd, STDERR_FILENO); + close(fd); + } + std::vector argv{ const_cast(program.c_str()) }; + for (const auto& arg : args) argv.push_back(const_cast(arg.c_str())); + argv.push_back(nullptr); + execv(program.c_str(), argv.data()); + _exit(127); + } +} + +namespace Sandbox { + std::unique_ptr Stack::Create(const Options& options, std::string& error) { + auto stack = std::unique_ptr(new Stack()); + stack->m_Options = options; + stack->m_Keep = options.keep; + std::error_code ec; + const auto stamp = std::to_string(std::chrono::system_clock::now().time_since_epoch().count() / 1000000) + "-" + std::to_string(getpid()); + stack->m_Dir = fs::absolute(options.root / ("sandbox-" + stamp)); + const auto& dir = stack->m_Dir; + if (!fs::create_directories(dir / "resServer", ec) || !fs::create_directories(dir / "logs", ec)) { + error = "Can't make " + dir.string(); + return nullptr; + } + // Binaries are copied: they find their settings and database next to themselves + for (const auto* name : { "MasterServer", "AuthServer", "ChatServer", "WorldServer" }) { + fs::copy_file(options.serverDir / name, dir / name, ec); + if (ec) { + error = "Can't copy " + (options.serverDir / name).string() + ": " + ec.message(); + return nullptr; + } + } + fs::copy_file(BinaryPathFinder::GetBinaryDir() / "CaptureTool", dir / "CaptureTool", ec); + for (const auto* name : { "migrations", "navmeshes", "vanity", "blocklist.dcf", "libmariadbcpp.so" }) { + if (fs::exists(options.serverDir / name)) fs::create_symlink(fs::absolute(options.serverDir / name), dir / name, ec); + } + for (const auto* name : { "sharedconfig.ini", "masterconfig.ini", "authconfig.ini", "chatconfig.ini", "worldconfig.ini" }) { + fs::copy_file(options.serverDir / name, dir / name, ec); + } + fs::copy_file(options.cdServer, dir / "resServer" / "CDServer.sqlite", ec); + if (ec) { + error = "Can't copy CDServer.sqlite from " + options.cdServer.string() + ": " + ec.message(); + return nullptr; + } + + const auto port = [&](int offset) { return std::to_string(options.basePort + offset); }; + Append(dir / "sharedconfig.ini", + "replay_sandbox=1\n" + "database_type=sqlite\n" + "sqlite_database_path=resServer/sandbox.sqlite\n" + "replay_live_sqlite_path=" + options.liveDatabase.string() + "\n" + "client_location=" + options.clientLocation.string() + "\n" + "external_ip=localhost\n" + "master_ip=localhost\n" + "bind_ip=127.0.0.1\n" + "master_server_port=" + port(0) + "\n" + "chat_server_port=" + port(20) + "\n" + "skip_account_creation=1\n" + "log_to_console=1\n"); + Append(dir / "masterconfig.ini", + "master_server_port=" + port(0) + "\n" + "world_port_start=" + port(100) + "\n" + "prestart_servers=1\n" + "enable_dashboard=0\n" + "enable_ugc_server=0\n"); + Append(dir / "authconfig.ini", "auth_server_port=" + port(10) + "\ndont_use_keys=1\n"); + Append(dir / "chatconfig.ini", "port=" + port(20) + "\nweb_server_enabled=0\n"); + Append(dir / "worldconfig.ini", "check_fdb=0\n"); + + // Read the settings back as the servers will (the first value of a key counts): never start a stack that could + // reach a normal server's ports or database + const auto first = [&](const char* file, const std::string& key) { + std::ifstream in(dir / file); + for (std::string line; std::getline(in, line);) if (line.starts_with(key + "=")) return line.substr(key.size() + 1); + return std::string(); + }; + if (first("sharedconfig.ini", "replay_sandbox") != "1" || first("sharedconfig.ini", "sqlite_database_path") != "resServer/sandbox.sqlite" || + first("sharedconfig.ini", "master_server_port") != port(0) || first("masterconfig.ini", "master_server_port") != port(0) || + first("masterconfig.ini", "world_port_start") != port(100) || first("authconfig.ini", "auth_server_port") != port(10)) { + error = "The sandbox's settings didn't take (see " + dir.string() + ")"; + return nullptr; + } + return stack; + } + + Stack::~Stack() { + Stop(); + std::error_code ec; + if (!m_Keep && !m_Dir.empty()) fs::remove_all(m_Dir, ec); + } + + bool Stack::Setup(const fs::path& bundle, const std::string& username, const std::string& password, bool characters, json& ids, std::string& error) { + const auto out = m_Dir / "setup.json"; + const pid_t pid = Launch(m_Dir, m_Dir / "CaptureTool", m_Dir / "logs" / "setup.out", + { "sandbox-setup", fs::absolute(bundle).string(), username, password, characters ? "1" : "0", out.string() }); + int status = 0; + waitpid(pid, &status, 0); + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + error = "Setting up the sandbox failed (see " + (m_Dir / "logs" / "setup.out").string() + ")"; + return false; + } + std::ifstream file(out); + ids = json::parse(file, nullptr, false); + if (ids.is_discarded()) { + error = "The sandbox setup wrote no result"; + return false; + } + return true; + } + + bool Stack::Start(std::string& error) { + m_Master = Launch(m_Dir, m_Dir / "MasterServer", m_Dir / "logs" / "master.out"); + // Auth answering is the sign the stack is up (master starts it after chat and the character select world) + const auto until = std::chrono::steady_clock::now() + std::chrono::seconds(120); + while (std::chrono::steady_clock::now() < until) { + int status = 0; + if (waitpid(m_Master, &status, WNOHANG) == m_Master) { + m_Master = 0; + error = "The sandbox's master server stopped (see " + (m_Dir / "logs").string() + ")"; + return false; + } + FakeClient probe; + if (probe.Connect("127.0.0.1", AuthPort(), std::chrono::milliseconds(500))) { + probe.Disconnect(); + // The character select world takes a moment longer + std::this_thread::sleep_for(std::chrono::seconds(3)); + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } + error = "The sandbox didn't start within 2 minutes (see " + (m_Dir / "logs").string() + ")"; + return false; + } + + void Stack::Stop() { + if (m_Master <= 0) return; + kill(-m_Master, SIGTERM); + const auto until = std::chrono::steady_clock::now() + std::chrono::seconds(15); + int status = 0; + while (waitpid(m_Master, &status, WNOHANG) == 0 && std::chrono::steady_clock::now() < until) std::this_thread::sleep_for(std::chrono::milliseconds(100)); + kill(-m_Master, SIGKILL); + waitpid(m_Master, &status, 0); + m_Master = 0; + } + + int SetupCommand(const fs::path& bundlePath, const std::string& username, const std::string& password, bool characters, const fs::path& out) { + Game::logger = new Logger((BinaryPathFinder::GetBinaryDir() / "logs" / "setup.log").string(), true, false); + Game::config = new dConfig("masterconfig.ini"); + // Only ever in a sandbox: Database::Connect refuses any database but the sandbox's own too + if (Game::config->GetValue("replay_sandbox") != "1") { + LOG("sandbox-setup only runs in a replay sandbox (replay_sandbox=1)"); + return 2; + } + CaptureBundle::Bundle bundle; + std::string error; + if (!CaptureBundle::Load(bundlePath, bundle, error)) { + LOG("%s", error.c_str()); + return 3; + } + try { + Database::Connect(); + MigrationRunner::RunMigrations(); + char salt[BCRYPT_HASHSIZE], hash[BCRYPT_HASHSIZE]; + bcrypt_gensalt(4, salt); + bcrypt_hashpw(password.c_str(), salt, hash); + Database::Get()->InsertNewAccount(username, hash, eGameMasterLevel::CIVILIAN); + const auto account = Database::Get()->GetAccountInfo(username); + if (!account) { + LOG("Couldn't make the replay account"); + return 4; + } + json ids = json::object(); + if (characters && bundle.meta.contains("setup")) { + auto range = Database::Get()->GetPersistentIdRange(); + for (const auto& character : bundle.meta["setup"].value("characters", json::array())) { + LWOOBJID id = static_cast(range.minID++); + GeneralUtils::SetBit(id, eObjectBits::CHARACTER); + auto name = character.value("name", character.value("symbol", std::string("replay"))); + for (int n = 2; Database::Get()->IsNameInUse(name); n++) name = character.value("name", std::string("replay")) + std::to_string(n); + ICharInfo::Info info; + info.name = name; + info.id = id; + info.accountId = account->id; + Database::Get()->InsertNewCharacter(info); + Database::Get()->InsertCharacterXml(id, character.value("xml", std::string())); + ids[character.value("symbol", std::string())] = { {"placeholder", character.value("placeholder", json()).is_string() ? character["placeholder"] : json(std::to_string(character.value("placeholder", 0LL)))}, + {"id", std::to_string(id)}, {"name", name} }; + LOG("Made character %s (%llu) for %s", name.c_str(), id, character.value("symbol", std::string()).c_str()); + } + } + std::ofstream(out) << ids.dump(1); + } catch (const std::exception& e) { + LOG("Sandbox setup failed: %s", e.what()); + return 5; + } + Game::logger->Flush(); + return 0; + } +} diff --git a/dCaptureTool/Sandbox.h b/dCaptureTool/Sandbox.h new file mode 100644 index 000000000..359fcc428 --- /dev/null +++ b/dCaptureTool/Sandbox.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "json.hpp" + +/** + * A replay sandbox (docs/CaptureReplay.md): a throwaway server stack for one replay, in a folder of its own. The + * server binaries are copied in (they read their settings and database from their own folder), the settings point + * at a fresh SQLite database inside that folder, ports that don't collide with a normal server, and + * replay_sandbox=1, with which every server refuses to start on any other database. The client files are shared + * read-only; CDServer.sqlite is copied. The folder is deleted afterwards unless it is kept. + */ +namespace Sandbox { + struct Options { + std::filesystem::path serverDir; // built servers (MasterServer, AuthServer, ChatServer, WorldServer, migrations, ...) + std::filesystem::path root; // sandboxes are made in here + std::filesystem::path clientLocation; // the game client's files (read only) + std::filesystem::path cdServer; // CDServer.sqlite to copy + std::filesystem::path liveDatabase; // the live SQLite database, which a sandbox must never be (optional) + uint16_t basePort{ 41000 }; + bool keep{}; + }; + + class Stack { + public: + static std::unique_ptr Create(const Options& options, std::string& error); + ~Stack(); + + // Makes the database (migrations), the replay's account and, from the bundle's setup section, its characters. + // `ids`: symbol -> {placeholder, id} of the characters made. + bool Setup(const std::filesystem::path& bundle, const std::string& username, const std::string& password, bool characters, + nlohmann::json& ids, std::string& error); + + // Starts master (which starts auth, chat and the character select world) and waits until auth answers + bool Start(std::string& error); + void Stop(); + + void Keep() { m_Keep = true; } + uint16_t AuthPort() const { return m_Options.basePort + 10; } + const std::filesystem::path& Dir() const { return m_Dir; } + + private: + Options m_Options; + std::filesystem::path m_Dir; + pid_t m_Master{}; + bool m_Keep{}; + }; + + // The sandbox-setup command, run by Stack::Setup inside the sandbox folder + int SetupCommand(const std::filesystem::path& bundle, const std::string& username, const std::string& password, bool characters, + const std::filesystem::path& out); +} diff --git a/dDatabase/GameDatabase/Database.cpp b/dDatabase/GameDatabase/Database.cpp index 82b594a25..abd9cd856 100644 --- a/dDatabase/GameDatabase/Database.cpp +++ b/dDatabase/GameDatabase/Database.cpp @@ -7,12 +7,37 @@ #include "SQLiteDatabase.h" #include "MySQLDatabase.h" +#include #include +#include + +#include "BinaryPathFinder.h" #pragma warning (disable:4251) //Disables SQL warnings namespace { GameDatabase* database = nullptr; + + /** + * A replay sandbox (replay_sandbox=1, docs/CaptureReplay.md) only ever uses its own SQLite file, inside its own + * folder, and never the live one: anything else and the server refuses to start. + */ + void CheckSandbox(const std::string& databaseType) { + if (!Game::config || Game::config->GetValue("replay_sandbox") != "1") return; + if (databaseType != "sqlite") throw std::runtime_error("A replay sandbox only runs on its own SQLite database (database_type=sqlite)"); + namespace fs = std::filesystem; + const auto folder = fs::weakly_canonical(BinaryPathFinder::GetBinaryDir()); + const auto path = fs::weakly_canonical(folder / Game::config->GetValue("sqlite_database_path")); + const auto relative = path.lexically_relative(folder); + if (Game::config->GetValue("sqlite_database_path").empty() || relative.empty() || *relative.begin() == "..") { + throw std::runtime_error("A replay sandbox's database must be inside its own folder (" + folder.string() + "), not " + path.string()); + } + const auto live = Game::config->GetValue("replay_live_sqlite_path"); + std::error_code ec; + if (!live.empty() && fs::exists(live, ec) && fs::equivalent(path, live, ec)) { + throw std::runtime_error("A replay sandbox must not use the live database " + live); + } + } } std::string Database::GetMigrationFolder() { @@ -34,6 +59,7 @@ void Database::Connect() { } const auto databaseType = GetMigrationFolder(); + CheckSandbox(databaseType); if (databaseType == "sqlite") database = new SQLiteDatabase(); else if (databaseType == "mysql") database = new MySQLDatabase(); @@ -55,6 +81,7 @@ GameDatabase* Database::Get() { std::unique_ptr Database::CreateConnection() { std::unique_ptr connection; + CheckSandbox(GetMigrationFolder()); if (GetMigrationFolder() == "sqlite") connection = std::make_unique(); else connection = std::make_unique(); connection->Connect(); diff --git a/dNet/CaptureTools.cpp b/dNet/CaptureTools.cpp index 1cb42d185..005349d1e 100644 --- a/dNet/CaptureTools.cpp +++ b/dNet/CaptureTools.cpp @@ -60,6 +60,13 @@ namespace { std::string NameOf(const Record& record) { return PacketDecoder::Decode(record.bytes, CaptureTools::FromClient(record.header)).name; } + + // How answers pair up in a diff: by name, and constructions by what they construct + std::string PairKey(const Record& record) { + const auto decoded = PacketDecoder::Decode(record.bytes, CaptureTools::FromClient(record.header)); + if (decoded.name == "ID_REPLICA_MANAGER_CONSTRUCTION" && decoded.fields) return decoded.name + " LOT " + std::to_string((*decoded.fields)["lot"].get()); + return decoded.name; + } } namespace CaptureTools { @@ -85,7 +92,7 @@ namespace CaptureTools { "timestamp", "stamps", "instanceID", "instanceId", "zoneInstance", "cloneID", "zoneClone", "serverIP", "serverPort", "worldServerIP", "worldServerPort", "processID", "port", // Per account on each server - "playerID", "targetID", "senderID", "username", + "playerID", "targetID", "senderID", "username", "networkID", }; return fields.contains(name); } @@ -207,25 +214,25 @@ namespace CaptureTools { }; const auto want = answers(expected), got = answers(actual); report.expected = want.size(); + std::vector gotNames; + gotNames.reserve(got.size()); + for (const auto* g : got) gotNames.push_back(PairKey(*g)); + // Each recorded answer pairs with the next unpaired replayed answer of the same name (order kept per name) + std::map> byName; + for (size_t i = 0; i < got.size(); i++) byName[gotNames[i]].push_back(i); + std::map next; std::vector used(got.size()); - size_t from = 0; for (const auto* w : want) { - const auto name = NameOf(*w); - // The next unused answer with the same name, looking a little ahead so one missing packet doesn't shift everything - size_t found = got.size(); - for (size_t i = from; i < got.size() && i < from + 200; i++) { - if (!used[i] && NameOf(*got[i]) == name) { - found = i; - break; - } - } + const auto name = PairKey(*w); + const auto& candidates = byName[name]; + auto& at = next[name]; + const size_t found = at < candidates.size() ? candidates[at++] : got.size(); if (found == got.size()) { report.missing++; report.missingByName[name]++; continue; } used[found] = true; - while (from < used.size() && used[from]) from++; auto a = PacketDecoder::Decode(w->bytes, false).fields.value_or(json()); auto b = PacketDecoder::Decode(got[found]->bytes, false).fields.value_or(json()); Strip(a); @@ -246,7 +253,7 @@ namespace CaptureTools { for (size_t i = 0; i < got.size(); i++) { if (used[i]) continue; report.extra++; - report.extraByName[NameOf(*got[i])]++; + report.extraByName[gotNames[i]]++; } return report; } diff --git a/dNet/PacketDecoder.cpp b/dNet/PacketDecoder.cpp index 29954f15a..48da75ec8 100644 --- a/dNet/PacketDecoder.cpp +++ b/dNet/PacketDecoder.cpp @@ -184,6 +184,11 @@ namespace { j = { {"selectedCharacterIndex", p.selectedCharacterIndex}, {"characters", characters} }; }), nullptr, Scrub(nullptr, [](auto& p) { for (auto& c : p.characters) { X(c.name); X(c.unapprovedName); } }) } }, + { K(S::CLIENT, MessageType::Client::CREATE_CHARACTER), { Make([](const auto& p, json& j) { + j = { {"objectID", Id(p.objectID)}, {"templateID", p.templateID}, {"name", GeneralUtils::UTF16ToWTF8(p.name)}, {"gmLevel", static_cast(p.gmLevel)}, + {"reputation", p.reputation}, {"propertyCloneID", p.propertyCloneID}, {"xmlBytes", p.xmlData.size()} }; + }), + nullptr, Scrub(nullptr, [](auto& p) { X(p.name); }) } }, { K(S::CLIENT, MessageType::Client::CHARACTER_CREATE_RESPONSE), { Make([](const auto& p, json& j) { j = { {"response", static_cast(p.response)} }; }) } }, @@ -257,6 +262,7 @@ namespace { { K(S::CLIENT, MessageType::Client::LOAD_STATIC_ZONE), Rewrite() }, { K(S::CLIENT, MessageType::Client::CHARACTER_LIST_RESPONSE), Rewrite() }, { K(S::CLIENT, MessageType::Client::CHARACTER_CREATE_RESPONSE), Rewrite() }, + { K(S::CLIENT, MessageType::Client::CREATE_CHARACTER), Rewrite() }, { K(S::CLIENT, MessageType::Client::TRANSFER_TO_WORLD), Rewrite() }, { K(S::CHAT, MessageType::Chat::GENERAL_CHAT_MESSAGE), Rewrite() }, { K(S::CHAT, MessageType::Chat::PRIVATE_CHAT_MESSAGE), Rewrite() }, @@ -328,6 +334,19 @@ namespace PacketDecoder { out.service = "RAKNET"; out.messageId = static_cast(bytes[0]); out.name = RakNetName(static_cast(bytes[0])); + // A construction starts with the object and its LOT: [ID][bit][u16 network ID][i64 object][i32 LOT] + if (out.messageId == ID_REPLICA_MANAGER_CONSTRUCTION) { + RakNet::BitStream stream(reinterpret_cast(const_cast(bytes.data())), static_cast(bytes.size()), false); + stream.IgnoreBytes(1); + bool flag{}; + uint16_t network{}; + LWOOBJID object{}; + int32_t lot{}; + if (stream.Read(flag) && stream.Read(network) && stream.Read(object) && stream.Read(lot)) { + out.objectId = object; + out.fields = json{ {"networkID", network}, {"objectID", Id(object)}, {"lot", lot} }; + } + } return out; } out.lu = true; diff --git a/docs/CaptureReplay.md b/docs/CaptureReplay.md new file mode 100644 index 000000000..01cb6e030 --- /dev/null +++ b/docs/CaptureReplay.md @@ -0,0 +1,213 @@ +# Packet capture and replay + +Staff record every packet of one account, one character, or everything, on all servers at once; play the recording +back on the dashboard; and replay it against a throwaway server to see how the server answers now. The same replay +takes the 2014 live captures, which makes them a conformance test for the server. The dashboard side is described in +[Dashboard.md](Dashboard.md#packet-captures); this document is how it works and the rules it follows. + +Captures, bundles and fixtures are player data. None of them is ever committed: `captures/`, `*.bundle` and +`tests/fixtures-local/` are in `.gitignore`. + +## Capturing + +Staff arm a capture on the Packet Captures page (`dev_message_inspector`; arming, stopping and exporting are audited). +The dashboard sends `MESSAGE_CAPTURE_CONTROL` with the appended `ARM` action to master, which passes it to every +world, auth and chat, and arms itself. The dashboard repeats it every 10 seconds (servers that started since, and +characters the account made since, are picked up) and sends `DISARM` at the end; every server also stops on its own at +the time limit. Up to 8 captures run at once; each has one bit (its *slot*) in every record's mask. + +| Target | What is recorded | +|---|---| +| Account | Its connections from its next login (or now, if online): auth, character select, every zone, chat, and its master link messages. Packets of a connection before it is known whose it is (the handshake, the login request, a world's session check) are held per connection and added once auth or the world names the account. | +| Character | The same, from when the character is picked in a world. | +| Everything | Every packet on every server's listening socket, and master's server-to-server traffic (not the dashboard's). | + +Where each server taps (`dNet/PacketCapture.*`, all on the server's main thread, since RakNet isn't thread safe): + +- received: `dServer::Receive` and `dServer::ReceiveFromMaster`; +- sent: a hook in `RakPeer::Send` (`g_RakPeerSendHook`, set only while armed), so replica constructions and + serializations that RakNet's ReplicaManager sends are seen too; +- who a packet belongs to: auth binds the connection when the login names the account, worlds when the session is + validated and when a character is picked; chat reads the player's object ID each chat packet starts with; master + link messages are matched by account name (session keys), by request (zone transfers answered later), by the + connection being handled when they are sent (player added and removed), and instance-wide ones (migration) go to + every captured player in that world. A capture's own traffic (`MESSAGE_CAPTURE_*`) is never recorded. + +### Secrets are never stored + +Packets that carry secrets are rewritten before they are recorded, per struct: the decoder registry +(`dNet/PacketDecoder.cpp`) declares a *redactor* for each such packet, which reads it with the struct's +`Deserialize`, blanks the secret fields and writes it again with `Serialize`. A packet that declares secrets but doesn't +read is dropped, never stored as is. Today that is: + +| Packet | Blanked | +|---|---| +| AUTH `LOGIN_REQUEST` | username and password | +| CLIENT `LOGIN_RESPONSE` | user key (the session key), CDN key | +| WORLD `VALIDATION` | session key | +| MASTER `SET_SESSION_KEY`, `SESSION_KEY_RESPONSE`, `NEW_SESSION_ALERT` | session key | + +Auth records only the handshake and the login request and response; anything else auth sees is left out. A new packet +with a secret opts in by adding its redactor next to its decoder. `PacketCaptureTests` checks that a captured login +and session never contain the test account's password, user key or session key. + +*Why not start capturing at character select?* The login is where most "can't log in" reports happen, and its +response code, stamps and timing are what staff need. With redaction by struct there is nothing secret left in it, +and the replay fills in its own account and session key anyway, so recording it costs nothing. Everything from the +world's validation on is recorded the same way. + +### Buffering, flushing and overhead + +Nothing is written or sent per packet. Each server appends records to one preallocated chunk; the chunk is sealed +when it reaches `capture_flush_bytes` (default 256 KB) or `capture_flush_interval_ms` has passed (default 1000), and +sealed chunks are sent to master (master sends its own straight to the dashboard) from the main loop. While master +can't take them they are kept up to `capture_buffer_max_mb` (default 16); past that the oldest are dropped, the next +batch says how many, and the dashboard writes a gap marker ("N packets lost here"). All three are shared settings +(Settings, Packet capture). With nothing armed, a received packet costs one flag check and a sent one a null check. + +The dashboard appends each batch to the capture's file with one write, and saves the session row (counts, end) every +5 seconds. Measured by `PacketCaptureTest.OverheadOfCapturingEverything` (one core, unoptimised build, 300,000 +position updates with EVERYTHING armed): + +| | per packet | throughput | +|---|---|---| +| server tap (record, redact check, buffer) | about 580 ns | about 1.7 million packets/s, 173 MB/s of records | +| dashboard: append batch to file | about 43 ns | | +| dashboard: a SQLite row per packet, one transaction per batch (for comparison) | about 3,500 ns | | + +A busy world sends a few thousand packets a second, so capturing everything there costs well under 1% of a core. +Appending to a file is about 80 times cheaper than a database row per packet, so packets go to files and the +database keeps only the session row (the game message inspector keeps its rows as before). + +## The bundle format + +One format for the dashboard's capture files, exported bundles and converted live captures (`dNet/CaptureBundle.h`): + +``` +"DLUBNDL1" 8 bytes: the format and its version +u32 length little endian +metadata UTF-8 JSON, `length` bytes +records to the end of the file +``` + +Each record is a 52 byte little-endian header (`PacketRecordHeader` in `dNet/PacketRecord.h`: time in µs, per-server +sequence, capture mask, source server, direction, flags, peer, account, character, zone, instance, clone, full size in +bits, stored length) followed by the packet's bytes exactly as they went over RakNet (up to 256 KB each; longer ones +are cut and flagged). Flags: master link, broadcast, cut, gap. + +Metadata keys: + +| Key | | +|---|---| +| `format` | 1 | +| `origin` | `dlu-capture` or `live-2014` | +| `server` | `version`, `commit` of the server that recorded it | +| `target`, `captureId`, `startedAt`, `exportedAt`, `scenario` | where it came from | +| `zones` | map id -> `mapChecksum` from its `LOAD_STATIC_ZONE`s | +| `fdbChecksum` | the client data checksum from `VALIDATION` | +| `portable`, `anonymised` | see below | +| `ids` | `char#1`, `account#1`, ... -> placeholder | +| `setup.characters` | per character: `symbol`, `placeholder`, `name`, `xml` (its saved character XML, without the account) | + +## Portability + +A bundle made on one server replays on another (a copy, a fresh install, a friend's server): + +- **IDs are symbolic.** Exporting replaces each captured character's object ID, wherever it appears in a packet's + bytes and in the headers, with a placeholder (`0x1FEDC00000000000 + n`, listed as `char#n` in `ids`); accounts + become `account#n`. The replay maps placeholders to the IDs the target gave the characters it made. Object IDs the + server makes (spawned objects, loot) are learned during the replay by pairing the target's replica constructions + with the recorded ones by LOT and order. +- **Setup travels with it.** `setup.characters` holds what the target needs to make the characters: their saved + XML (appearance, level, stats, inventory, missions, flags), from the database for DLU captures and from + `CREATE_CHARACTER` for live ones. Account names and secrets are never in a bundle; the replay uses its own account. +- **Mismatches are reported, not diffed.** The bundle names the server version and commit it was recorded on, and the + zone and client data checksums. The replay report lists zones whose checksum differs on the target, so different + data isn't read as a server bug. +- **Anonymised** bundles (`Export anonymised`, `CaptureTool anonymise`) also replace character names and what players + typed (chat, whispers, character names in lists) with as many `x`, so packet sizes stay the same. + +## Replaying + +`CaptureTool` (built next to the servers) replays bundles: + +``` +CaptureTool replay ... --client [--cdserver ] [--mode setup|as-is] + [--speed 4] [--port 41000] [--sandbox-root ] [--keep | --keep-on-failure] [--report ] +CaptureTool import-live convert live captures +CaptureTool anonymise make a fixture +CaptureTool info|decode look inside +``` + +### The sandbox + +Every replay runs in its own sandbox and never touches a real game database: + +- The tool makes a folder (under `--sandbox-root`, default the system's temporary folder), copies the server binaries + into it (they read their settings and database from their own folder), links the migrations and navmeshes, and + copies `CDServer.sqlite`. The game client's files are shared read-only. +- The settings are rewritten there: `replay_sandbox=1`, `database_type=sqlite`, a fresh + `sqlite_database_path=resServer/sandbox.sqlite`, the live database's path as `replay_live_sqlite_path`, ports from + `--port` on (master, auth +10, chat +20, worlds +100), prestarted servers, no dashboard. The tool reads the + settings back as the servers will and refuses to start if any of them didn't take, and clears the environment + variables that could override them. +- **The sandbox database is always SQLite**, a new file per replay, whatever the source or target server normally + runs on; there are no throwaway MySQL schemas. +- **Enforced by the servers**: with `replay_sandbox=1` every server refuses to connect to a database that isn't SQLite, + isn't inside its own folder, or is the file named by `replay_live_sqlite_path` (`Database::Connect`). +- The tool runs itself inside the sandbox (`sandbox-setup`, which also refuses to run without `replay_sandbox=1`) to + apply the migrations, make the replay account and, in `setup` mode, the bundle's characters; then starts master and + waits for auth. Afterwards the whole stack is stopped (one process group) and the folder deleted (`--keep`, + `--keep-on-failure` keep it). Nothing from a sandbox is merged anywhere. +- `replay-target` replays against a server you run yourself; it refuses unless given `--i-know-this-is-not-a-sandbox`, + and says loudly that it isn't one. Never point it at a live server. + +### The fake client + +A headless RakNet client (`dCaptureTool/FakeClient.*`). The recording is split into connections (each starts with +the client's `VERSION_CONFIRM`). It logs in on the target itself when the recording has no login, and picks the +character itself when the recording starts in a zone (with a new plain character, made by the server's own character +creation, when the bundle has no character data). Before each client packet goes out it fills in what must differ: +the target account and password, the session key the target's auth gave, and the target's IDs. Timing follows the +recording (4 times faster by default, at most 3 seconds between packets), and it waits for what a real client waits +for: the handshake answer, the character list, the zone, and, before each packet, the answer the recorded server had +sent just before it (10 seconds the first time; an answer the target never sends isn't waited for again). + +### The diff + +Only what the server answered (auth and world packets to the client) is compared. Each recorded answer pairs with the +target's next answer of the same name (constructions: the same LOT). Paired answers compare by their decoded fields, +leaving out what legitimately differs between runs: object and request IDs, handles, timestamps and stamps, instance +and clone IDs, server addresses and ports, player IDs and account names (`CaptureTools::IsVolatileField`); packets +without decoded fields compare by size. The report counts same, different, missing and extra answers by name, with +examples, plus notes (answers waited for in vain, zone data that differs). + +## Live captures + +`CaptureTool import-live ` converts every folder of `*_traffic.zip` under `` (the 2014 +captures as extracted from the packet capture archives: one packet per `.bin`, named +`_-[_|_joined]_[
]...bin`) into one bundle per folder, zips in play order +(auth, char, world, world1, ...). Split packets are taken from their joined file. Only those zips are read: raw +`.pcap` files and encrypted captures (key files, XML exports) are left alone. Secrets are removed as when capturing, +and `CREATE_CHARACTER` gives the setup section. Converted bundles stay local like any other. + +### Results against this branch + + + +## Local fixtures + +Export a capture anonymised (or `CaptureTool anonymise`) and put it in `tests/fixtures-local/` (never committed). +`CaptureFixtureTests.RecordedPacketsRoundTrip` (in `dGameTests`) reads every packet of every fixture whose struct the +decoder registry knows and checks it writes back to the same bytes; without fixtures it is skipped. + +## To check in game + +- Arm an account capture, log in with a real client: the auth, character select and zone packets appear on one + timeline, the login request shows a blank username and password, and chat (a whisper, a friend request) shows up + from the chat server. +- Arm a character capture before picking another character of the same account: nothing is recorded until the + captured character is picked. +- Arm everything on a busy test server for a minute: no stutter; the capture's size grows about once a second. +- Play a capture with movement back and open World 3D: the player moves with the playhead. +- Export a bundle, replay it with `CaptureTool replay`, and open a kept sandbox's logs. diff --git a/docs/Dashboard.md b/docs/Dashboard.md index c965ba925..b44de75ee 100644 --- a/docs/Dashboard.md +++ b/docs/Dashboard.md @@ -1775,6 +1775,33 @@ retention. The **Message capture pruning** task applies them every night (Tasks deleted. Messages are stored as the world captured them (bytes plus the fields it decoded), so decoders added to the world server later only apply to new captures. +#### Packet captures + +**Packet captures** (`/inspector/packets`, the **Packet captures** button on the inspector, same permission) record +whole packets, not only game messages, on every server at once, on one timeline: auth (the handshake and the login), +chat (friends, teams, whispers), every world (character list, creation and login, zone loading, position updates, +game messages, replica constructions and serializations, routed chat) and the server-to-server messages that belong to +the player (session keys, zone transfers, player added and removed, instance migration). Details, the file format, +limits and the replay are in [CaptureReplay.md](CaptureReplay.md). + +- **Arm** a capture for an **account** (from its next login, or at once if it is online; every character), one + **character** (from when it is picked in a world) or **everything** (all traffic on all servers; one at a time). + The picker searches as you type: part of an account or character name, or a pasted account ID or character object + ID; online ones are marked. Captures run 1 to 15 minutes and stop by themselves; at most 8 run at once. Arming, + stopping and exporting are audited. +- Passwords, session keys and user keys are never recorded: the servers blank them before a packet is kept. +- The viewer plays a capture back: **Play**/**Pause** (space), speed, and a slider to seek; packets appear in order up + to the playhead. Filter by name or server; click a packet for its decoded fields (from the server's own packet + structs) and its bytes. Game messages show their names; their fields are decoded by the game message inspector. +- **World 3D** opens the captured movement in World 3D's replay (needs `players_history`); while the capture page + plays, its playhead drives World 3D. +- **Export bundle** downloads a portable bundle for the capture tool's replay; **Export anonymised** also replaces + character names and chat, for a local test fixture. + +Packet captures are saved like game message captures (same list, same retention settings; a **Packets** badge marks +them), with their packets in a file under `capture_dir` (default `captures`, next to the dashboard) instead of the +database. How often servers send what they recorded is set under Settings, **Packet capture**. + ### CDClient browser **CDClient Browser** (`dev_cdclient`) is a raw viewer for the game's CDClient database (`resServer/CDServer.sqlite`), diff --git a/docs/PacketArchitecture.md b/docs/PacketArchitecture.md index 33fd6d574..30f2d3a41 100644 --- a/docs/PacketArchitecture.md +++ b/docs/PacketArchitecture.md @@ -45,6 +45,7 @@ The frozen oracles in `tests/**/Legacy/` still use the macros verbatim through t | `EntityManager` | `ID_REPLICA_MANAGER_CONSTRUCTION`/`SERIALIZE`/`DESTRUCTION` headers written before the components | Replica serialization, out of scope (see below). | | `dGame/dBehaviors/*`, the `sBitStream` of skill messages | Behavior bit streams | The skill payload is its own format, carried as bytes inside the skill structs. | | `MessageInspector` | Copies the payload bytes of sent/received game messages | A capture tap; the header is read with `NetGameMsg::ReadPacketHeader`. | +| `PacketCapture`, `RakPeer::Send` hook | Copies whole packets for packet captures ([CaptureReplay.md](CaptureReplay.md)) | A capture tap; reads only the 8 byte LU header, and rewrites packets with secrets through their structs (`PacketDecoder::Redact`). | Out of scope: replica/component serialization (`Component::Serialize`) and LDF/AMF, which are separate formats with their own tests.