From 5e40166cbd0e560b54b1aa3e6c211b26455aff26 Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Wed, 30 Sep 2026 04:23:32 -0500 Subject: [PATCH] feat(capture): read replica constructions, serializations and destructions ReplicaDecoder mirrors Entity::WriteBaseReplicaData and each component's Serialize, keeps what each network ID is per world instance, and shows the rest as bits when no layout fits. CaptureTool links the game: game messages are decoded (and compared in replays), and decode --cdserver reads replica packets. Co-Authored-By: Claude Opus 5.5 --- dCaptureTool/CMakeLists.txt | 4 +- dCaptureTool/CaptureTool.cpp | 47 +- dGame/dUtilities/CMakeLists.txt | 1 + dGame/dUtilities/ReplicaDecoder.cpp | 832 ++++++++++++++++++++++++++++ dGame/dUtilities/ReplicaDecoder.h | 61 ++ dNet/CaptureTools.cpp | 9 +- dNet/CaptureTools.h | 9 +- 7 files changed, 954 insertions(+), 9 deletions(-) create mode 100644 dGame/dUtilities/ReplicaDecoder.cpp create mode 100644 dGame/dUtilities/ReplicaDecoder.h diff --git a/dCaptureTool/CMakeLists.txt b/dCaptureTool/CMakeLists.txt index c1c202918..648080bbf 100644 --- a/dCaptureTool/CMakeLists.txt +++ b/dCaptureTool/CMakeLists.txt @@ -2,4 +2,6 @@ add_executable(CaptureTool "CaptureTool.cpp" "FakeClient.cpp" "LiveImport.cpp" "Replayer.cpp" "Sandbox.cpp") target_include_directories(CaptureTool PRIVATE ${PROJECT_SOURCE_DIR}/dServer) target_compile_definitions(CaptureTool PRIVATE PROJECT_VERSION="\"${PROJECT_VERSION}\"") -target_link_libraries(CaptureTool ${COMMON_LIBRARIES} bcrypt dServer) +# The game's libraries: game messages are read and compared with the game's own message structs +target_link_libraries(CaptureTool ${COMMON_LIBRARIES} bcrypt dServer + dScripts dGameBase dComponents dUtilities dGameMessages dInventory dGame dChatFilter dZoneManager dPhysics Detour Recast tinyxml2 dWorldServer dNavigation) diff --git a/dCaptureTool/CaptureTool.cpp b/dCaptureTool/CaptureTool.cpp index 3ccacf9f8..9a6186454 100644 --- a/dCaptureTool/CaptureTool.cpp +++ b/dCaptureTool/CaptureTool.cpp @@ -16,7 +16,11 @@ #include "CaptureBundle.h" #include "CaptureTools.h" #include "LiveImport.h" +#include "GameMessageDecoder.h" #include "PacketDecoder.h" +#include "CDClientDatabase.h" +#include "MessageIdentifiers.h" +#include "ReplicaDecoder.h" #include "Replayer.h" #include "Sandbox.h" @@ -37,6 +41,15 @@ namespace Game { dConfig* config = nullptr; Game::signal_t lastSignal = 0; std::mt19937 randomEngine; + // Also defined by every program that links the game's libraries (it decodes game messages with the game's own + // message structs); the dashboard runs no game, so they stay empty + dChatFilter* chatFilter = nullptr; + AssetManager* assetManager = nullptr; + RakPeerInterface* chatServer = nullptr; + SystemAddress chatSysAddr; + EntityManager* entityManager = nullptr; + dZoneManager* zoneManager = nullptr; + std::string projectVersion = PROJECT_VERSION; } namespace { @@ -44,7 +57,8 @@ namespace { std::cout << "CaptureTool: packet bundles (docs/CaptureReplay.md)\n" " info what is in a bundle\n" - " decode [--fields] [--limit N] its packets on one timeline\n" + " decode [--fields] [--limit N] [--cdserver ]\n" + " its packets on one timeline (replica packets too with --cdserver)\n" " anonymise a test fixture: names and chat replaced, IDs placeholders\n" " import-live convert live captures (every folder of *_traffic.zip under )\n" " replay ... [options] replay against a fresh sandbox stack per bundle and compare\n" @@ -113,16 +127,39 @@ namespace { return 0; } - int Decode(const std::string& path, bool fields, size_t limit) { + int Decode(const std::string& path, bool fields, size_t limit, const std::string& cdServer) { CaptureBundle::Bundle bundle; if (!LoadBundle(path, bundle)) return 1; CaptureTools::SortTimeline(bundle.records); + // Replica packets need the components of each LOT, from the CDClient + ReplicaDecoder::ComponentTable components; + if (!cdServer.empty()) { + try { + CDClientDatabase::Connect(cdServer); + ReplicaDecoder::LoadComponentTable(components); + } catch (const std::exception& e) { + std::cerr << "Can't read " << cdServer << ": " << e.what() << "\n"; + return 1; + } + } + ReplicaDecoder::Session replicas(components); + size_t constructions = 0, unmatched = 0; const auto start = bundle.records.empty() ? 0 : bundle.records.front().header.timeUs; for (size_t i = 0; i < bundle.records.size() && i < limit; i++) { - const auto j = CaptureTools::RecordJson(bundle.records[i], i, start, fields); + const auto& record = bundle.records[i]; + std::optional replica; + if (!cdServer.empty() && !(record.header.flags & PacketRecordFlags::GAP) && !CaptureTools::FromClient(record.header)) { + replica = replicas.Decode(record.bytes, CaptureTools::ReplicaConnection(record.header)); + if (replica && !record.bytes.empty() && static_cast(record.bytes[0]) == ID_REPLICA_MANAGER_CONSTRUCTION) { + constructions++; + if (replica->contains("(layout did not match)")) unmatched++; + } + } + const auto j = CaptureTools::RecordJson(record, i, start, fields, replica ? &*replica : nullptr); std::printf("%7zu %10.3f %-12s %-18s %s%s\n", i, j["t"].get() / 1000.0, (j.value("from", std::string()) + ">" + j.value("to", std::string())).c_str(), j.value("source", std::string()).c_str(), j.value("name", std::string()).c_str(), fields && j.contains("fields") ? (" " + j["fields"].dump()).c_str() : ""); } + if (!cdServer.empty()) std::printf("%zu constructions, %zu whose components didn't read exactly\n", constructions, unmatched); return 0; } @@ -298,6 +335,8 @@ namespace { } int main(int argc, char** argv) { + // Game messages are read with the game's own message structs + PacketDecoder::SetGameMessageDecoder(GameMessageDecoder::Decode); const std::vector args(argv, argv + argc); if (args.size() < 2) { Usage(); @@ -305,7 +344,7 @@ int main(int argc, char** argv) { } const auto& command = args[1]; if (command == "info" && args.size() >= 3) return Info(args[2]); - if (command == "decode" && args.size() >= 3) return Decode(args[2], Flag(args, "--fields"), std::stoul(Arg(args, "--limit", "1000000"))); + if (command == "decode" && args.size() >= 3) return Decode(args[2], Flag(args, "--fields"), std::stoul(Arg(args, "--limit", "1000000")), Arg(args, "--cdserver", "")); if (command == "anonymise" && args.size() >= 4) return Anonymise(args[2], args[3]); if (command == "import-live" && args.size() >= 4) return ImportLive(args[2], args[3]); if (command == "replay") return Replay(args); diff --git a/dGame/dUtilities/CMakeLists.txt b/dGame/dUtilities/CMakeLists.txt index 05115b9ba..7b762bae4 100644 --- a/dGame/dUtilities/CMakeLists.txt +++ b/dGame/dUtilities/CMakeLists.txt @@ -17,6 +17,7 @@ set(DGAME_DUTILITIES_SOURCES "BrickDatabase.cpp" "MessageInspector.cpp" "ObjectIDManager.cpp" "PlayerReports.cpp" + "ReplicaDecoder.cpp" "Preconditions.cpp" "SlashCommandHandler.cpp" "UgcManifest.cpp" diff --git a/dGame/dUtilities/ReplicaDecoder.cpp b/dGame/dUtilities/ReplicaDecoder.cpp new file mode 100644 index 000000000..93b6024d4 --- /dev/null +++ b/dGame/dUtilities/ReplicaDecoder.cpp @@ -0,0 +1,832 @@ +#include "ReplicaDecoder.h" + +#include +#include +#include +#include + +#include "BitStream.h" +#include "CDClientDatabase.h" +#include "eReplicaComponentType.h" +#include "GeneralUtils.h" +#include "magic_enum.hpp" +#include "MessageIdentifiers.h" + +namespace { + using json = nlohmann::json; + using enum eReplicaComponentType; + + // Reads values in order; the first read past the end marks the reader as failed and every later read gives 0 + struct Reader { + RakNet::BitStream& stream; + bool ok{ true }; + + template T Get() { + T value{}; + if (ok && !stream.Read(value)) ok = false; + return value; + } + bool Bit() { return Get(); } + std::string Id() { return std::to_string(Get()); } + json Point() { + const auto x = Get(), y = Get(), z = Get(); + return json::array({ x, y, z }); + } + // As the server writes them: x, y, z, w + json Rotation() { + const auto x = Get(), y = Get(), z = Get(), w = Get(); + return json::array({ x, y, z, w }); + } + // A glm quaternion written whole: w, x, y, z + json RawRotation() { + const auto w = Get(), x = Get(), y = Get(), z = Get(); + return json::array({ x, y, z, w }); + } + template std::string WideText() { + const auto length = Get(); + std::u16string text; + for (Length i = 0; ok && i < length; i++) text += static_cast(Get()); + return GeneralUtils::UTF16ToWTF8(text); + } + template std::string Text() { + const auto length = Get(); + std::string text; + for (Length i = 0; ok && i < length; i++) text += static_cast(Get()); + return text; + } + }; + + template + json Enum(uint64_t value) { + const auto name = magic_enum::enum_name(static_cast(value)); + return name.empty() ? json(value) : json(std::string(name) + " (" + std::to_string(value) + ")"); + } + + // LDF entries as the client reads them: "key=type:value" + json ReadLdfEntries(Reader& r, int32_t count) { + json out = json::array(); + for (int32_t i = 0; r.ok && i < count && i < 4096; i++) { + const auto keyBytes = r.Get(); + std::u16string key; + for (uint8_t c = 0; r.ok && c < keyBytes / 2; c++) key += static_cast(r.Get()); + const auto type = r.Get(); + std::string value; + switch (type) { + case 0: value = r.WideText(); break; + case 1: value = std::to_string(r.Get()); break; + case 3: value = std::to_string(r.Get()); break; + case 4: value = std::to_string(r.Get()); break; + case 5: value = std::to_string(r.Get()); break; + case 7: value = std::to_string(r.Get()); break; + case 8: value = std::to_string(r.Get()); break; + case 9: value = std::to_string(r.Get()); break; + case 13: value = r.Text(); break; + default: r.ok = false; break; + } + out.push_back(GeneralUtils::UTF16ToWTF8(key) + "=" + std::to_string(type) + ":" + value); + } + return out; + } + + // u32 size, u8 compressed, then the entries (or the compressed bytes, which are shown as their size) + json ReadLdf(Reader& r) { + const auto size = r.Get(); + const auto compressed = r.Get(); + if (compressed) { + const auto compressedSize = r.Get(); + for (uint32_t i = 0; r.ok && i < compressedSize; i++) r.Get(); + return json{ {"compressed", true}, {"size", size}, {"compressedSize", compressedSize} }; + } + return ReadLdfEntries(r, r.Get()); + } + + // Activity user info: object ID and 10 values each + json ActivityPlayers(Reader& r) { + json players = json::array(); + const auto count = r.Get(); + for (uint32_t i = 0; r.ok && i < count && i < 256; i++) { + json values = json::array(); + const auto player = r.Id(); + for (int v = 0; v < 10; v++) values.push_back(r.Get()); + players.push_back({ {"player", player}, {"values", values} }); + } + return players; + } + + // A part the server never writes (only live did), whose layout isn't known here: stop instead of guessing + void NotRead(Reader& r, json& j, const char* what) { + j["(" + std::string(what) + " present, not read)"] = true; + r.ok = false; + } + + // PhysicsComponent::Serialize + void Position(Reader& r, json& j) { + if (r.Bit()) { + j["position"] = r.Point(); + j["rotation"] = r.Rotation(); + } + } + + using ComponentReader = std::function& components)>; + + // One reader per component, each mirroring the component's Serialize(bIsInitialUpdate) + const std::map& Readers() { + static const std::map readers{ + { POSSESSABLE, [](Reader& r, json& j, bool, const auto&) { + if (!r.Bit()) return; + if (r.Bit()) j["possessor"] = r.Id(); + if (r.Bit()) j["animationFlag"] = r.Get(); + j["immediatelyDepossess"] = r.Bit(); + } }, + { MODULE_ASSEMBLY, [](Reader& r, json& j, bool initial, const auto&) { + if (!initial || !r.Bit()) return; + if (r.Bit()) j["subKey"] = r.Id(); + j["useOptionalParts"] = r.Bit(); + j["assemblyPartLOTs"] = r.WideText(); + } }, + { CONTROLLABLE_PHYSICS, [](Reader& r, json& j, bool initial, const auto&) { + if (initial) { + if ((j["inJetpackMode"] = r.Bit()).get()) { + j["jetpackEffectID"] = r.Get(); + j["jetpackFlying"] = r.Bit(); + j["jetpackBypassChecks"] = r.Bit(); + } + if (r.Bit()) { + json stun = json::array(); + for (int i = 0; i < 7; i++) stun.push_back(r.Get()); + j["immuneToStunCounts"] = stun; // move, jump, turn, attack, use item, equip, interact + } + } + if (r.Bit()) { + j["gravityScale"] = r.Get(); + j["speedMultiplier"] = r.Get(); + } + if (r.Bit()) { + j["pickupRadius"] = r.Get(); + j["inJetpackModeEquipped"] = r.Bit(); + } + if (r.Bit()) { + if ((j["inBubble"] = r.Bit()).get()) { + j["bubbleType"] = r.Get(); + j["specialAnims"] = r.Bit(); + } + } + if (r.Bit()) { + j["position"] = r.Point(); + j["rotation"] = r.Rotation(); + j["onGround"] = r.Bit(); + j["onRail"] = r.Bit(); + if (r.Bit()) j["velocity"] = r.Point(); + if (r.Bit()) j["angularVelocity"] = r.Point(); + if (r.Bit()) NotRead(r, j, "localSpaceInfo"); + if (!initial) j["teleporting"] = r.Bit(); + } + } }, + { SIMPLE_PHYSICS, [](Reader& r, json& j, bool initial, const auto&) { + if (initial) { + j["climbable"] = r.Bit(); + j["climbableType"] = r.Get(); + } + if (r.Bit()) { + j["velocity"] = r.Point(); + j["angularVelocity"] = r.Point(); + } + if (r.Bit()) j["motionType"] = r.Get(); + Position(r, j); + } }, + { RIGID_BODY_PHANTOM_PHYSICS, [](Reader& r, json& j, bool, const auto&) { Position(r, j); } }, + { HAVOK_VEHICLE_PHYSICS, [](Reader& r, json& j, bool initial, const auto&) { + if (r.Bit()) { + j["position"] = r.Point(); + j["rotation"] = r.Rotation(); + j["onGround"] = r.Bit(); + j["onRail"] = r.Bit(); + if (r.Bit()) j["velocity"] = r.Point(); + if (r.Bit()) j["angularVelocity"] = r.Point(); + if (r.Bit()) NotRead(r, j, "localSpaceInfo"); + if (r.Bit()) { + j["remoteInputX"] = r.Get(); + j["remoteInputY"] = r.Get(); + j["powersliding"] = r.Bit(); + j["modified"] = r.Bit(); + j["remoteInputPing"] = r.Get(); + } + if (!initial) j["teleporting"] = r.Bit(); + } + if (initial) { + j["endBehavior"] = r.Get(); + j["inputLocked"] = r.Bit(); + } + if (r.Bit()) NotRead(r, j, "trailingFlag"); + } }, + { PHANTOM_PHYSICS, [](Reader& r, json& j, bool, const auto&) { + Position(r, j); + if (!r.Bit()) return; + if (!(j["effectActive"] = r.Bit()).get()) return; + j["effectType"] = r.Get(); + j["directionalMultiplier"] = r.Get(); + if (r.Bit()) return NotRead(r, j, "distanceInfo"); + if ((j["directional"] = r.Bit()).get()) j["direction"] = r.Point(); + } }, + { SOUND_TRIGGER, [](Reader& r, json& j, bool, const auto&) { + if (!r.Bit()) return; + json cues = json::array(); + for (uint8_t n = r.Get(), i = 0; r.ok && i < n; i++) { + const auto name = r.Text(); + const auto result = r.Get(); + cues.push_back({ {"name", name}, {"result", result}, {"boredomTime", r.Get()} }); + } + j["musicCues"] = cues; + json parameters = json::array(); + for (uint8_t n = r.Get(), i = 0; r.ok && i < n; i++) { + const auto name = r.Text(); + parameters.push_back({ {"name", name}, {"value", r.Get()} }); + } + j["musicParameters"] = parameters; + for (const char* key : { "ambientSounds2D", "ambientSounds3D" }) { + json sounds = json::array(); + for (uint8_t n = r.Get(), i = 0; r.ok && i < n; i++) { + const auto data1 = r.Get(); + const auto data2 = r.Get(); + const auto data3 = r.Get(); + std::string data4; + for (int b = 0; b < 8; b++) data4 += std::to_string(r.Get()) + (b < 7 ? "," : ""); + sounds.push_back({ {"guid", std::to_string(data1) + "-" + std::to_string(data2) + "-" + std::to_string(data3) + "-" + data4}, {"result", r.Get()} }); + } + j[key] = sounds; + } + json mixers = json::array(); + for (uint8_t n = r.Get(), i = 0; r.ok && i < n; i++) { + const auto name = r.Text(); + mixers.push_back({ {"name", name}, {"result", r.Get()} }); + } + j["mixerPrograms"] = mixers; + } }, + { BUFF, [](Reader& r, json& j, bool initial, const auto&) { + if (!initial) return; + if (r.Bit()) { + json buffs = json::array(); + const auto count = r.Get(); + for (uint32_t i = 0; r.ok && i < count && i < 256; i++) { + json buff{ {"id", r.Get()} }; + if (r.Bit()) buff["timeMs"] = r.Get(); + for (const char* flag : { "cancelOnDeath", "cancelOnZone", "cancelOnDamaged", "cancelOnRemoveBuff", "cancelOnUi", "cancelOnLogout", "cancelOnUnequip", "cancelOnDamageAbsorbRanOut" }) buff[flag] = r.Bit(); + const bool addedByTeammate = r.Bit(); + buff["addedByTeammate"] = addedByTeammate; + buff["applyOnTeammates"] = r.Bit(); + if (addedByTeammate) buff["source"] = r.Id(); + buff["refCount"] = r.Get(); + buffs.push_back(buff); + } + j["buffs"] = buffs; + } + if (r.Bit()) NotRead(r, j, "immunityBuffs"); + } }, + { DESTROYABLE, [](Reader& r, json& j, bool initial, const auto&) { + if (initial && r.Bit()) { + json immunities = json::object(); + for (const char* key : { "basicAttack", "damageOverTime", "knockback", "interrupt", "speed", "imaginationGain", "imaginationLoss", "quickbuildInterrupt", "pullToPoint" }) immunities[key] = r.Get(); + j["immuneToCounts"] = immunities; + } + if (r.Bit()) { + j["health"] = r.Get(); + j["maxHealth"] = r.Get(); + j["armor"] = r.Get(); + j["maxArmor"] = r.Get(); + j["imagination"] = r.Get(); + j["maxImagination"] = r.Get(); + j["damageAbsorptionPoints"] = r.Get(); + j["immune"] = r.Bit(); + j["gmImmune"] = r.Bit(); + j["shielded"] = r.Bit(); + j["actualMaxHealth"] = r.Get(); + j["actualMaxArmor"] = r.Get(); + j["actualMaxImagination"] = r.Get(); + json factions = json::array(); + const auto count = r.Get(); + for (uint32_t i = 0; r.ok && i < count && i < 256; i++) factions.push_back(r.Get()); + j["factions"] = factions; + const bool smashable = r.Bit(); + j["smashable"] = smashable; + if (initial) { + j["dead"] = r.Bit(); + j["smashed"] = r.Bit(); + if (smashable) { + j["moduleAssembly"] = r.Bit(); + if (r.Bit()) j["explodeFactor"] = r.Get(); + } + } + } + if (r.Bit()) j["onThreatList"] = r.Bit(); + } }, + { COLLECTIBLE, [](Reader& r, json& j, bool, const auto&) { j["collectibleID"] = r.Get(); } }, + { PET, [](Reader& r, json& j, bool initial, const auto&) { + if (r.Bit()) { + j["status"] = r.Get(); + j["ability"] = r.Get(); + if (r.Bit()) j["interaction"] = r.Id(); + if (r.Bit()) j["owner"] = r.Id(); + } + if (initial && r.Bit()) { + j["moderationStatus"] = r.Get(); + j["name"] = r.WideText(); + j["ownerName"] = r.WideText(); + } + } }, + { POSSESSOR, [](Reader& r, json& j, bool, const auto&) { + if (!r.Bit()) return; + if (r.Bit()) j["possessable"] = r.Id(); + j["possessableType"] = r.Get(); + } }, + { LEVEL_PROGRESSION, [](Reader& r, json& j, bool, const auto&) { if (r.Bit()) j["level"] = r.Get(); } }, + { PLAYER_FORCED_MOVEMENT, [](Reader& r, json& j, bool, const auto&) { + if (!r.Bit()) return; + j["onRail"] = r.Bit(); + j["showBillboard"] = r.Bit(); + } }, + { CHARACTER, [](Reader& r, json& j, bool initial, const auto&) { + if (initial) { + json claimCodes = json::array(); + for (int i = 0; i < 4; i++) claimCodes.push_back(r.Bit() ? json(std::to_string(r.Get())) : json(nullptr)); + j["claimCodes"] = claimCodes; + for (const char* key : { "hairColor", "hairStyle", "head", "shirtColor", "pantsColor", "shirtStyle", "headColor", "eyebrows", "eyes", "mouth" }) j[key] = r.Get(); + j["accountID"] = std::to_string(r.Get()); + j["lastLogin"] = std::to_string(r.Get()); + j["propModLastDisplayTime"] = std::to_string(r.Get()); + j["uscore"] = std::to_string(r.Get()); + j["freeToPlay"] = r.Bit(); + json stats = json::array(); + for (int i = 0; i < 27; i++) stats.push_back(std::to_string(r.Get())); + j["statistics"] = stats; + j["unknownFlag"] = r.Bit(); + if ((j["landing"] = r.Bit()).get()) j["lastRocketConfig"] = r.WideText(); + } + if (r.Bit()) { + j["pvpEnabled"] = r.Bit(); + j["isGM"] = r.Bit(); + j["gmLevel"] = r.Get(); + j["editorEnabled"] = r.Bit(); + j["editorLevel"] = r.Get(); + } + if (r.Bit()) j["currentActivity"] = r.Get(); + if (r.Bit()) { + j["guildID"] = r.Id(); + j["guildName"] = r.WideText(); + j["legoClubMember"] = r.Bit(); + j["countryCode"] = r.Get(); + } + } }, + { INVENTORY, [](Reader& r, json& j, bool initial, const auto&) { + if (r.Bit()) { + json items = json::array(); + const auto count = r.Get(); + for (uint32_t i = 0; r.ok && i < count && i < 1024; i++) { + json item{ {"id", r.Id()}, {"lot", r.Get()} }; + if (r.Bit()) item["subkey"] = r.Id(); + if (r.Bit()) item["count"] = r.Get(); + if (r.Bit()) item["slot"] = r.Get(); + if (r.Bit()) item["inventoryType"] = r.Get(); + if (r.Bit()) item["config"] = ReadLdf(r); + item["bound"] = r.Bit(); + items.push_back(item); + } + j["equipped"] = items; + } + if (r.Bit()) j["equippedModelTransforms"] = r.Get(); + } }, + { SCRIPT, [](Reader& r, json& j, bool initial, const auto&) { + if (initial && r.Bit()) j["networkSettings"] = ReadLdf(r); + } }, + { SKILL, [](Reader& r, json& j, bool initial, const auto&) { + if (initial && r.Bit()) NotRead(r, j, "skillsInProgress"); + } }, + { BASE_COMBAT_AI, [](Reader& r, json& j, bool, const auto&) { + if (!r.Bit()) return; + j["state"] = r.Get(); + j["target"] = r.Id(); + } }, + { ITEM, [](Reader& r, json& j, bool, const auto&) { + if (!r.Bit()) return; + j["ugID"] = r.Id(); + j["ugModerationStatus"] = r.Get(); + if (r.Bit()) j["ugDescription"] = r.WideText(); + } }, + { QUICK_BUILD, [](Reader& r, json& j, bool initial, const auto&) { + if (r.Bit()) j["players"] = ActivityPlayers(r); + if (r.Bit()) { + j["state"] = r.Get(); + j["success"] = r.Bit(); + j["enabled"] = r.Bit(); + j["timeSinceStart"] = r.Get(); + j["pausedTime"] = r.Get(); + if (initial) { + j["choiceBuild"] = r.Bit(); + j["activatorPosition"] = r.Point(); + j["repositionPlayer"] = r.Bit(); + } + } + } }, + { MOVING_PLATFORM, [](Reader& r, json& j, bool, const auto&) { + if (!r.Bit()) { + r.Bit(); + return; + } + if (r.Bit()) { + j["pathFlag"] = r.Bit(); + j["pathName"] = r.WideText(); + j["startingWaypoint"] = r.Get(); + j["reverse"] = r.Bit(); + } + if (!r.Bit()) return; + const auto type = r.Get(); + j["moverType"] = type; + if (type == 5) return; // simple mover: nothing more + if (!r.Bit()) return; + j["state"] = r.Get(); + j["desiredWaypoint"] = r.Get(); + j["stopAtDesiredWaypoint"] = r.Bit(); + j["reverse"] = r.Bit(); + j["percentBetweenPoints"] = r.Get(); + j["position"] = r.Point(); + j["currentWaypoint"] = r.Get(); + j["nextWaypoint"] = r.Get(); + j["idleTimeElapsed"] = r.Get(); + j["moveTimeElapsed"] = r.Get(); + } }, + { SWITCH, [](Reader& r, json& j, bool, const auto&) { j["active"] = r.Bit(); } }, + { VENDOR, [](Reader& r, json& j, bool, const auto&) { + if (!r.Bit()) return; + j["hasStandardCostItems"] = r.Bit(); + j["hasMultiCostItems"] = r.Bit(); + } }, + { DONATION_VENDOR, [](Reader& r, json& j, bool, const auto&) { + if (r.Bit()) { + j["hasStandardCostItems"] = r.Bit(); + j["hasMultiCostItems"] = r.Bit(); + } + if (!r.Bit()) return; + j["percentComplete"] = r.Get(); + j["totalDonated"] = r.Get(); + j["totalRemaining"] = r.Get(); + } }, + { ACHIEVEMENT_VENDOR, [](Reader& r, json& j, bool, const auto&) { + if (!r.Bit()) return; + j["hasStandardCostItems"] = r.Bit(); + j["hasMultiCostItems"] = r.Bit(); + } }, + { BOUNCER, [](Reader& r, json& j, bool, const auto&) { + if ((j["petEnabled"] = r.Bit()).get()) j["petBouncerEnabled"] = r.Bit(); + } }, + { SCRIPTED_ACTIVITY, [](Reader& r, json& j, bool, const auto&) { if (r.Bit()) j["players"] = ActivityPlayers(r); } }, + { SHOOTING_GALLERY, [](Reader& r, json& j, bool initial, const auto&) { + if (r.Bit()) j["players"] = ActivityPlayers(r); + if (initial) { + j["cameraPosition"] = r.Point(); + j["cameraLookatPosition"] = r.Point(); + } + if (!r.Bit()) return; + j["cannonVelocity"] = r.Get(); + j["cannonRefireRate"] = r.Get(); + j["cannonMinDistance"] = r.Get(); + j["cameraBarrelOffset"] = r.Point(); + j["cannonAngle"] = r.Get(); + j["facing"] = r.Point(); + j["currentPlayer"] = r.Id(); + j["cannonTimeout"] = r.Get(); + j["cannonFOV"] = r.Get(); + } }, + { RACING_CONTROL, [](Reader& r, json& j, bool, const auto&) { + if (r.Bit()) j["players"] = ActivityPlayers(r); + if (r.Bit()) { + j["expectedPlayers"] = r.Get(); + if (r.Bit()) { + json loading = json::array(); + while (r.ok && r.Bit()) { + json p{ {"player", r.Id()}, {"vehicle", r.Id()} }; + p["index"] = r.Get(); + p["loaded"] = r.Bit(); + loading.push_back(p); + } + j["preRacePlayers"] = loading; + } + } + if (r.Bit()) { + json finished = json::array(); + while (r.ok && r.Bit()) { + json p{ {"player", r.Id()} }; + p["finished"] = r.Get(); + finished.push_back(p); + } + j["postRacePlayers"] = finished; + } + if (r.Bit()) { + j["remainingLaps"] = r.Get(); + j["pathName"] = r.WideText(); + } + if (r.Bit()) { + json results = json::array(); + while (r.ok && r.Bit()) { + json p{ {"player", r.Id()} }; + p["bestLapTime"] = r.Get(); + p["raceTime"] = r.Get(); + results.push_back(p); + } + j["results"] = results; + } + } }, + { LUP_EXHIBIT, [](Reader& r, json& j, bool, const auto&) { if (r.Bit()) j["exhibitLOT"] = r.Get(); } }, + { MODEL, [](Reader& r, json& j, bool initial, const std::vector& components) { + if (std::find(components.begin(), components.end(), PET) == components.end() && r.Bit()) { + j["modelID"] = r.Id(); + j["modelModerationStatus"] = r.Get(); + if (r.Bit()) j["ugDescription"] = r.WideText(); + } + if (r.Bit()) { + j["pickable"] = r.Bit(); + j["physicsType"] = r.Get(); + j["originalPosition"] = r.Point(); + j["originalRotation"] = r.RawRotation(); + } + if (r.Bit()) { + j["behaviors"] = r.Get(); + j["paused"] = r.Bit(); + } + if (initial && r.Bit()) NotRead(r, j, "editingInfo"); + } }, + { RENDER, [](Reader& r, json& j, bool initial, const auto&) { + if (!initial) return; + json effects = json::array(); + const auto count = r.Get(); + for (uint32_t i = 0; r.ok && i < count && i < 1024; i++) { + const auto name = r.Text(); + if (name.empty()) { + effects.push_back(json::object()); + continue; + } + json effect{ {"name", name}, {"effectID", r.Get()} }; + effect["type"] = r.WideText(); + effect["priority"] = r.Get(); + effect["secondary"] = r.Id(); + effects.push_back(effect); + } + j["effects"] = effects; + } }, + { MINI_GAME_CONTROL, [](Reader& r, json& j, bool, const auto&) { j["value"] = r.Get(); } }, + }; + return readers; + } + + // The order the client reads components in (Entity.cpp SERIALIZATION_ORDER), with the character's parts in front + constexpr std::array ORDER{ + POSSESSABLE, MODULE_ASSEMBLY, CONTROLLABLE_PHYSICS, SIMPLE_PHYSICS, RIGID_BODY_PHANTOM_PHYSICS, HAVOK_VEHICLE_PHYSICS, + PHANTOM_PHYSICS, SOUND_TRIGGER, RACING_SOUND_TRIGGER, BUFF, DESTROYABLE, COLLECTIBLE, PET, POSSESSOR, LEVEL_PROGRESSION, + PLAYER_FORCED_MOVEMENT, CHARACTER, INVENTORY, SCRIPT, SKILL, BASE_COMBAT_AI, ITEM, QUICK_BUILD, MOVING_PLATFORM, SWITCH, + VENDOR, DONATION_VENDOR, ACHIEVEMENT_VENDOR, BOUNCER, SCRIPTED_ACTIVITY, SHOOTING_GALLERY, RACING_CONTROL, LUP_EXHIBIT, + MODEL, RENDER, MINI_GAME_CONTROL, + }; + + /** + * The components as Entity::Initialize makes them from the registry, in the order Entity::WriteComponents writes + * them (the destroyable where DestroyableSerializationSlot puts it). `extraDestroyable`: one the registry doesn't + * list (is_smashable objects, models). + */ + std::vector Arrange(std::set has, bool extraDestroyable) { + if (has.contains(DESTROYABLE)) has.insert(BUFF); + if (has.contains(CHARACTER)) has.insert({ POSSESSOR, LEVEL_PROGRESSION, PLAYER_FORCED_MOVEMENT }); + if (has.contains(PET)) has.erase(MODEL); + // Collectibles get one; a quick build without one writes the same empty bits itself in the same place + const bool destroyable = has.contains(DESTROYABLE) || has.contains(COLLECTIBLE) || has.contains(QUICK_BUILD) || extraDestroyable; + eReplicaComponentType slot = MINI_GAME_CONTROL; + if (has.contains(BUFF) || has.contains(COLLECTIBLE)) slot = DESTROYABLE; + else if (has.contains(QUICK_BUILD)) slot = QUICK_BUILD; + has.erase(DESTROYABLE); + std::vector out; + for (const auto type : ORDER) { + if (destroyable && type == slot) out.push_back(DESTROYABLE); + if (type != DESTROYABLE && has.contains(type)) out.push_back(type); + } + return out; + } + + std::set Registered(LOT lot, const ReplicaDecoder::ComponentTable& table) { + // BBB models (LOT 14) are made up in code: simple physics, model, render and a destroyable after them + if (lot == 14) return { SIMPLE_PHYSICS, MODEL, RENDER }; + const auto it = table.find(lot); + if (it == table.end()) return {}; + return { it->second.begin(), it->second.end() }; + } + + // The layouts to try for an object, the registry's own first + std::vector> Candidates(LOT lot, const ReplicaDecoder::ComponentTable& table) { + const auto has = Registered(lot, table); + // Models get a destroyable too (Entity::Initialize), BBB models always + const bool model = has.contains(MODEL) && !has.contains(PET); + std::vector> out{ Arrange(has, lot == 14 || model) }; + auto add = [&out](std::vector candidate) { + if (std::find(out.begin(), out.end(), candidate) == out.end()) out.push_back(std::move(candidate)); + }; + // Set up by the zone file rather than the registry: a smashable's destroyable, a moving platform's path, a script + add(Arrange(has, true)); + auto withPlatform = has; + withPlatform.insert(MOVING_PLATFORM); + add(Arrange(withPlatform, false)); + add(Arrange(withPlatform, true)); + auto withScript = has; + withScript.insert(SCRIPT); + add(Arrange(withScript, false)); + add(Arrange(withScript, true)); + auto withoutScript = has; + withoutScript.erase(SCRIPT); + add(Arrange(withoutScript, false)); + return out; + } + + std::string Name(eReplicaComponentType type) { + const auto name = magic_enum::enum_name(type); + return name.empty() ? std::to_string(static_cast(type)) : std::string(name); + } + + std::string RestHex(RakNet::BitStream& stream) { + std::string out; + static constexpr char digits[] = "0123456789abcdef"; + const auto offset = stream.GetReadOffset(); + while (stream.GetNumberOfUnreadBits() >= 8) { + uint8_t byte{}; + stream.Read(byte); + out += digits[byte >> 4]; + out += digits[byte & 15]; + } + const auto bits = stream.GetNumberOfUnreadBits(); + if (bits > 0) { + out += " +"; + for (uint32_t i = 0; i < bits; i++) { + bool bit{}; + stream.Read(bit); + out += bit ? '1' : '0'; + } + } + stream.SetReadOffset(offset); + return out; + } + + // What is left after the last component: padding (under a byte, all zero) or data no reader took + bool OnlyPadding(RakNet::BitStream& stream) { + const auto unread = stream.GetNumberOfUnreadBits(); + if (unread >= 8) return false; + const auto offset = stream.GetReadOffset(); + bool zero = true; + for (uint32_t i = 0; i < unread; i++) { + bool bit{}; + stream.Read(bit); + zero = zero && !bit; + } + stream.SetReadOffset(offset); + return zero; + } + + // Reads the components in order from `start`; true when they read the stream exactly + bool ReadComponents(RakNet::BitStream& stream, BitSize_t start, const std::vector& components, bool initial, json& out) { + stream.SetReadOffset(start); + out = json::array(); + Reader r{ stream }; + for (const auto type : components) { + json fields = json::object(); + const auto reader = Readers().find(type); + if (reader == Readers().end()) continue; + const auto before = stream.GetReadOffset(); + reader->second(r, fields, initial, components); + if (!r.ok) { + stream.SetReadOffset(before); + out.push_back({ {"component", Name(type)}, {"fields", fields}, {"(did not read)", true} }); + return false; + } + out.push_back({ {"component", Name(type)}, {"fields", fields} }); + } + return OnlyPadding(stream); + } + + json ReadParentChild(Reader& r) { + json out = json::object(); + if (!r.Bit()) return out; + if (r.Bit()) { + out["parent"] = r.Id(); + out["updatePositionWithParent"] = r.Bit(); + } + if (r.Bit()) { + json children = json::array(); + const auto count = r.Get(); + for (uint16_t i = 0; r.ok && i < count; i++) children.push_back(r.Id()); + out["children"] = children; + } + return out; + } +} + +namespace ReplicaDecoder { + size_t LoadComponentTable(ComponentTable& table) { + table.clear(); + auto result = CDClientDatabase::ExecuteQuery("SELECT id, component_type FROM ComponentsRegistry"); + while (!result.eof()) { + table[result.getIntField(0)].push_back(static_cast(result.getIntField(1))); + result.nextRow(); + } + return table.size(); + } + + std::vector ComponentsOf(LOT lot, const ComponentTable& table) { + return Candidates(lot, table).front(); + } + + std::optional Session::Decode(std::string_view bytes, uint64_t connection) { + if (bytes.empty()) return std::nullopt; + const auto id = static_cast(bytes[0]); + if (id != ID_REPLICA_MANAGER_CONSTRUCTION && id != ID_REPLICA_MANAGER_SERIALIZE && id != ID_REPLICA_MANAGER_DESTRUCTION) return std::nullopt; + RakNet::BitStream stream(reinterpret_cast(const_cast(bytes.data())), static_cast(bytes.size()), false); + stream.IgnoreBytes(1); + Reader r{ stream }; + json out = json::object(); + + if (id == ID_REPLICA_MANAGER_DESTRUCTION) { + const auto network = r.Get(); + if (!r.ok) return json{ {"(did not read)", true} }; + out["networkID"] = network; + const auto it = m_Objects.find({ connection, network }); + if (it != m_Objects.end()) { + out["objectID"] = std::to_string(it->second.objectId); + out["lot"] = it->second.lot; + m_Objects.erase(it); + } + return out; + } + + if (id == ID_REPLICA_MANAGER_SERIALIZE) { + const auto network = r.Get(); + out["networkID"] = network; + const auto it = m_Objects.find({ connection, network }); + if (it == m_Objects.end()) { + out["(object not constructed in this capture)"] = true; + out["(rest)"] = RestHex(stream); + return out; + } + out["objectID"] = std::to_string(it->second.objectId); + out["lot"] = it->second.lot; + out["parentChild"] = ReadParentChild(r); + json components; + const auto start = stream.GetReadOffset(); + if (!r.ok || !ReadComponents(stream, start, it->second.components, false, components)) { + out["(layout did not match)"] = true; + out["(rest)"] = RestHex(stream); + } + out["components"] = components; + return out; + } + + // Construction: Entity::WriteBaseReplicaData, then the components + r.Bit(); + const auto network = r.Get(); + const auto objectId = r.Get(); + const auto lot = r.Get(); + out["networkID"] = network; + out["objectID"] = std::to_string(objectId); + out["lot"] = lot; + out["name"] = r.WideText(); + out["timeSinceCreatedMs"] = r.Get(); + if (r.Bit()) out["config"] = ReadLdf(r); + out["trigger"] = r.Bit(); + if (r.Bit()) out["spawner"] = r.Id(); + if (r.Bit()) out["spawnerNode"] = r.Get(); + if (r.Bit()) out["scale"] = r.Get(); + if (r.Bit()) out["worldState"] = r.Get(); + if (r.Bit()) out["gmLevel"] = r.Get(); + out["parentChild"] = ReadParentChild(r); + if (!r.ok) { + out["(did not read)"] = true; + return out; + } + + const auto start = stream.GetReadOffset(); + const auto candidates = Candidates(lot, m_Table); + std::vector chosen = candidates.front(); + json components; + bool matched = false; + for (const auto& candidate : candidates) { + json attempt; + if (ReadComponents(stream, start, candidate, true, attempt)) { + chosen = candidate; + components = std::move(attempt); + matched = true; + break; + } + } + if (!matched) { + // Shown as the registry says, as far as it reads, with the rest as bytes + ReadComponents(stream, start, chosen, true, components); + out["(layout did not match)"] = true; + out["(rest)"] = RestHex(stream); + } + if (!m_Table.contains(lot) && lot != 14) out["(LOT not in ComponentsRegistry)"] = true; + out["components"] = components; + m_Objects[{ connection, network }] = Object{ objectId, lot, chosen }; + return out; + } +} diff --git a/dGame/dUtilities/ReplicaDecoder.h b/dGame/dUtilities/ReplicaDecoder.h new file mode 100644 index 000000000..c2f64d73a --- /dev/null +++ b/dGame/dUtilities/ReplicaDecoder.h @@ -0,0 +1,61 @@ +#ifndef __REPLICADECODER__H__ +#define __REPLICADECODER__H__ + +#include +#include +#include +#include +#include +#include + +#include "dCommonVars.h" +#include "json.hpp" + +enum class eReplicaComponentType : uint32_t; + +/** + * Reads recorded replica packets (ID_REPLICA_MANAGER_CONSTRUCTION, _SERIALIZE and _DESTRUCTION) for the capture + * viewer. A construction is the object's header (Entity::WriteBaseReplicaData) followed by each component's data in + * the order the client reads them (Entity::WriteComponents); a serialization is the same for an object constructed + * earlier on the same connection, so a Session is fed a capture's packets in timeline order and remembers what each + * network ID is. + * + * Each component reader mirrors its component's Serialize(bIsInitialUpdate) bit for bit (ReplicaDecoderTests checks + * them against the server's own serializers). Which components an object has comes from the ComponentsRegistry rows + * of its LOT, as the client decides it. Some objects have components their LOT doesn't list (a smashable or a moving + * platform set up by the zone file): when the listed ones don't read the packet exactly, those variants are tried, and + * when none fits the packet is shown as far as it read, with the rest as bits (never a guess). + */ +namespace ReplicaDecoder { + // LOT -> the component types its ComponentsRegistry rows list + using ComponentTable = std::unordered_map>; + + // Reads the ComponentsRegistry from the CDClient (on the thread that owns it). Returns how many LOTs it read. + size_t LoadComponentTable(ComponentTable& table); + + // The components the client reads for a LOT, in reading order + std::vector ComponentsOf(LOT lot, const ComponentTable& table); + + class Session { + public: + explicit Session(const ComponentTable& table) : m_Table{ table } {} + + /** + * The packet's fields, or nullopt when it isn't a replica packet. `connection` tells apart the network IDs of + * different connections (worlds, characters): use the same value for every packet of one connection. + */ + std::optional Decode(std::string_view bytes, uint64_t connection); + + private: + struct Object { + LWOOBJID objectId{}; + LOT lot{}; + std::vector components; + }; + + const ComponentTable& m_Table; + std::map, Object> m_Objects; + }; +} + +#endif //!__REPLICADECODER__H__ diff --git a/dNet/CaptureTools.cpp b/dNet/CaptureTools.cpp index e07854a9d..4b18928d7 100644 --- a/dNet/CaptureTools.cpp +++ b/dNet/CaptureTools.cpp @@ -97,7 +97,11 @@ namespace CaptureTools { return fields.contains(name); } - json RecordJson(const Record& record, size_t index, int64_t startUs, bool fields) { + uint64_t ReplicaConnection(const PacketRecordHeader& h) { + return (static_cast(h.zoneId) << 48) ^ (static_cast(h.instanceId) << 32) ^ h.cloneId; + } + + json RecordJson(const Record& record, size_t index, int64_t startUs, bool fields, const json* replicaFields) { const auto& h = record.header; json out{ {"i", index}, @@ -130,7 +134,8 @@ namespace CaptureTools { out["object"] = std::to_string(decoded.objectId); } if (decoded.failed) out["unreadable"] = true; - if (fields && decoded.fields) out["fields"] = *decoded.fields; + if (fields && replicaFields) out["fields"] = *replicaFields; + else if (fields && decoded.fields) out["fields"] = *decoded.fields; return out; } diff --git a/dNet/CaptureTools.h b/dNet/CaptureTools.h index c560fe7da..7fdd07faa 100644 --- a/dNet/CaptureTools.h +++ b/dNet/CaptureTools.h @@ -23,8 +23,13 @@ namespace CaptureTools { // Whether a record went from a game client to a server bool FromClient(const PacketRecordHeader& header); - // One record for the viewer: where it went, its name and, when `fields`, its decoded fields - nlohmann::json RecordJson(const CaptureBundle::Record& record, size_t index, int64_t startUs, bool fields); + // One record for the viewer: where it went, its name and, when `fields`, its decoded fields. `replicaFields`: the + // fields of a replica packet, which only a pass over the capture in order can read (ReplicaDecoder::Session) + nlohmann::json RecordJson(const CaptureBundle::Record& record, size_t index, int64_t startUs, bool fields, + const nlohmann::json* replicaFields = nullptr); + + // Tells apart the network IDs of different world instances in one capture + uint64_t ReplicaConnection(const PacketRecordHeader& header); struct Track { LWOOBJID characterId{};