Merge exp/decoder-live-check: replica decoders checked on live captures, moving platform/pet/model wire fixes

This commit is contained in:
Aaron Kimbrell
2026-09-30 08:40:02 -05:00
10 changed files with 506 additions and 95 deletions

View File

@@ -157,7 +157,7 @@ namespace {
}
const auto j = CaptureTools::RecordJson(record, i, start, fields, replica ? &*replica : nullptr);
std::printf("%7zu %10.3f %-12s %-18s %s%s\n", i, j["t"].get<double>() / 1000.0, (j.value("from", std::string()) + ">" + j.value("to", std::string())).c_str(),
j.value("source", std::string()).c_str(), j.value("name", std::string()).c_str(), fields && j.contains("fields") ? (" " + j["fields"].dump()).c_str() : "");
j.value("source", std::string()).c_str(), j.value("name", std::string()).c_str(), fields && j.contains("fields") ? (" " + j["fields"].dump(-1, ' ', false, json::error_handler_t::replace)).c_str() : "");
}
if (!cdServer.empty()) std::printf("%zu constructions, %zu whose components didn't read exactly\n", constructions, unmatched);
return 0;

View File

@@ -146,6 +146,11 @@ void ModelComponent::Serialize(RakNet::BitStream& outBitStream, bool bIsInitialU
outBitStream.Write(m_OriginalPosition); // Original position
outBitStream.Write(m_OriginalRotation); // Original rotation
// The client makes the mutable model component, which reads the behaviors and editing info, only for objects whose
// config (the settings sent with a model) has propertyObjectID or inInventory; the plain one it makes for every
// other model reads only the block above, as live wrote it
if (!m_Parent->HasVar(u"propertyObjectID") && !m_Parent->GetVar<bool>(u"inInventory")) return;
outBitStream.Write1(); // We are writing behavior info
outBitStream.Write<uint32_t>(m_Behaviors.size()); // Number of behaviors
outBitStream.Write(m_IsPaused); // Is this model paused

View File

@@ -118,6 +118,9 @@ void MovingPlatformComponent::Serialize(RakNet::BitStream& outBitStream, bool bI
} else {
mover->Serialize(outBitStream, bIsInitialUpdate);
}
// The client reads subcomponents while a 1 bit comes before one: a 0 bit ends the list
outBitStream.Write0();
}
}

View File

@@ -96,7 +96,10 @@ void PetComponent::Serialize(RakNet::BitStream& outBitStream, bool bIsInitialUpd
outBitStream.Write(m_Owner);
}
if (bIsInitialUpdate) {
// The client reads this bit on every update while dirty (as live wrote it): the names follow when it is set
if (!bIsInitialUpdate) {
outBitStream.Write0();
} else {
outBitStream.Write(tamed);
if (tamed) {
outBitStream.Write(m_ModerationStatus);

View File

@@ -11,11 +11,25 @@
#include "GeneralUtils.h"
#include "magic_enum.hpp"
#include "MessageIdentifiers.h"
#include "ZCompression.h"
namespace {
using json = nlohmann::json;
using enum eReplicaComponentType;
bool IsUtf8(std::string_view text) {
for (size_t i = 0; i < text.size();) {
const auto byte = static_cast<uint8_t>(text[i]);
const size_t length = byte < 0x80 ? 1 : (byte >> 5) == 0x6 ? 2 : (byte >> 4) == 0xE ? 3 : (byte >> 3) == 0x1E ? 4 : 0;
if (length == 0 || i + length > text.size()) return false;
for (size_t k = 1; k < length; k++) {
if ((static_cast<uint8_t>(text[i + k]) & 0xC0) != 0x80) return false;
}
i += length;
}
return true;
}
// Reads values in order; the first read past the end marks the reader as failed and every later read gives 0
struct Reader {
RakNet::BitStream& stream;
@@ -46,13 +60,28 @@ namespace {
const auto length = Get<Length>();
std::u16string text;
for (Length i = 0; ok && i < length; i++) text += static_cast<char16_t>(Get<uint16_t>());
return GeneralUtils::UTF16ToWTF8(text);
return Printable(GeneralUtils::UTF16ToWTF8(text));
}
template<typename Length> std::string Text() {
const auto length = Get<Length>();
std::string text;
for (Length i = 0; ok && i < length; i++) text += static_cast<char>(Get<uint8_t>());
return text;
return Printable(text);
}
// Narrow text is shown as is when it is UTF-8, else byte by byte as Latin-1 (JSON only takes UTF-8)
static std::string Printable(const std::string& text) {
if (IsUtf8(text)) return text;
std::string out;
for (const auto c : text) {
const auto byte = static_cast<uint8_t>(c);
if (byte < 0x80) out += c;
else {
out += static_cast<char>(0xC0 | (byte >> 6));
out += static_cast<char>(0x80 | (byte & 0x3F));
}
}
return out;
}
};
@@ -83,21 +112,38 @@ namespace {
case 13: value = r.Text<uint32_t>(); break;
default: r.ok = false; break;
}
out.push_back(GeneralUtils::UTF16ToWTF8(key) + "=" + std::to_string(type) + ":" + value);
out.push_back(Reader::Printable(GeneralUtils::UTF16ToWTF8(key)) + "=" + std::to_string(type) + ":" + value);
}
return out;
}
// u32 size, u8 compressed, then the entries (or the compressed bytes, which are shown as their size)
/**
* u32 size, u8 compressed, then the entries (a u32 count and each entry). Compressed: u32 uncompressed size, u32
* compressed size and that many bytes of zlib data holding the entries, inflated here.
*/
json ReadLdf(Reader& r) {
const auto size = r.Get<uint32_t>();
const auto compressed = r.Get<uint8_t>();
if (compressed) {
const auto compressedSize = r.Get<uint32_t>();
for (uint32_t i = 0; r.ok && i < compressedSize; i++) r.Get<uint8_t>();
return json{ {"compressed", true}, {"size", size}, {"compressedSize", compressedSize} };
if (!compressed) return ReadLdfEntries(r, r.Get<int32_t>());
const auto uncompressedSize = r.Get<uint32_t>();
const auto compressedSize = r.Get<uint32_t>();
std::string data;
for (uint32_t i = 0; r.ok && i < compressedSize; i++) data += static_cast<char>(r.Get<uint8_t>());
json out{ {"compressed", true}, {"size", size}, {"uncompressedSize", uncompressedSize}, {"compressedSize", compressedSize} };
if (!r.ok || uncompressedSize > 1024 * 1024) return out;
std::string entries(uncompressedSize, '\0');
int32_t error = 0;
const auto inflated = ZCompression::Decompress(reinterpret_cast<const uint8_t*>(data.data()), compressedSize,
reinterpret_cast<uint8_t*>(entries.data()), uncompressedSize, error);
if (inflated != static_cast<int32_t>(uncompressedSize)) {
out["(did not inflate)"] = true;
return out;
}
return ReadLdfEntries(r, r.Get<int32_t>());
RakNet::BitStream stream(reinterpret_cast<unsigned char*>(entries.data()), uncompressedSize, false);
Reader inner{ stream };
out["entries"] = ReadLdfEntries(inner, inner.Get<int32_t>());
if (!inner.ok || stream.GetNumberOfUnreadBits() > 0) out["(entries did not read)"] = true;
return out;
}
// Activity user info: object ID and 10 values each
@@ -127,6 +173,39 @@ namespace {
}
}
// The object a character stands on and where on it (LWOBasePhysComponent's frame stats)
json LocalSpace(Reader& r) {
json out{ {"object", r.Id()}, {"position", r.Point()} };
if (r.Bit()) out["velocity"] = r.Point();
return out;
}
// LWOBuffComponent::ReadBuffs: a u32 count and each buff
json Buffs(Reader& r) {
json buffs = json::array();
const auto count = r.Get<uint32_t>();
for (uint32_t i = 0; r.ok && i < count && i < 256; i++) {
json buff{ {"id", r.Get<uint32_t>()} };
if (r.Bit()) buff["timeMs"] = r.Get<uint32_t>();
for (const char* flag : { "cancelOnDeath", "cancelOnZone", "cancelOnDamaged", "cancelOnRemoveBuff", "cancelOnUi", "cancelOnLogout", "cancelOnUnequip", "cancelOnDamageAbsorbRanOut" }) buff[flag] = r.Bit();
const bool addedByTeammate = r.Bit();
buff["addedByTeammate"] = addedByTeammate;
buff["applyOnTeammates"] = r.Bit();
if (addedByTeammate) buff["source"] = r.Id();
buff["refCount"] = r.Get<uint32_t>();
buffs.push_back(buff);
}
return buffs;
}
void ModelBase(Reader& r, json& j) {
if (!r.Bit()) return;
j["pickable"] = r.Bit();
j["modelType"] = r.Get<uint32_t>();
j["originalPosition"] = r.Point();
j["originalRotation"] = r.RawRotation();
}
using ComponentReader = std::function<void(Reader&, json&, bool initial, const std::vector<eReplicaComponentType>& components)>;
// One reader per component, each mirroring the component's Serialize(bIsInitialUpdate)
@@ -178,7 +257,7 @@ namespace {
j["onRail"] = r.Bit();
if (r.Bit()) j["velocity"] = r.Point();
if (r.Bit()) j["angularVelocity"] = r.Point();
if (r.Bit()) NotRead(r, j, "localSpaceInfo");
if (r.Bit()) j["localSpace"] = LocalSpace(r);
if (!initial) j["teleporting"] = r.Bit();
}
} },
@@ -203,7 +282,7 @@ namespace {
j["onRail"] = r.Bit();
if (r.Bit()) j["velocity"] = r.Point();
if (r.Bit()) j["angularVelocity"] = r.Point();
if (r.Bit()) NotRead(r, j, "localSpaceInfo");
if (r.Bit()) j["localSpace"] = LocalSpace(r);
if (r.Bit()) {
j["remoteInputX"] = r.Get<float>();
j["remoteInputY"] = r.Get<float>();
@@ -225,7 +304,10 @@ namespace {
if (!(j["effectActive"] = r.Bit()).get<bool>()) return;
j["effectType"] = r.Get<uint32_t>();
j["directionalMultiplier"] = r.Get<float>();
if (r.Bit()) return NotRead(r, j, "distanceInfo");
if (r.Bit()) {
j["minDistance"] = r.Get<float>();
j["maxDistance"] = r.Get<float>();
}
if ((j["directional"] = r.Bit()).get<bool>()) j["direction"] = r.Point();
} },
{ SOUND_TRIGGER, [](Reader& r, json& j, bool, const auto&) {
@@ -264,23 +346,8 @@ namespace {
} },
{ BUFF, [](Reader& r, json& j, bool initial, const auto&) {
if (!initial) return;
if (r.Bit()) {
json buffs = json::array();
const auto count = r.Get<uint32_t>();
for (uint32_t i = 0; r.ok && i < count && i < 256; i++) {
json buff{ {"id", r.Get<uint32_t>()} };
if (r.Bit()) buff["timeMs"] = r.Get<uint32_t>();
for (const char* flag : { "cancelOnDeath", "cancelOnZone", "cancelOnDamaged", "cancelOnRemoveBuff", "cancelOnUi", "cancelOnLogout", "cancelOnUnequip", "cancelOnDamageAbsorbRanOut" }) buff[flag] = r.Bit();
const bool addedByTeammate = r.Bit();
buff["addedByTeammate"] = addedByTeammate;
buff["applyOnTeammates"] = r.Bit();
if (addedByTeammate) buff["source"] = r.Id();
buff["refCount"] = r.Get<uint32_t>();
buffs.push_back(buff);
}
j["buffs"] = buffs;
}
if (r.Bit()) NotRead(r, j, "immunityBuffs");
if (r.Bit()) j["buffs"] = Buffs(r);
if (r.Bit()) j["immunities"] = Buffs(r);
} },
{ DESTROYABLE, [](Reader& r, json& j, bool initial, const auto&) {
if (initial && r.Bit()) {
@@ -320,14 +387,14 @@ namespace {
if (r.Bit()) j["onThreatList"] = r.Bit();
} },
{ COLLECTIBLE, [](Reader& r, json& j, bool, const auto&) { j["collectibleID"] = r.Get<int16_t>(); } },
{ PET, [](Reader& r, json& j, bool initial, const auto&) {
// LWOPetComponent::Deserialize: everything under the dirty bit, the names on updates too
{ PET, [](Reader& r, json& j, bool, const auto&) {
if (!r.Bit()) return;
j["status"] = r.Get<uint32_t>();
j["ability"] = r.Get<uint32_t>();
if (r.Bit()) j["interaction"] = r.Id();
if (r.Bit()) j["owner"] = r.Id();
if (r.Bit()) {
j["status"] = r.Get<uint32_t>();
j["ability"] = r.Get<uint32_t>();
if (r.Bit()) j["interaction"] = r.Id();
if (r.Bit()) j["owner"] = r.Id();
}
if (initial && r.Bit()) {
j["moderationStatus"] = r.Get<uint32_t>();
j["name"] = r.WideText<uint8_t>();
j["ownerName"] = r.WideText<uint8_t>();
@@ -398,7 +465,35 @@ namespace {
if (initial && r.Bit()) j["networkSettings"] = ReadLdf(r);
} },
{ SKILL, [](Reader& r, json& j, bool initial, const auto&) {
if (initial && r.Bit()) NotRead(r, j, "skillsInProgress");
// LWOSkillComponent::Deserialize: the skills being cast and each one's running behaviors
if (!initial || !r.Bit()) return;
json skills = json::array();
const auto count = r.Get<uint32_t>();
for (uint32_t i = 0; r.ok && i < count && i < 256; i++) {
json skill{ {"skillUID", r.Get<uint32_t>()} };
skill["skillID"] = r.Get<uint32_t>();
skill["castType"] = r.Get<uint32_t>();
skill["cancelType"] = r.Get<uint32_t>();
json behaviors = json::array();
const auto behaviorCount = r.Get<uint32_t>();
for (uint32_t b = 0; r.ok && b < behaviorCount && b < 256; b++) {
json behavior{ {"behaviorHandle", r.Get<uint32_t>()} };
behavior["action"] = r.Get<uint32_t>();
behavior["waitTimeMs"] = r.Get<uint32_t>();
behavior["templateID"] = r.Get<uint32_t>();
behavior["caster"] = r.Id();
behavior["originator"] = r.Id();
behavior["target"] = r.Id();
behavior["usedMouse"] = r.Bit();
behavior["cooldown"] = r.Get<float>();
behavior["chargeTime"] = r.Get<float>();
behavior["imaginationCost"] = r.Get<float>();
behaviors.push_back(behavior);
}
skill["behaviors"] = behaviors;
skills.push_back(skill);
}
j["skillsInProgress"] = skills;
} },
{ BASE_COMBAT_AI, [](Reader& r, json& j, bool, const auto&) {
if (!r.Bit()) return;
@@ -420,38 +515,58 @@ namespace {
j["timeSinceStart"] = r.Get<float>();
j["pausedTime"] = r.Get<float>();
if (initial) {
j["choiceBuild"] = r.Bit();
// LWOQuickBuildComponent::Deserialize: a choice build has a u32 of its settings after the bit
if ((j["choiceBuild"] = r.Bit()).get<bool>()) j["choiceBuildSetting"] = r.Get<uint32_t>();
j["activatorPosition"] = r.Point();
j["repositionPlayer"] = r.Bit();
}
}
} },
// LWOMovingPlatformComponent::Deserialize: the path when dirty, then each subcomponent (a 1 bit before
// each, a 0 bit after the last) as its type
{ MOVING_PLATFORM, [](Reader& r, json& j, bool, const auto&) {
if (!r.Bit()) {
r.Bit();
return;
}
if (r.Bit()) {
j["pathFlag"] = r.Bit();
const bool hasSubcomponents = r.Bit();
if (r.Bit() && r.Bit()) {
j["pathName"] = r.WideText<uint16_t>();
j["startingWaypoint"] = r.Get<uint32_t>();
j["reverse"] = r.Bit();
}
if (!r.Bit()) return;
const auto type = r.Get<uint32_t>();
j["moverType"] = type;
if (type == 5) return; // simple mover: nothing more
if (!r.Bit()) return;
j["state"] = r.Get<uint32_t>();
j["desiredWaypoint"] = r.Get<int32_t>();
j["stopAtDesiredWaypoint"] = r.Bit();
j["reverse"] = r.Bit();
j["percentBetweenPoints"] = r.Get<float>();
j["position"] = r.Point();
j["currentWaypoint"] = r.Get<uint32_t>();
j["nextWaypoint"] = r.Get<uint32_t>();
j["idleTimeElapsed"] = r.Get<float>();
j["moveTimeElapsed"] = r.Get<float>();
if (!hasSubcomponents) return;
json subcomponents = json::array();
while (r.ok && subcomponents.size() < 16 && r.Bit()) {
const auto type = r.Get<uint32_t>();
json sub{ {"type", type} };
if (type == 4) {
// LWOPlatformMover
if (r.Bit()) {
sub["state"] = r.Get<uint32_t>();
sub["desiredWaypoint"] = r.Get<int32_t>();
sub["stopAtDesiredWaypoint"] = r.Bit();
sub["reverse"] = r.Bit();
sub["percentBetweenPoints"] = r.Get<float>();
sub["position"] = r.Point();
sub["currentWaypoint"] = r.Get<uint32_t>();
sub["nextWaypoint"] = r.Get<uint32_t>();
sub["idleTimeElapsed"] = r.Get<float>();
sub["moveTimeElapsed"] = r.Get<float>();
}
} else if (type == 5) {
// LWOPlatformSimpleMover
if (r.Bit() && r.Bit()) {
sub["startPosition"] = r.Point();
sub["startRotation"] = r.Rotation();
}
if (r.Bit()) {
sub["state"] = r.Get<uint32_t>();
sub["currentWaypoint"] = r.Get<uint32_t>();
sub["reverse"] = r.Bit();
}
} else {
NotRead(r, sub, "subcomponent");
}
subcomponents.push_back(sub);
}
j["subcomponents"] = subcomponents;
} },
{ SWITCH, [](Reader& r, json& j, bool, const auto&) { j["active"] = r.Bit(); } },
{ VENDOR, [](Reader& r, json& j, bool, const auto&) {
@@ -535,24 +650,25 @@ namespace {
}
} },
{ LUP_EXHIBIT, [](Reader& r, json& j, bool, const auto&) { if (r.Bit()) j["exhibitLOT"] = r.Get<int32_t>(); } },
{ MODEL, [](Reader& r, json& j, bool initial, const std::vector<eReplicaComponentType>& components) {
if (std::find(components.begin(), components.end(), PET) == components.end() && r.Bit()) {
j["modelID"] = r.Id();
j["modelModerationStatus"] = r.Get<int32_t>();
if (r.Bit()) j["ugDescription"] = r.WideText<uint32_t>();
}
if (r.Bit()) {
j["pickable"] = r.Bit();
j["physicsType"] = r.Get<uint32_t>();
j["originalPosition"] = r.Point();
j["originalRotation"] = r.RawRotation();
}
// LWOModelBehaviorComponent::Deserialize (the user-generated-content block before it is the item's)
{ MODEL, [](Reader& r, json& j, bool, const auto&) { ModelBase(r, j); } },
// LWOMutableModelBehaviorComponent::Deserialize: the model's block, the behaviors, and on construction who
// is editing it
{ MUTABLE_MODEL_BEHAVIORS, [](Reader& r, json& j, bool initial, const auto&) {
ModelBase(r, j);
if (r.Bit()) {
j["behaviors"] = r.Get<uint32_t>();
j["paused"] = r.Bit();
}
if (initial && r.Bit()) NotRead(r, j, "editingInfo");
if (initial && r.Bit()) {
j["oldObjectID"] = r.Id();
j["editor"] = r.Id();
}
} },
// LWOBBBComponent::Deserialize (characters carry one in live captures)
{ BBB, [](Reader& r, json& j, bool, const auto&) { if (r.Bit()) j["metadataSourceItem"] = r.Id(); } },
// LWOTriggerComponent::Deserialize, on objects whose header sets the trigger bit
{ TRIGGER, [](Reader& r, json& j, bool, const auto&) { if (r.Bit()) j["triggerID"] = r.Get<int32_t>(); } },
{ RENDER, [](Reader& r, json& j, bool initial, const auto&) {
if (!initial) return;
json effects = json::array();
@@ -582,7 +698,7 @@ namespace {
PHANTOM_PHYSICS, SOUND_TRIGGER, RACING_SOUND_TRIGGER, BUFF, DESTROYABLE, COLLECTIBLE, PET, POSSESSOR, LEVEL_PROGRESSION,
PLAYER_FORCED_MOVEMENT, CHARACTER, INVENTORY, SCRIPT, SKILL, BASE_COMBAT_AI, ITEM, QUICK_BUILD, MOVING_PLATFORM, SWITCH,
VENDOR, DONATION_VENDOR, ACHIEVEMENT_VENDOR, BOUNCER, SCRIPTED_ACTIVITY, SHOOTING_GALLERY, RACING_CONTROL, LUP_EXHIBIT,
MODEL, RENDER, MINI_GAME_CONTROL,
MODEL, MUTABLE_MODEL_BEHAVIORS, RENDER, MINI_GAME_CONTROL, BBB,
};
/**
@@ -593,7 +709,14 @@ namespace {
std::vector<eReplicaComponentType> Arrange(std::set<eReplicaComponentType> has, bool extraDestroyable) {
if (has.contains(DESTROYABLE)) has.insert(BUFF);
if (has.contains(CHARACTER)) has.insert({ POSSESSOR, LEVEL_PROGRESSION, PLAYER_FORCED_MOVEMENT });
if (has.contains(PET)) has.erase(MODEL);
// The client drops a pet's model and item components (ObjectLoader2::DoObjectComponentLoad)
if (has.contains(PET)) {
has.erase(MODEL);
has.erase(MUTABLE_MODEL_BEHAVIORS);
has.erase(ITEM);
}
// Objects listing component 107 (characters) write a BBB component's data after the others in live captures
if (has.contains(CRAFTING)) has.insert(BBB);
// Collectibles get one; a quick build without one writes the same empty bits itself in the same place
const bool destroyable = has.contains(DESTROYABLE) || has.contains(COLLECTIBLE) || has.contains(QUICK_BUILD) || extraDestroyable;
eReplicaComponentType slot = MINI_GAME_CONTROL;
@@ -610,17 +733,25 @@ namespace {
std::set<eReplicaComponentType> Registered(LOT lot, const ReplicaDecoder::ComponentTable& table) {
// BBB models (LOT 14) are made up in code: simple physics, model, render and a destroyable after them
if (lot == 14) return { SIMPLE_PHYSICS, MODEL, RENDER };
if (lot == 14) return { SIMPLE_PHYSICS, ITEM, MODEL, RENDER };
const auto it = table.find(lot);
if (it == table.end()) return {};
return { it->second.begin(), it->second.end() };
}
// The layouts to try for an object, the registry's own first
std::vector<std::vector<eReplicaComponentType>> Candidates(LOT lot, const ReplicaDecoder::ComponentTable& table) {
const auto has = Registered(lot, table);
/**
* `mutableModel`: the object's config has propertyObjectID or inInventory set, for which the client makes a model's
* mutable component instead of the plain one (ObjectLoader2::LoadModelBehaviorsComponent)
*/
std::vector<std::vector<eReplicaComponentType>> Candidates(LOT lot, const ReplicaDecoder::ComponentTable& table, bool mutableModel) {
auto has = Registered(lot, table);
if (mutableModel && has.contains(MODEL)) {
has.erase(MODEL);
has.insert(MUTABLE_MODEL_BEHAVIORS);
}
// Models get a destroyable too (Entity::Initialize), BBB models always
const bool model = has.contains(MODEL) && !has.contains(PET);
const bool model = (has.contains(MODEL) || has.contains(MUTABLE_MODEL_BEHAVIORS)) && !has.contains(PET);
std::vector<std::vector<eReplicaComponentType>> out{ Arrange(has, lot == 14 || model) };
auto add = [&out](std::vector<eReplicaComponentType> candidate) {
if (std::find(out.begin(), out.end(), candidate) == out.end()) out.push_back(std::move(candidate));
@@ -638,6 +769,23 @@ namespace {
auto withoutScript = has;
withoutScript.erase(SCRIPT);
add(Arrange(withoutScript, false));
// A simple physics object the zone file sets markedAsPhantom on gets phantom physics instead
// (LWOSimplePhysicsComponent::Allocator)
if (has.contains(SIMPLE_PHYSICS)) {
auto phantom = has;
phantom.erase(SIMPLE_PHYSICS);
phantom.insert(PHANTOM_PHYSICS);
add(Arrange(phantom, false));
add(Arrange(phantom, true));
}
// The client makes no FX component (the render data) when the zone file sets renderDisabled, as on trigger
// volumes (ObjectLoader2::LoadRenderComponent)
if (has.contains(RENDER)) {
auto withoutRender = has;
withoutRender.erase(RENDER);
add(Arrange(withoutRender, false));
add(Arrange(withoutRender, true));
}
return out;
}
@@ -685,10 +833,12 @@ namespace {
}
// Reads the components in order from `start`; true when they read the stream exactly
bool ReadComponents(RakNet::BitStream& stream, BitSize_t start, const std::vector<eReplicaComponentType>& components, bool initial, json& out) {
bool ReadComponents(RakNet::BitStream& stream, BitSize_t start, std::vector<eReplicaComponentType> components, bool trigger, bool initial, json& out) {
stream.SetReadOffset(start);
out = json::array();
Reader r{ stream };
// Made for objects with a trigger after all the others (ObjectLoader2::DoObjectComponentLoad), so read last
if (trigger) components.push_back(TRIGGER);
for (const auto type : components) {
json fields = json::object();
const auto reader = Readers().find(type);
@@ -705,6 +855,18 @@ namespace {
return OnlyPadding(stream);
}
// Whether a construction's config (entries as "key=type:value") names a property or says it is in an inventory
bool MutableModel(const json& config) {
const auto& entries = config.is_object() ? config.value("entries", json::array()) : config;
if (!entries.is_array()) return false;
for (const auto& entry : entries) {
if (!entry.is_string()) continue;
const auto text = entry.get<std::string>();
if (text.starts_with("propertyObjectID=") || text == "inInventory=7:1") return true;
}
return false;
}
json ReadParentChild(Reader& r) {
json out = json::object();
if (!r.Bit()) return out;
@@ -734,7 +896,7 @@ namespace ReplicaDecoder {
}
std::vector<eReplicaComponentType> ComponentsOf(LOT lot, const ComponentTable& table) {
return Candidates(lot, table).front();
return Candidates(lot, table, false).front();
}
std::optional<json> Session::Decode(std::string_view bytes, uint64_t connection) {
@@ -773,7 +935,7 @@ namespace ReplicaDecoder {
out["parentChild"] = ReadParentChild(r);
json components;
const auto start = stream.GetReadOffset();
if (!r.ok || !ReadComponents(stream, start, it->second.components, false, components)) {
if (!r.ok || !ReadComponents(stream, start, it->second.components, it->second.trigger, false, components)) {
out["(layout did not match)"] = true;
out["(rest)"] = RestHex(stream);
}
@@ -792,7 +954,8 @@ namespace ReplicaDecoder {
out["name"] = r.WideText<uint8_t>();
out["timeSinceCreatedMs"] = r.Get<uint32_t>();
if (r.Bit()) out["config"] = ReadLdf(r);
out["trigger"] = r.Bit();
const bool trigger = r.Bit();
out["trigger"] = trigger;
if (r.Bit()) out["spawner"] = r.Id();
if (r.Bit()) out["spawnerNode"] = r.Get<uint32_t>();
if (r.Bit()) out["scale"] = r.Get<float>();
@@ -805,13 +968,13 @@ namespace ReplicaDecoder {
}
const auto start = stream.GetReadOffset();
const auto candidates = Candidates(lot, m_Table);
const auto candidates = Candidates(lot, m_Table, out.contains("config") && MutableModel(out["config"]));
std::vector<eReplicaComponentType> chosen = candidates.front();
json components;
bool matched = false;
for (const auto& candidate : candidates) {
json attempt;
if (ReadComponents(stream, start, candidate, true, attempt)) {
if (ReadComponents(stream, start, candidate, trigger, true, attempt)) {
chosen = candidate;
components = std::move(attempt);
matched = true;
@@ -820,13 +983,13 @@ namespace ReplicaDecoder {
}
if (!matched) {
// Shown as the registry says, as far as it reads, with the rest as bytes
ReadComponents(stream, start, chosen, true, components);
ReadComponents(stream, start, chosen, trigger, true, components);
out["(layout did not match)"] = true;
out["(rest)"] = RestHex(stream);
}
if (!m_Table.contains(lot) && lot != 14) out["(LOT not in ComponentsRegistry)"] = true;
out["components"] = components;
m_Objects[{ connection, network }] = Object{ objectId, lot, chosen };
m_Objects[{ connection, network }] = Object{ objectId, lot, chosen, trigger };
return out;
}
}

View File

@@ -51,6 +51,7 @@ namespace ReplicaDecoder {
LWOOBJID objectId{};
LOT lot{};
std::vector<eReplicaComponentType> components;
bool trigger{}; // its trigger component is read after the others
};
const ComponentTable& m_Table;

View File

@@ -127,12 +127,28 @@ thread, and the result is kept with the loaded capture:
each component's update. An object constructed before the capture started shows "object not constructed in this
capture" and the bytes.
Each component reader mirrors the component's `Serialize(bIsInitialUpdate)` (`ReplicaDecoderTests` writes real
components with the server's serializers and reads them back). Some objects have components their LOT doesn't list
(a smashable, moving platform or script set up by the zone file): when the registry's list doesn't read the packet
exactly (to the last whole byte, padding zero), those variants are tried. When none fits, what read is shown with
`(layout did not match)` and the rest as bits; parts the server never writes (only live did, such as local space
info) stop the reader with `(... present, not read)`. Nothing is guessed.
Each component reader follows the client's `Deserialize` for that component, checked against the 2014 live captures
and against the server's own serializers (`ReplicaDecoderTests` has live byte samples, and writes real components and
reads them back). Beyond the registry's list, the client makes a few components from other data, which the reader
adds the same way:
| Component | When |
|---|---|
| Trigger (a 1 bit, then the trigger ID) | the header's trigger bit is set; read after every other component |
| BBB (a 1 bit, then an object ID) | the registry lists component 107 (characters) |
| Mutable model behaviors (model block, behaviors, editing info) | a model whose config has `propertyObjectID` or `inInventory`; any other model reads only the model block |
| none for a pet's model and item | the client drops them |
Some objects differ from their LOT because of the zone file: a smashable's destroyable, a moving platform, a script,
`markedAsPhantom` (phantom physics instead of simple) and `renderDisabled` (no render data, as on trigger volumes).
When the registry's list doesn't read the packet exactly (to the last whole byte, padding zero), those variants are
tried. When none fits, what read is shown with `(layout did not match)` and the rest as bits. Nothing is guessed.
Compressed LDF (item and script settings, object config) is inflated and shown as its entries; narrow text that isn't
UTF-8 is shown byte by byte.
Against the live captures (every capture in the archive imported: 60,659 constructions, and 609,919 serializations of
objects constructed in the same capture), 83 constructions and 5 serializations don't read exactly: primitive models
(LOT 10042), the campfire activator (LOT 2248) and a quick build's updates (LOT 7001).
The capture tool reads game messages the same way (it links the game), so replays compare their fields, not only their
size.
@@ -275,3 +291,7 @@ decoder registry knows and checks it writes back to the same bytes; without fixt
`(layout did not match)` on players, enemies, smashables, NPCs); a later `ID_REPLICA_MANAGER_SERIALIZE` of an enemy
hit in the capture shows its new health; game messages both ways (a skill, an emote, a vendor purchase) show fields.
- Export a bundle, replay it with `CaptureTool replay`, and open a kept sandbox's logs.
- Moving platforms (Starbase 3001's snail, Nexus Tower's lifts) move as before, and objects built with them look right.
- A tamed pet that follows you through a fight and an untamed one being tamed: status and name updates show as before.
- A model placed in the world outside a property (a model reward) renders without stray effects; property models still
show their behaviors in property edit mode.

View File

@@ -41,3 +41,15 @@ TEST_F(PetTest, PlacementNewAddComponentTest) {
ASSERT_EQ(petComponent->GetParent()->GetObjectID(), 15);
ASSERT_EQ(petComponent->GetAbility(), ePetAbilityType::Invalid);
}
// The client reads the names bit on every update while the pet is dirty (as live wrote it); DLU wrote it only on
// construction, so the client took the next component's first bit for it
TEST_F(PetTest, UpdateWritesTheNamesBit) {
petComponent->Serialize(bitStream, false);
// dirty, status, ability, interaction, owner, names
EXPECT_EQ(bitStream.GetNumberOfBitsUsed(), 1u + 32 + 32 + 1 + 1 + 1);
bitStream.IgnoreBits(1 + 32 + 32 + 1 + 1);
bool names = true;
ASSERT_TRUE(bitStream.Read(names));
EXPECT_FALSE(names);
}

View File

@@ -258,6 +258,33 @@ TEST_F(ReplicaConstructionTest, SimplePhysicsConstructionLikeLive) {
info.settings.values.clear();
}
// The client reads a moving platform's subcomponents while a 1 bit comes before one (as live wrote them): the list
// ends with a 0 bit, which DLU left out, so the client took the next component's first bit for it
TEST_F(ReplicaConstructionTest, MovingPlatformEndsItsSubcomponentList) {
info.lot = 2349;
Entity platform(288300744895900005, info);
auto* const component = platform.AddComponent<MovingPlatformComponent>(-1, "");
component->SetSerialized(true);
RakNet::BitStream stream;
component->Serialize(stream, true);
bool bit{};
ASSERT_TRUE(stream.Read(bit));
EXPECT_TRUE(bit); // has subcomponents
ASSERT_TRUE(stream.Read(bit));
EXPECT_FALSE(bit); // no path
ASSERT_TRUE(stream.Read(bit));
EXPECT_TRUE(bit); // a subcomponent follows
uint32_t type{};
ASSERT_TRUE(stream.Read(type));
EXPECT_EQ(type, 4u); // mover
// The mover: a 1 bit, state, desired waypoint, 2 flags, percent, position, current and next waypoint, 2 times
stream.IgnoreBits(1 + 32 + 32 + 2 + 32 + 96 + 32 + 32 + 32 + 32);
ASSERT_TRUE(stream.Read(bit));
EXPECT_FALSE(bit); // no more subcomponents
EXPECT_EQ(stream.GetNumberOfUnreadBits(), 0u);
}
// Live replicated the DestructibleComponent factionList, so a list of -1 as [-1] (12,829 constructions: vendors,
// quickbuilds, bouncers; DLU dropped it and sent []), and a row with no faction but factionList 6 as [6].
TEST_F(ReplicaConstructionTest, TemplateFactionMinusOneIsReplicated) {

View File

@@ -19,6 +19,7 @@
#include "PlayerForcedMovementComponent.h"
#include "PossessorComponent.h"
#include "ReplicaDecoder.h"
#include "ZCompression.h"
#include "SimplePhysicsComponent.h"
#include "SkillComponent.h"
#include "User.h"
@@ -146,8 +147,8 @@ TEST_F(ReplicaDecoderTest, ModelWithTheDestroyableTheRegistryDoesNotList) {
info.lot = 6000;
Entity model(288300744895900200, info);
model.AddComponent<SimplePhysicsComponent>(-1);
// The model component writes the item's user-generated-content block itself, where the client reads the item
model.AddComponent<ModelComponent>(-1)->LoadBehaviors();
model.AddComponent<ItemComponent>(-1);
auto* const destroyable = model.AddComponent<DestroyableComponent>(-1);
destroyable->SetIsSmashable(true);
@@ -190,3 +191,179 @@ TEST_F(ReplicaDecoderTest, LiveConstructionHeader) {
EXPECT_EQ((*constructed)["lot"], 13006);
EXPECT_EQ((*constructed)["timeSinceCreatedMs"], 3814335);
}
namespace {
std::string FromHex(std::string_view hex) {
std::string bytes;
for (size_t i = 0; i + 1 < hex.size(); i += 2) bytes += static_cast<char>(std::stoi(std::string(hex.substr(i, 2)), nullptr, 16));
return bytes;
}
nlohmann::json DecodeLive(std::string_view hex, LOT lot, std::vector<eReplicaComponentType> registry) {
const ReplicaDecoder::ComponentTable table{ { lot, std::move(registry) } };
ReplicaDecoder::Session session(table);
const auto decoded = session.Decode(FromHex(hex), 1);
return decoded ? *decoded : nlohmann::json();
}
}
// Live constructions (2014 captures): each reads to its last whole byte
// A trigger object (the header's trigger bit set) has a trigger component read after all the others: a 1 bit and the
// trigger ID, -1 in every live capture. This trigger volume has no render data: its zone file sets renderDisabled.
TEST_F(ReplicaDecoderTest, LiveTriggerIsReadLast) {
const auto constructed = DecodeLive("24818c061f8000002000020a0b000000734cf8803de3f04008040000080000000773fe3d00928d12ec38dcef343c926e0c200000000a010e6bd000000002c617e3fbfffffffe", 5652, { RENDER, PHANTOM_PHYSICS });
EXPECT_FALSE(constructed.contains("(layout did not match)")) << constructed.dump();
EXPECT_EQ(constructed["trigger"], true);
EXPECT_EQ(Names(constructed), "PHANTOM_PHYSICS TRIGGER ");
EXPECT_EQ((*FieldsOf(constructed, "TRIGGER"))["triggerID"], -1);
}
// A phantom physics effect (a gravity scale, 4) with its amount
TEST_F(ReplicaDecoderTest, LivePhantomPhysicsEffect) {
const auto constructed = DecodeLive("24da8d409f0000002000023f09000000445b780031126020040420000800000004675ecd009b85b16c4cfa9514363a7b4c10000000086db54bf000000005b390e3fc10000003373130f4ffffffff80", 4734, { RENDER, PHANTOM_PHYSICS });
EXPECT_FALSE(constructed.contains("(layout did not match)")) << constructed.dump();
const auto* physics = FieldsOf(constructed, "PHANTOM_PHYSICS");
ASSERT_TRUE(physics);
EXPECT_EQ((*physics)["effectType"], 4);
EXPECT_FLOAT_EQ((*physics)["directionalMultiplier"].get<float>(), 0.05f);
}
// A phantom physics effect with its distance range (min and max)
TEST_F(ReplicaDecoderTest, LivePhantomPhysicsDistance) {
const auto constructed = DecodeLive("24ac038f86800000200002710680000040d88400398e03a00004000008000000009891de442d7ad85434aaaa9c30000000000000000000000000000803fc000000000001210a0001007e0000908522b93642588a88c107b330c1ffffffff80", 3554, { RENDER, PHANTOM_PHYSICS });
EXPECT_FALSE(constructed.contains("(layout did not match)")) << constructed.dump();
const auto* physics = FieldsOf(constructed, "PHANTOM_PHYSICS");
ASSERT_TRUE(physics);
EXPECT_TRUE(physics->contains("minDistance"));
EXPECT_TRUE(physics->contains("maxDistance"));
}
// A moving platform: its mover subcomponent, then the 0 bit that ends the subcomponent list
TEST_F(ReplicaDecoderTest, LiveMovingPlatform) {
const auto constructed = DecodeLive("248d800e0000000020000216848000006f3786801da2c3b0000400000800000005d45238fc80000000080000000000000000000000000000000000000000000000041000000346f43d874d0ca28689b34f82177a657ee056e67a8f0a6c7f7f99a679a0800000109000000ffffffff0000000000000000000000000000000000000000000000000000000000000000000000000", 2349, { RENDER, SIMPLE_PHYSICS, MOVING_PLATFORM, PLATFORM_BOUNDARY });
EXPECT_FALSE(constructed.contains("(layout did not match)")) << constructed.dump();
const auto* platform = FieldsOf(constructed, "MOVING_PLATFORM");
ASSERT_TRUE(platform);
ASSERT_EQ((*platform)["subcomponents"].size(), 1u);
EXPECT_EQ((*platform)["subcomponents"][0]["type"], 4);
}
// An enemy mid-attack: the skill it is casting with its running behaviors
TEST_F(ReplicaDecoderTest, LiveSkillsInProgress) {
const auto constructed = DecodeLive("24dff38ba48300002000021612000000200a80001c9f33a0000000000808000000800000000414000002a3720b896d24b48794199a820000000000000000000000000001007e4000000000000000000000000000000000000000000000000000000000000000000000002020000000001007e00000000000000000000000000000000000000000000200fc0000000000000000040000002c000000480800000008000001f81800000000000000000000080000001000000471e0000465c8000090000000ba48300002000020ba48300002000020000000000000000000000000000000000000000000000000", 9260, { RENDER, SIMPLE_PHYSICS, SCRIPT, DESTROYABLE, SKILL });
EXPECT_FALSE(constructed.contains("(layout did not match)")) << constructed.dump();
const auto* skill = FieldsOf(constructed, "SKILL");
ASSERT_TRUE(skill);
ASSERT_FALSE((*skill)["skillsInProgress"].empty());
EXPECT_FALSE((*skill)["skillsInProgress"][0]["behaviors"].empty());
}
// An object the zone file sets markedAsPhantom on: phantom physics where its LOT lists simple physics
TEST_F(ReplicaDecoderTest, LiveMarkedAsPhantom) {
const auto constructed = DecodeLive("24a302e403000000200002390e0000002cdd22009a506004040420000800000005e050b8f89da600ec2e412b3439e8626c300000000defb533f00000000e2850f3f8800000000000000000000000000000000000000000000000000000000000000000000000404000000000200fc00000000000000000000000000000000000000000000401f8000000000000000008000001b80000008000000000", 7282, { RENDER, SIMPLE_PHYSICS, DESTROYABLE, SKILL });
EXPECT_FALSE(constructed.contains("(layout did not match)")) << constructed.dump();
EXPECT_EQ(Names(constructed), "PHANTOM_PHYSICS BUFF DESTROYABLE SKILL RENDER ");
}
// Compressed LDF (as live sent item and script settings): u32 size, a 1 byte, u32 uncompressed and compressed sizes,
// then zlib data holding the entries, shown inflated
TEST_F(ReplicaDecoderTest, CompressedLdfIsInflated) {
RakNet::BitStream entries;
entries.Write<int32_t>(1);
const std::u16string key = u"name";
entries.Write<uint8_t>(key.size() * 2);
for (const auto c : key) entries.Write<uint16_t>(c);
entries.Write<uint8_t>(1); // i32
entries.Write<int32_t>(42);
std::vector<uint8_t> compressed(ZCompression::GetMaxCompressedLength(entries.GetNumberOfBytesUsed()));
const auto compressedSize = ZCompression::Compress(entries.GetData(), entries.GetNumberOfBytesUsed(), compressed.data(), compressed.size());
ASSERT_GT(compressedSize, 0);
RakNet::BitStream packet;
packet.Write<uint8_t>(ID_REPLICA_MANAGER_CONSTRUCTION);
packet.Write1();
packet.Write<uint16_t>(5);
packet.Write<int64_t>(288300744895900300);
packet.Write<int32_t>(7000);
packet.Write<uint8_t>(0); // name
packet.Write<uint32_t>(0);
packet.Write1(); // config
packet.Write<uint32_t>(1 + 4 + 4 + compressedSize);
packet.Write<uint8_t>(1);
packet.Write<uint32_t>(entries.GetNumberOfBytesUsed());
packet.Write<uint32_t>(compressedSize);
for (int32_t i = 0; i < compressedSize; i++) packet.Write<uint8_t>(compressed[i]);
for (int i = 0; i < 7; i++) packet.Write0(); // trigger, spawner, spawner node, scale, world state, GM level, parent/child
const ReplicaDecoder::ComponentTable table{ { 7000, {} } };
ReplicaDecoder::Session session(table);
const auto constructed = session.Decode(Bytes(packet), 1);
ASSERT_TRUE(constructed);
EXPECT_FALSE(constructed->contains("(layout did not match)")) << constructed->dump();
ASSERT_TRUE(constructed->contains("config")) << constructed->dump();
EXPECT_EQ((*constructed)["config"].value("entries", json()), json::array({ "name=1:42" })) << constructed->dump();
}
// Narrow text that isn't UTF-8 is shown byte by byte, so the viewer's JSON always writes
TEST_F(ReplicaDecoderTest, TextThatIsNotUtf8StillWrites) {
RakNet::BitStream packet;
packet.Write<uint8_t>(ID_REPLICA_MANAGER_CONSTRUCTION);
packet.Write1();
packet.Write<uint16_t>(6);
packet.Write<int64_t>(288300744895900301);
packet.Write<int32_t>(7001);
packet.Write<uint8_t>(0);
packet.Write<uint32_t>(0);
packet.Write1(); // config: one narrow string entry with a byte that isn't UTF-8
RakNet::BitStream entries;
entries.Write<int32_t>(1);
entries.Write<uint8_t>(2);
entries.Write<uint16_t>(u'k');
entries.Write<uint8_t>(13);
entries.Write<uint32_t>(2);
entries.Write<uint8_t>(0x10);
entries.Write<uint8_t>(0xE9);
packet.Write<uint32_t>(1 + entries.GetNumberOfBytesUsed());
packet.Write<uint8_t>(0);
for (uint32_t i = 0; i < entries.GetNumberOfBytesUsed(); i++) packet.Write<uint8_t>(entries.GetData()[i]);
for (int i = 0; i < 7; i++) packet.Write0();
const ReplicaDecoder::ComponentTable table{ { 7001, {} } };
ReplicaDecoder::Session session(table);
const auto constructed = session.Decode(Bytes(packet), 1);
ASSERT_TRUE(constructed);
EXPECT_NO_THROW(constructed->dump());
EXPECT_EQ((*constructed)["config"][0], "k=13:\x10\xC3\xA9");
}
// A model outside a property (a model reward in the world): the client makes the plain model component, which reads
// only the model's block, no behaviors; not in an inventory, it is smashable, with its destroyable last
TEST_F(ReplicaDecoderTest, LiveModelOutsideAProperty) {
const auto constructed = DecodeLive("248a802380000000200002458b800000375d87004f800000004f0000000d4000001e2718d898180432997219d2185219721842186a190a002c8620708318a94331501d8460cbd737a4f061606cd6e700430005b78231d2f2f3b0000400000800000000800000000414000002546af7863532ba87960375840000000000000000000000000001007f8b1700000000000000000000400000000546af7863532ba87960375840001007e000000000000000000000000000000000", 6027, { RENDER, SIMPLE_PHYSICS, ITEM, MODEL });
EXPECT_FALSE(constructed.contains("(layout did not match)")) << constructed.dump();
EXPECT_EQ(Names(constructed), "SIMPLE_PHYSICS ITEM MODEL RENDER DESTROYABLE ");
EXPECT_FALSE(FieldsOf(constructed, "MODEL")->contains("behaviors"));
}
// DLU's models: a property model (propertyObjectID in its settings) writes its behaviors, which the client reads with
// the mutable model component; any other writes only the model's block
TEST_F(ReplicaDecoderTest, ModelBehaviorsOnlyOnPropertyModels) {
const ReplicaDecoder::ComponentTable table{ { 6001, { SIMPLE_PHYSICS, ITEM, MODEL } } };
for (const bool onProperty : { false, true }) {
info.lot = 6001;
info.settings.values.clear();
if (onProperty) info.settings.Insert<bool>(u"propertyObjectID", true);
Entity model(288300744895900210, info);
model.AddComponent<SimplePhysicsComponent>(-1);
model.AddComponent<ModelComponent>(-1)->LoadBehaviors();
ReplicaDecoder::Session session(table);
const auto constructed = session.Decode(Construction(model, 4), 1);
ASSERT_TRUE(constructed);
EXPECT_FALSE(constructed->contains("(layout did not match)")) << constructed->dump();
const auto* fields = FieldsOf(*constructed, onProperty ? "MUTABLE_MODEL_BEHAVIORS" : "MODEL");
ASSERT_TRUE(fields) << Names(*constructed);
EXPECT_EQ(fields->contains("behaviors"), onProperty);
}
info.settings.values.clear();
}