From 4d7bea6f06bb69ab14936723b71ca57b23782893 Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Wed, 30 Sep 2026 04:43:09 -0500 Subject: [PATCH] test(capture): decoder coverage, replica read-back and generator check; docs Every LU MessageType has a struct or is listed as never used by this server; every game message struct reads; samples of each family and of sent game messages read back; replica constructions, updates and destructions written by the server's own serializers read back. Generator finds constructors defined in the .cpp. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 +- dGame/dGameMessages/GameMessageFields.inc | 17 ++ docs/CaptureReplay.md | 52 ++++- tests/dGameTests/CMakeLists.txt | 7 + .../dComponentsTests/CMakeLists.txt | 1 + .../dComponentsTests/ReplicaDecoderTests.cpp | 192 +++++++++++++++++ .../GameMessageDecoderTests.cpp | 15 +- tests/dGameTests/dNetTests/CMakeLists.txt | 1 + .../dNetTests/PacketDecoderCoverageTests.cpp | 195 ++++++++++++++++++ tools/gen_game_message_fields.py | 7 + 10 files changed, 482 insertions(+), 8 deletions(-) create mode 100644 tests/dGameTests/dComponentsTests/ReplicaDecoderTests.cpp create mode 100644 tests/dGameTests/dNetTests/PacketDecoderCoverageTests.cpp diff --git a/README.md b/README.md index 38ad219ba..28728013d 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,8 @@ A web dashboard for running and moderating a server, started and supervised by m viewer's language (picker in the user menu, or the browser's language), never written into the pages. * **Packet capture and replay:** record whole packets for an account, a character or everything; play them back with movement in World 3D, which follows a player across worlds (live too) with the world changes marked on the - timeline; replay bundles against a sandbox stack with a headless client. See + timeline; every packet decoded (LU packets and game messages by their own structs, replica constructions and + updates component by component); replay bundles against a sandbox stack with a headless client. See [docs/CaptureReplay.md](docs/CaptureReplay.md). * **Settings:** every setting the servers read, grouped by purpose with typed inputs, conditions, fuzzy search, history, and hot reload; values can be set on the page or kept in the `.ini` files. diff --git a/dGame/dGameMessages/GameMessageFields.inc b/dGame/dGameMessages/GameMessageFields.inc index 78104641d..cc7c5d05c 100644 --- a/dGame/dGameMessages/GameMessageFields.inc +++ b/dGame/dGameMessages/GameMessageFields.inc @@ -129,6 +129,7 @@ json ToJson(const GameMessages::RequestLinkedMission& m); json ToJson(const GameMessages::SetFlag& m); json ToJson(const GameMessages::NotifyClientFlagChange& m); json ToJson(const GameMessages::HasBeenCollected& m); +json ToJson(const GameMessages::SetMissionTypeState& m); json ToJson(const GameMessages::NotifyLevelRewards& m); json ToJson(const GameMessages::Teleport& m); json ToJson(const GameMessages::StartPathing& m); @@ -230,6 +231,7 @@ json ToJson(const GameMessages::EnterProperty1& m); json ToJson(const GameMessages::UpdatePropertyPerformanceCost& m); json ToJson(const GameMessages::ReportOffensiveModel& m); json ToJson(const GameMessages::ReportOffensiveProperty& m); +json ToJson(const GameMessages::ControlBehaviors& m); json ToJson(const GameMessages::GetHotPropertyData& m); json ToJson(const GameMessages::NewsSendHotPropertiesInfoToClient& m); json ToJson(const GameMessages::NewsSendHotPropertiesInfoToClient::HotPropertyInfo& m); @@ -1310,6 +1312,13 @@ json ToJson(const GameMessages::HasBeenCollected& m) { j["playerID"] = ToJson(m.playerID); return j; } +json ToJson(const GameMessages::SetMissionTypeState& m) { + json j = json::object(); + j["state"] = ToJson(m.state); + j["subtype"] = ToJson(m.subtype); + j["type"] = ToJson(m.type); + return j; +} json ToJson(const GameMessages::NotifyLevelRewards& m) { json j = json::object(); j["level"] = ToJson(m.level); @@ -2007,6 +2016,12 @@ json ToJson(const GameMessages::ReportOffensiveProperty& m) { j["propertyPlaqueObjectID"] = ToJson(m.propertyPlaqueObjectID); return j; } +json ToJson(const GameMessages::ControlBehaviors& m) { + json j = json::object(); + j["args"] = ToJson(m.args); + j["command"] = ToJson(m.command); + return j; +} json ToJson(const GameMessages::GetHotPropertyData& m) { json j = json::object(); return j; @@ -2627,6 +2642,7 @@ const std::vector& Entries() { { MessageType::Game::SET_FLAG, eDirection::TO_SERVER, "SetFlag", &ReadWith }, { MessageType::Game::NOTIFY_CLIENT_FLAG_CHANGE, eDirection::TO_CLIENT, "NotifyClientFlagChange", &ReadWith }, { MessageType::Game::HAS_BEEN_COLLECTED, eDirection::TO_SERVER, "HasBeenCollected", &ReadWith }, + { MessageType::Game::SET_MISSION_TYPE_STATE, eDirection::TO_SERVER, "SetMissionTypeState", &ReadWith }, { MessageType::Game::NOTIFY_LEVEL_REWARDS, eDirection::TO_CLIENT, "NotifyLevelRewards", &ReadWith }, { MessageType::Game::TELEPORT, eDirection::TO_CLIENT, "Teleport", &ReadWith }, { MessageType::Game::START_PATHING, eDirection::TO_CLIENT, "StartPathing", &ReadWith }, @@ -2727,6 +2743,7 @@ const std::vector& Entries() { { MessageType::Game::UPDATE_PROPERTY_PERFORMANCE_COST, eDirection::TO_SERVER, "UpdatePropertyPerformanceCost", &ReadWith }, { MessageType::Game::REPORT_OFFENSIVE_MODEL, eDirection::TO_SERVER, "ReportOffensiveModel", &ReadWith }, { MessageType::Game::REPORT_OFFENSIVE_PROPERTY, eDirection::TO_SERVER, "ReportOffensiveProperty", &ReadWith }, + { MessageType::Game::CONTROL_BEHAVIORS, eDirection::TO_SERVER, "ControlBehaviors", &ReadWith }, { MessageType::Game::GET_HOT_PROPERTY_DATA, eDirection::TO_SERVER, "GetHotPropertyData", &ReadWith }, { MessageType::Game::SEND_HOT_PROPERTY_DATA, eDirection::TO_CLIENT, "NewsSendHotPropertiesInfoToClient", &ReadWith }, { MessageType::Game::PLAY_BEHAVIOR_SOUND, eDirection::TO_CLIENT, "PlayBehaviorSound", &ReadWith }, diff --git a/docs/CaptureReplay.md b/docs/CaptureReplay.md index 3310db184..75a85de7c 100644 --- a/docs/CaptureReplay.md +++ b/docs/CaptureReplay.md @@ -90,6 +90,53 @@ A busy world sends a few thousand packets a second, so capturing everything ther Appending to a file is about 80 times cheaper than a database row per packet, so packets go to files and the database keeps only the session row (the game message inspector keeps its rows as before). +## Decoding + +The viewer shows every packet with named fields and values, and its bytes below. Nothing is parsed twice: packets are +read with the server's own structs and their `Deserialize`. + +| Packets | How | +|---|---| +| LU packets (auth, chat, client, master, world, common) | `dNet/PacketDecoder.cpp`: the struct for its (service, ID). An ID with a struct per direction is read with the first that reads the whole packet, the direction's own first. | +| Game messages, both directions | `dGame/dGameMessages/GameMessageDecoder.cpp`: the `NetGameMsg` struct for its ID (the one the server reads a client's message with, else the one it sends). | +| Replica constructions, serializations, destructions | `dGame/dUtilities/ReplicaDecoder.cpp`, below. | + +The members of each struct are listed by `tools/gen_game_message_fields.py`, which writes `dNet/PacketFields.inc` and +`dGame/dGameMessages/GameMessageFields.inc` from the struct definitions; run it after changing a packet struct (the +`PacketFieldsUpToDate` test fails until then). Every member type needs a `ToJson` overload (`dNet/PacketJson.h`), or +the build fails. Members holding secrets (passwords, session and user keys) are never listed. Whole bytes a struct +leaves unread are shown as `(unread bits)`. + +What isn't decoded shows its name (from the `MessageType` enums) and its bytes: LU packet IDs this server never sends +or handles (listed in `PacketDecoderCoverageTests`, which fails for any other ID without a struct), game message IDs +the server has no struct for (live-only messages), and the client's `MAIL` packet (its sub-messages are read by the +game's mail code). Every game message the server sends or reads has a struct. + +### Replica packets + +Replica packets depend on what came before, so a capture is read once in timeline order, on a dashboard worker +thread, and the result is kept with the loaded capture: + +- **Destruction:** the network ID, and the object it was when the capture had its construction. +- **Construction:** the object header as `Entity::WriteBaseReplicaData` writes it (network ID, object ID, LOT, name, + age, config, trigger, spawner, spawner node, scale, world state, GM level, parent and children), then each + component's data in the order the client reads them (`Entity::WriteComponents`). Which components an object has + comes from the ComponentsRegistry rows of its LOT, read from the CDClient at startup, with the ones the server adds + itself (a destroyable for collectibles, quick builds and models, the character's parts, a buff with a destroyable). +- **Serialization:** the object's LOT and components from its construction earlier on the same world instance, then + each component's update. An object constructed before the capture started shows "object not constructed in this + capture" and the bytes. + +Each component reader mirrors the component's `Serialize(bIsInitialUpdate)` (`ReplicaDecoderTests` writes real +components with the server's serializers and reads them back). Some objects have components their LOT doesn't list +(a smashable, moving platform or script set up by the zone file): when the registry's list doesn't read the packet +exactly (to the last whole byte, padding zero), those variants are tried. When none fits, what read is shown with +`(layout did not match)` and the rest as bits; parts the server never writes (only live did, such as local space +info) stop the reader with `(... present, not read)`. Nothing is guessed. + +The capture tool reads game messages the same way (it links the game), so replays compare their fields, not only their +size. + ## The bundle format One format for the dashboard's capture files, exported bundles and converted live captures (`dNet/CaptureBundle.h`): @@ -147,7 +194,7 @@ CaptureTool replay ... --client [--cdserver ] [--keep | --keep-on-failure] [--report ] CaptureTool import-live convert live captures CaptureTool anonymise make a fixture -CaptureTool info|decode look inside +CaptureTool info|decode [--cdserver ] look inside (replica packets too with --cdserver) ``` ### The sandbox @@ -224,4 +271,7 @@ decoder registry knows and checks it writes back to the same bytes; without fixt - Capture a character that changes worlds (a rocket or a portal): the capture page's slider has a mark at each change named after the new zone; in World 3D, following the character switches the scene at the mark and the camera stays on them, forward and when seeking back. +- In a capture of a zone load: every packet shows fields; constructions list their components (no + `(layout did not match)` on players, enemies, smashables, NPCs); a later `ID_REPLICA_MANAGER_SERIALIZE` of an enemy + hit in the capture shows its new health; game messages both ways (a skill, an emote, a vendor purchase) show fields. - Export a bundle, replay it with `CaptureTool replay`, and open a kept sandbox's logs. diff --git a/tests/dGameTests/CMakeLists.txt b/tests/dGameTests/CMakeLists.txt index 5637b6f51..450cec427 100644 --- a/tests/dGameTests/CMakeLists.txt +++ b/tests/dGameTests/CMakeLists.txt @@ -66,3 +66,10 @@ target_link_libraries(dGameTests ${COMMON_LIBRARIES} sqlite3 GTest::gtest_main # Discover the tests gtest_discover_tests(dGameTests) + +# The member lists the capture viewer shows packets with are generated from the packet structs: fail when a struct +# changed without running the generator +find_package(Python3 COMPONENTS Interpreter) +if(Python3_Interpreter_FOUND) + add_test(NAME PacketFieldsUpToDate COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/tools/gen_game_message_fields.py --check) +endif() diff --git a/tests/dGameTests/dComponentsTests/CMakeLists.txt b/tests/dGameTests/dComponentsTests/CMakeLists.txt index 6efad2888..28e0f3be7 100644 --- a/tests/dGameTests/dComponentsTests/CMakeLists.txt +++ b/tests/dGameTests/dComponentsTests/CMakeLists.txt @@ -17,6 +17,7 @@ set(DCOMPONENTS_TESTS "QuickBuildCompleteTests.cpp" "RocketLaunchTests.cpp" "ReplicaConstructionTests.cpp" + "ReplicaDecoderTests.cpp" "ItemRemovalTests.cpp" ) diff --git a/tests/dGameTests/dComponentsTests/ReplicaDecoderTests.cpp b/tests/dGameTests/dComponentsTests/ReplicaDecoderTests.cpp new file mode 100644 index 000000000..3f23f5959 --- /dev/null +++ b/tests/dGameTests/dComponentsTests/ReplicaDecoderTests.cpp @@ -0,0 +1,192 @@ +// The capture viewer's replica reader (ReplicaDecoder) against the server's own writers: objects are built with real +// components, written with Entity::WriteBaseReplicaData and WriteComponents as a construction or serialization packet, +// and read back. A component whose Serialize changes without its reader fails here. +#include "GameDependencies.h" +#include + +#include "BitStream.h" +#include "BuffComponent.h" +#include "Character.h" +#include "CharacterComponent.h" +#include "ControllablePhysicsComponent.h" +#include "DestroyableComponent.h" +#include "Entity.h" +#include "InventoryComponent.h" +#include "ItemComponent.h" +#include "LevelProgressionComponent.h" +#include "MessageIdentifiers.h" +#include "ModelComponent.h" +#include "PlayerForcedMovementComponent.h" +#include "PossessorComponent.h" +#include "ReplicaDecoder.h" +#include "SimplePhysicsComponent.h" +#include "SkillComponent.h" +#include "User.h" +#include "eReplicaComponentType.h" +#include "eReplicaPacketType.h" + +namespace { + using json = nlohmann::json; + using enum eReplicaComponentType; + + std::string Bytes(const RakNet::BitStream& stream) { return std::string(reinterpret_cast(stream.GetData()), stream.GetNumberOfBytesUsed()); } + + std::string Construction(Entity& entity, uint16_t network) { + RakNet::BitStream stream; + stream.Write(ID_REPLICA_MANAGER_CONSTRUCTION); + stream.Write1(); + stream.Write(network); + entity.WriteBaseReplicaData(stream, eReplicaPacketType::CONSTRUCTION); + entity.WriteComponents(stream, eReplicaPacketType::CONSTRUCTION); + return Bytes(stream); + } + + std::string Serialization(Entity& entity, uint16_t network) { + RakNet::BitStream stream; + stream.Write(ID_REPLICA_MANAGER_SERIALIZE); + stream.Write(network); + entity.WriteBaseReplicaData(stream, eReplicaPacketType::SERIALIZATION); + entity.WriteComponents(stream, eReplicaPacketType::SERIALIZATION); + return Bytes(stream); + } + + const json* FieldsOf(const json& fields, const std::string& name) { + for (const auto& component : fields["components"]) { + if (component["component"] == name) return &component["fields"]; + } + return nullptr; + } + + std::string Names(const json& fields) { + std::string names; + for (const auto& component : fields["components"]) names += component["component"].get() + " "; + return names; + } +} + +class ReplicaDecoderTest : public GameDependenciesTest { +protected: + void SetUp() override { SetUpDependencies(); } + void TearDown() override { TearDownDependencies(); } +}; + +TEST_F(ReplicaDecoderTest, EnemyConstructionAndSerializationReadBack) { + info.lot = 4712; + Entity enemy(288300744895900100, info); + auto* const physics = enemy.AddComponent(-1); + auto* const destroyable = enemy.AddComponent(-1); + destroyable->SetMaxHealth(8.0f); + destroyable->SetHealth(8); + physics->SetPosition(NiPoint3(10.0f, 20.0f, 30.0f)); + + // A smashable (is_smashable in the zone file): its destroyable isn't in the registry and is written last, which the + // reader finds by trying that layout when the registry's doesn't read exactly + const ReplicaDecoder::ComponentTable table{ { 4712, { CONTROLLABLE_PHYSICS } } }; + ReplicaDecoder::Session session(table); + const auto constructed = session.Decode(Construction(enemy, 7), 1); + ASSERT_TRUE(constructed); + EXPECT_FALSE(constructed->contains("(layout did not match)")) << constructed->dump(); + EXPECT_EQ((*constructed)["networkID"], 7); + EXPECT_EQ((*constructed)["objectID"], "288300744895900100"); + EXPECT_EQ((*constructed)["lot"], 4712); + EXPECT_EQ(Names(*constructed), "CONTROLLABLE_PHYSICS DESTROYABLE "); + const auto* health = FieldsOf(*constructed, "DESTROYABLE"); + ASSERT_TRUE(health); + EXPECT_EQ((*health)["health"], 8); + EXPECT_EQ((*FieldsOf(*constructed, "CONTROLLABLE_PHYSICS"))["position"], json::array({ 10.0f, 20.0f, 30.0f })); + + // An update: only what changed, read with the components the construction had + destroyable->SetHealth(3); + const auto updated = session.Decode(Serialization(enemy, 7), 1); + ASSERT_TRUE(updated); + EXPECT_FALSE(updated->contains("(layout did not match)")) << updated->dump(); + EXPECT_EQ((*updated)["lot"], 4712); + EXPECT_EQ((*FieldsOf(*updated, "DESTROYABLE"))["health"], 3); + + // Another connection never saw the construction + const auto unknown = session.Decode(Serialization(enemy, 7), 2); + ASSERT_TRUE(unknown); + EXPECT_TRUE(unknown->contains("(object not constructed in this capture)")); + + // Destruction: the network ID, and the object it was + RakNet::BitStream destruction; + destruction.Write(ID_REPLICA_MANAGER_DESTRUCTION); + destruction.Write(7); + const auto destroyed = session.Decode(Bytes(destruction), 1); + ASSERT_TRUE(destroyed); + EXPECT_EQ((*destroyed)["networkID"], 7); + EXPECT_EQ((*destroyed)["lot"], 4712); + EXPECT_TRUE(session.Decode(Serialization(enemy, 7), 1)->contains("(object not constructed in this capture)")); +} + +TEST_F(ReplicaDecoderTest, PlayerConstructionReadsTheCharacterParts) { + User user(UNASSIGNED_SYSTEM_ADDRESS, "tester", "key"); + Character character(1, &user); + info.lot = 1; + Entity player(1152921506064087003, info); + player.SetCharacter(&character); + character.SetEntity(&player); + player.AddComponent(-1); + player.AddComponent(-1); + player.AddComponent(-1); + player.AddComponent(-1); + player.AddComponent(-1, &character, UNASSIGNED_SYSTEM_ADDRESS)->InitializeStatisticsFromString(""); + + const ReplicaDecoder::ComponentTable table{ { 1, { CONTROLLABLE_PHYSICS, CHARACTER } } }; + ReplicaDecoder::Session session(table); + const auto constructed = session.Decode(Construction(player, 1), 1); + ASSERT_TRUE(constructed); + EXPECT_FALSE(constructed->contains("(layout did not match)")) << constructed->dump(); + EXPECT_EQ(Names(*constructed), "CONTROLLABLE_PHYSICS POSSESSOR LEVEL_PROGRESSION PLAYER_FORCED_MOVEMENT CHARACTER "); + EXPECT_EQ((*FieldsOf(*constructed, "CHARACTER"))["statistics"].size(), 27u); +} + +// A model's destroyable is not in its registry rows: it is written after the other components +TEST_F(ReplicaDecoderTest, ModelWithTheDestroyableTheRegistryDoesNotList) { + info.lot = 6000; + Entity model(288300744895900200, info); + model.AddComponent(-1); + model.AddComponent(-1)->LoadBehaviors(); + model.AddComponent(-1); + auto* const destroyable = model.AddComponent(-1); + destroyable->SetIsSmashable(true); + + const ReplicaDecoder::ComponentTable table{ { 6000, { SIMPLE_PHYSICS, MODEL, ITEM } } }; + ReplicaDecoder::Session session(table); + const auto constructed = session.Decode(Construction(model, 3), 1); + ASSERT_TRUE(constructed); + EXPECT_FALSE(constructed->contains("(layout did not match)")) << constructed->dump(); + EXPECT_EQ(Names(*constructed), "SIMPLE_PHYSICS ITEM MODEL DESTROYABLE "); + EXPECT_EQ((*FieldsOf(*constructed, "DESTROYABLE"))["smashable"], true); +} + +// When no layout reads the packet exactly, what read is shown with the rest as bits, never a guess +TEST_F(ReplicaDecoderTest, UnknownLayoutShowsTheRest) { + info.lot = 4712; + Entity enemy(288300744895900100, info); + enemy.AddComponent(-1); + auto packet = Construction(enemy, 9); + packet += std::string("\x12\x34\x56", 3); + + const ReplicaDecoder::ComponentTable table{ { 4712, { CONTROLLABLE_PHYSICS } } }; + ReplicaDecoder::Session session(table); + const auto constructed = session.Decode(packet, 1); + ASSERT_TRUE(constructed); + EXPECT_TRUE(constructed->contains("(layout did not match)")); + EXPECT_FALSE((*constructed)["(rest)"].get().empty()); +} + +// A live construction (network ID 11, LOT 13006, no components, parent/child info present but empty): the object +// header reads the same as the server writes it +TEST_F(ReplicaDecoderTest, LiveConstructionHeader) { + const unsigned char live[] = { 0x24, 0x85, 0x80, 0x7f, 0x7f, 0xff, 0xff, 0xff, 0x9f, 0x80, 0x00, 0x67, 0x19, 0x00, 0x00, 0x00, 0x5f, 0x99, 0x9d, 0x00, 0x00, 0x80 }; + const ReplicaDecoder::ComponentTable table{ { 13006, {} } }; + ReplicaDecoder::Session session(table); + const auto constructed = session.Decode(std::string(reinterpret_cast(live), sizeof(live)), 1); + ASSERT_TRUE(constructed); + EXPECT_FALSE(constructed->contains("(layout did not match)")) << constructed->dump(); + EXPECT_EQ((*constructed)["networkID"], 11); + EXPECT_EQ((*constructed)["objectID"], "70368744177662"); + EXPECT_EQ((*constructed)["lot"], 13006); + EXPECT_EQ((*constructed)["timeSinceCreatedMs"], 3814335); +} diff --git a/tests/dGameTests/dGameMessagesTests/GameMessageDecoderTests.cpp b/tests/dGameTests/dGameMessagesTests/GameMessageDecoderTests.cpp index 5529b305d..f2793c9ac 100644 --- a/tests/dGameTests/dGameMessagesTests/GameMessageDecoderTests.cpp +++ b/tests/dGameTests/dGameMessagesTests/GameMessageDecoderTests.cpp @@ -39,28 +39,31 @@ TEST(GameMessageDecoderTest, ReadsWhatTheServerWrites) { EXPECT_EQ((*fields)["skillID"], 42); EXPECT_EQ((*fields)["optionalOriginatorID"], "1234"); EXPECT_EQ((*fields)["optionalTargetID"], "77"); - EXPECT_EQ((*fields)["sBitStream"], "0102"); + EXPECT_EQ((*fields)["sBitStream"], nlohmann::json({ {"hex", "0102"} })); } -TEST(GameMessageDecoderTest, DirectionMatters) { +// A message only sent one way is read with its struct in either direction (the layout is the same) +TEST(GameMessageDecoderTest, SentMessagesDecode) { GameMessages::EchoSyncSkill sent; sent.bDone = true; sent.uiSkillHandle = 5; RakNet::BitStream stream; sent.Serialize(stream); - EXPECT_FALSE(GameMessageDecoder::CanDecode(MessageType::Game::ECHO_SYNC_SKILL, true)); + EXPECT_TRUE(GameMessageDecoder::CanDecode(MessageType::Game::ECHO_SYNC_SKILL, false)); const auto fields = GameMessageDecoder::Decode(MessageType::Game::ECHO_SYNC_SKILL, false, stream); ASSERT_TRUE(fields); EXPECT_EQ((*fields)["bDone"], true); EXPECT_EQ((*fields)["uiSkillHandle"], 5); } -TEST(GameMessageDecoderTest, UntypedMessagesAreNotDecoded) { +// A message the server has no struct for (live only) isn't decoded +TEST(GameMessageDecoderTest, MessagesWithoutAStructAreNotDecoded) { RakNet::BitStream payload; payload.Write(1); - EXPECT_FALSE(GameMessageDecoder::CanDecode(MessageType::Game::PLAY_EMOTE, true)); - EXPECT_FALSE(GameMessageDecoder::Decode(MessageType::Game::PLAY_EMOTE, true, payload)); + EXPECT_FALSE(GameMessageDecoder::HasStruct(MessageType::Game::SET_PVP_STATUS)); + EXPECT_FALSE(GameMessageDecoder::CanDecode(MessageType::Game::SET_PVP_STATUS, true)); + EXPECT_FALSE(GameMessageDecoder::Decode(MessageType::Game::SET_PVP_STATUS, true, payload)); } TEST(GameMessageDecoderTest, ShortPayloadsFail) { diff --git a/tests/dGameTests/dNetTests/CMakeLists.txt b/tests/dGameTests/dNetTests/CMakeLists.txt index 0f2c36ef4..fb204ca5e 100644 --- a/tests/dGameTests/dNetTests/CMakeLists.txt +++ b/tests/dGameTests/dNetTests/CMakeLists.txt @@ -5,6 +5,7 @@ SET(DNET_TESTS "MasterPacketsTests.cpp" "MatchmakingPacketsTests.cpp" "PacketCaptureTests.cpp" + "PacketDecoderCoverageTests.cpp" "ServerTrafficTests.cpp" "WorldPacketsTests.cpp") diff --git a/tests/dGameTests/dNetTests/PacketDecoderCoverageTests.cpp b/tests/dGameTests/dNetTests/PacketDecoderCoverageTests.cpp new file mode 100644 index 000000000..a1e1dd349 --- /dev/null +++ b/tests/dGameTests/dNetTests/PacketDecoderCoverageTests.cpp @@ -0,0 +1,195 @@ +// The capture viewer decodes every packet the server sends or reads: every LU message ID has a struct (or is on the +// list below of IDs this server never sends or handles), every game message struct reads, and samples of each family +// come back with the values they were written with. +#include + +#include +#include + +#include "AuthPackets.h" +#include "ChatPackets.h" +#include "ClientPackets.h" +#include "CommonPackets.h" +#include "EffectsMessages.h" +#include "GameMessageDecoder.h" +#include "GameMessageHandler.h" +#include "MasterPackets.h" +#include "MessageType/Auth.h" +#include "MessageType/Chat.h" +#include "MessageType/Client.h" +#include "MessageType/Master.h" +#include "MessageType/Server.h" +#include "MessageType/World.h" +#include "PacketDecoder.h" +#include "ServiceType.h" +#include "SkillMessages.h" +#include "TradeMessages.h" +#include "WorldPackets.h" +#include "magic_enum.hpp" + +namespace { + using json = nlohmann::json; + + std::string Bytes(const RakNet::BitStream& stream) { return std::string(reinterpret_cast(stream.GetData()), stream.GetNumberOfBytesUsed()); } + + template + PacketDecoder::Decoded Written(const T& packet, bool fromClient) { + RakNet::BitStream stream; + packet.WritePacket(stream); + return PacketDecoder::Decode(Bytes(stream), fromClient); + } + + // IDs this server never sends and never handles: no struct, so the viewer shows their name and bytes + const std::set NOT_USED{ + "AUTH LOGOUT_REQUEST", "AUTH CREATE_NEW_ACCOUNT_REQUEST", "AUTH LEGOINTERFACE_AUTH_RESPONSE", "AUTH SESSIONKEY_RECEIVED_CONFIRM", + "CHAT USER_CHANNEL_CHAT_MESSAGE", "CHAT WORLD_DISCONNECT_REQUEST", "CHAT WORLD_PROXIMITY_RESPONSE", "CHAT WORLD_PARCEL_RESPONSE", + "CHAT TEAM_MISSED_INVITE_CHECK", "CHAT BLUEPRINT_MODERATED", "CHAT BLUEPRINT_MODEL_READY", "CHAT PROPERTY_READY_FOR_APPROVAL", + "CHAT PROPERTY_MODERATION_CHANGED", "CHAT PROPERTY_BUILDMODE_CHANGED", "CHAT PROPERTY_BUILDMODE_CHANGED_REPORT", + "CHAT WORLD_INSTANCE_LOCATION_REQUEST", "CHAT REPUTATION_UPDATE", "CHAT SEND_CANNED_TEXT", "CHAT CHARACTER_NAME_CHANGE_REQUEST", + "CHAT CSR_REQUEST", "CHAT CSR_REPLY", "CHAT GM_KICK", "CHAT ACTIVITY_UPDATE", "CHAT GET_ZONE_POPULATIONS", "CHAT MATCH_REQUEST", + "CHAT UGCMANIFEST_REPORT_MISSING_FILE", "CHAT UGCMANIFEST_REPORT_DONE_FILE", "CHAT UGCMANIFEST_REPORT_DONE_BLUEPRINT", + "CHAT UGCC_REQUEST", "CHAT WORLD_PLAYERS_PET_MODERATED_ACKNOWLEDGE", "CHAT GM_CLOSE_PRIVATE_CHAT_WINDOW", "CHAT PLAYER_READY", + "CHAT GET_DONATION_TOTAL", "CHAT UPDATE_DONATION", "CHAT PRG_CSR_COMMAND", "CHAT HEARTBEAT_REQUEST_FROM_WORLD", + "CHAT UPDATE_FREE_TRIAL_STATUS", + "CLIENT LOGOUT_RESPONSE", "CLIENT CREATE_OBJECT", "CLIENT CREATE_CHARACTER_EXTENDED", "CLIENT CHAT_CONNECT_RESPONSE", + "CLIENT AUTH_ACCOUNT_CREATE_RESPONSE", "CLIENT CONNECT_CHAT", "CLIENT IMPENDING_RELOAD_NOTIFY", "CLIENT SLASH_PUSH_MAP_RESPONSE", + "CLIENT SLASH_PULL_MAP_RESPONSE", "CLIENT SLASH_LOCK_MAP_RESPONSE", "CLIENT BLUEPRINT_LUP_SAVE_RESPONSE", + "CLIENT BLUEPRINT_GET_ALL_DATA_RESPONSE", "CLIENT MODEL_INSTANTIATE_RESPONSE", "CLIENT GUILD_GET_STATUS_RESPONSE", + "CLIENT GUILD_RANK_CHANGE", "CLIENT GUILD_STATUS", "CLIENT DB_PROXY_RESULT", "CLIENT UPDATE_CHARACTER_NAME", + "CLIENT SET_NETWORK_SIMULATOR", "CLIENT INVALID_CHAT_MESSAGE", "CLIENT IN_LOGIN_QUEUE", "CLIENT GM_CLOSE_TARGET_CHAT_WINDOW", + "CLIENT GENERAL_TEXT_FOR_LOCALIZATION", "CLIENT UPDATE_FREE_TRIAL_STATUS", + "MASTER SHUTDOWN_IMMEDIATE", "AUTH RUNTIME_CONFIG", "CLIENT UGC_DOWNLOAD_FAILED", + "WORLD HAPPY_FLOWER_MODE_NOTIFY", "WORLD SLASH_RELOAD_MAP", "WORLD SLASH_PUSH_MAP_REQUEST", "WORLD SLASH_PUSH_MAP", + "WORLD SLASH_PULL_MAP", "WORLD LOCK_MAP_REQUEST", "WORLD HTTP_MONITOR_INFO_REQUEST", "WORLD SLASH_DEBUG_SCRIPTS", + "WORLD MODELS_CLEAR", "WORLD EXHIBIT_INSERT_MODEL", "WORLD WORD_CHECK", "WORLD GET_PLAYERS_IN_ZONE", + "WORLD BLUEPRINT_GET_ALL_DATA_REQUEST", "WORLD CANCEL_MAP_QUEUE", "WORLD FAKE_PRG_CSR_MESSAGE", + "WORLD REQUEST_FREE_TRIAL_REFRESH", "WORLD GM_SET_FREE_TRIAL_STATUS", + }; + + // Decoded some other way: game messages (GameMessageDecoder) and mail (a sub-header of its own, read by the game) + const std::set ELSEWHERE{ "CLIENT GAME_MSG", "CLIENT MAIL" }; + + template + void ExpectEveryIdDecodes(ServiceType service, std::set& missing) { + for (const auto id : magic_enum::enum_values()) { + const auto name = std::string(magic_enum::enum_name(service)) + " " + std::string(magic_enum::enum_name(id)); + if (PacketDecoder::HasFields(service, static_cast(id))) { + EXPECT_FALSE(NOT_USED.contains(name)) << name << " has a struct now: take it off the list"; + continue; + } + if (!NOT_USED.contains(name) && !ELSEWHERE.contains(name)) missing.insert(name); + } + } +} + +TEST(PacketDecoderCoverageTest, EveryMessageTypeHasAStruct) { + std::set missing; + ExpectEveryIdDecodes(ServiceType::COMMON, missing); + ExpectEveryIdDecodes(ServiceType::AUTH, missing); + ExpectEveryIdDecodes(ServiceType::CHAT, missing); + ExpectEveryIdDecodes(ServiceType::CLIENT, missing); + ExpectEveryIdDecodes(ServiceType::MASTER, missing); + ExpectEveryIdDecodes(ServiceType::WORLD, missing); + std::string list; + for (const auto& name : missing) list += name + "\n"; + EXPECT_TRUE(missing.empty()) << "No struct decodes these (add one, or list them as not used):\n" << list; +} + +// Every game message the server has a struct for reads, and so does every message it handles from a client +TEST(PacketDecoderCoverageTest, EveryGameMessageStructReads) { + const auto decodable = GameMessageDecoder::Decodable(); + EXPECT_GT(decodable.size(), 290u); + for (const auto id : magic_enum::enum_values()) { + if (!GameMessageHandler::CreateReceived(id)) continue; + EXPECT_TRUE(GameMessageDecoder::CanDecode(id, true)) << magic_enum::enum_name(id); + } + for (const auto id : decodable) { + EXPECT_TRUE(GameMessageDecoder::CanDecode(id, true) && GameMessageDecoder::CanDecode(id, false)) << magic_enum::enum_name(id); + } +} + +TEST(PacketDecoderCoverageTest, AuthLoginRequestNeverShowsThePassword) { + AuthPackets::LoginRequest login; + login.username.string = u"tester"; + login.password.string = u"hunter2"; + login.numberOfProcessors = 8; + const auto decoded = Written(login, true); + ASSERT_TRUE(decoded.fields); + EXPECT_EQ((*decoded.fields)["username"], "tester"); + EXPECT_EQ((*decoded.fields)["numberOfProcessors"], 8); + EXPECT_FALSE(decoded.fields->contains("password")); + EXPECT_EQ(decoded.fields->dump().find("hunter2"), std::string::npos); +} + +TEST(PacketDecoderCoverageTest, SamplesOfEachFamilyReadBack) { + CommonPackets::ServerVersionConfirm version; + version.netVersion = 171022; + auto decoded = Written(version, false); + ASSERT_TRUE(decoded.fields); + EXPECT_EQ((*decoded.fields)["netVersion"], 171022); + + ChatPackets::GMMute mute; + mute.playerID = 1152921504606846999LL; + mute.expire = 60; + decoded = Written(mute, false); + ASSERT_TRUE(decoded.fields) << decoded.name; + EXPECT_EQ((*decoded.fields)["playerID"], "1152921504606846999"); + EXPECT_EQ((*decoded.fields)["expire"], "60"); + + MasterPackets::PlayerAdded added; + added.zoneID = 1200; + added.instanceID = 3; + decoded = Written(added, false); + ASSERT_TRUE(decoded.fields); + EXPECT_EQ((*decoded.fields)["zoneID"], 1200); + EXPECT_EQ((*decoded.fields)["instanceID"], 3); + + ClientPackets::LoadStaticZone zone; + zone.mapID = 1100; + zone.mapChecksum = 0x49525511; + zone.playerPosition = NiPoint3(1.0f, 2.0f, 3.0f); + decoded = Written(zone, false); + ASSERT_TRUE(decoded.fields); + EXPECT_EQ((*decoded.fields)["mapID"], 1100); + EXPECT_EQ((*decoded.fields)["mapChecksum"], 0x49525511); + EXPECT_EQ((*decoded.fields)["playerPosition"], json::array({ 1.0f, 2.0f, 3.0f })); + + WorldPackets::LevelLoadComplete loaded; + loaded.mapID = 1100; + decoded = Written(loaded, true); + ASSERT_TRUE(decoded.fields); + EXPECT_EQ((*decoded.fields)["mapID"], 1100); +} + +// Game messages the other way round from the ones the server reads: what it sends to clients +TEST(PacketDecoderCoverageTest, SentGameMessagesReadBack) { + GameMessages::DisplayTooltip tooltip; + tooltip.show = true; + tooltip.time = 5000; + tooltip.id = u"tip"; + tooltip.text = u"Press E"; + tooltip.localizeParams.Insert(u"count", 3); + RakNet::BitStream payload; + tooltip.Serialize(payload); + const auto fields = GameMessageDecoder::Decode(MessageType::Game::DISPLAY_TOOLTIP, false, payload); + ASSERT_TRUE(fields); + EXPECT_EQ((*fields)["show"], true); + EXPECT_EQ((*fields)["time"], 5000); + EXPECT_EQ((*fields)["text"], "Press E"); + EXPECT_EQ((*fields)["localizeParams"], json::array({ "count=1:3" })); + + GameMessages::ServerTradeUpdate update; + GameMessages::TradeItemEntry item; + item.key = 5; + item.itemID = 5; + item.templateID = 1727; + item.count = 2u; + update.inventoryMap.push_back(item); + RakNet::BitStream tradePayload; + update.Serialize(tradePayload); + const auto trade = GameMessageDecoder::Decode(MessageType::Game::SERVER_TRADE_UPDATE, false, tradePayload); + ASSERT_TRUE(trade); + ASSERT_EQ((*trade)["inventoryMap"].size(), 1u); + EXPECT_EQ((*trade)["inventoryMap"][0]["templateID"], 1727); + EXPECT_EQ((*trade)["inventoryMap"][0]["count"], 2); +} diff --git a/tools/gen_game_message_fields.py b/tools/gen_game_message_fields.py index cf99bd12a..277d038ea 100644 --- a/tools/gen_game_message_fields.py +++ b/tools/gen_game_message_fields.py @@ -134,6 +134,13 @@ def game_messages(): if path.name == "GameMessages.h": continue structs += [s for s in parse(path) if s["base"] != "GameMsg"] + # Constructors defined in the .cpp + for s in structs: + source = directory / s["file"].replace(".h", ".cpp") + if s["base"] == "NetGameMsg" and not s["id"] and source.exists(): + own = s["short"].split("::")[-1] + if (mid := re.search(rf"\b{own}::{own}\s*\(\s*\)\s*:\s*NetGameMsg\(\s*MessageType::Game::(\w+)", source.read_text())): + s["enum"], s["id"] = "Game", mid.group(1) wire = [s for s in structs if s["enum"] == "Game" and s["id"]] lines = member_lines(reachable(structs, wire)) lines.append("// Wire messages: (ID, direction, reader). A struct with Handle is what the server reads from a client.")