test(capture): decoder coverage, replica read-back and generator check; docs

Every LU MessageType has a struct or is listed as never used by this server; every game message struct reads;
samples of each family and of sent game messages read back; replica constructions, updates and destructions written
by the server's own serializers read back. Generator finds constructors defined in the .cpp.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

README.md

@@ -48,7 +48,8 @@ A web dashboard for running and moderating a server, started and supervised by m
   viewer's language (picker in the user menu, or the browser's language), never written into the pages.
 * **Packet capture and replay:** record whole packets for an account, a character or everything; play them back with
   movement in World 3D, which follows a player across worlds (live too) with the world changes marked on the
-  timeline; replay bundles against a sandbox stack with a headless client. See
+  timeline; every packet decoded (LU packets and game messages by their own structs, replica constructions and
+  updates component by component); replay bundles against a sandbox stack with a headless client. See
   [docs/CaptureReplay.md](docs/CaptureReplay.md).
 * **Settings:** every setting the servers read, grouped by purpose with typed inputs, conditions, fuzzy search,
   history, and hot reload; values can be set on the page or kept in the `.ini` files.

dGame/dGameMessages/GameMessageFields.inc

@@ -129,6 +129,7 @@ json ToJson(const GameMessages::RequestLinkedMission& m);
 json ToJson(const GameMessages::SetFlag& m);
 json ToJson(const GameMessages::NotifyClientFlagChange& m);
 json ToJson(const GameMessages::HasBeenCollected& m);
+json ToJson(const GameMessages::SetMissionTypeState& m);
 json ToJson(const GameMessages::NotifyLevelRewards& m);
 json ToJson(const GameMessages::Teleport& m);
 json ToJson(const GameMessages::StartPathing& m);
@@ -230,6 +231,7 @@ json ToJson(const GameMessages::EnterProperty1& m);
 json ToJson(const GameMessages::UpdatePropertyPerformanceCost& m);
 json ToJson(const GameMessages::ReportOffensiveModel& m);
 json ToJson(const GameMessages::ReportOffensiveProperty& m);
+json ToJson(const GameMessages::ControlBehaviors& m);
 json ToJson(const GameMessages::GetHotPropertyData& m);
 json ToJson(const GameMessages::NewsSendHotPropertiesInfoToClient& m);
 json ToJson(const GameMessages::NewsSendHotPropertiesInfoToClient::HotPropertyInfo& m);
@@ -1310,6 +1312,13 @@ json ToJson(const GameMessages::HasBeenCollected& m) {
 	j["playerID"] = ToJson(m.playerID);
 	return j;
 }
+json ToJson(const GameMessages::SetMissionTypeState& m) {
+	json j = json::object();
+	j["state"] = ToJson(m.state);
+	j["subtype"] = ToJson(m.subtype);
+	j["type"] = ToJson(m.type);
+	return j;
+}
 json ToJson(const GameMessages::NotifyLevelRewards& m) {
 	json j = json::object();
 	j["level"] = ToJson(m.level);
@@ -2007,6 +2016,12 @@ json ToJson(const GameMessages::ReportOffensiveProperty& m) {
 	j["propertyPlaqueObjectID"] = ToJson(m.propertyPlaqueObjectID);
 	return j;
 }
+json ToJson(const GameMessages::ControlBehaviors& m) {
+	json j = json::object();
+	j["args"] = ToJson(m.args);
+	j["command"] = ToJson(m.command);
+	return j;
+}
 json ToJson(const GameMessages::GetHotPropertyData& m) {
 	json j = json::object();
 	return j;
@@ -2627,6 +2642,7 @@ const std::vector<Entry>& Entries() {
 		{ MessageType::Game::SET_FLAG, eDirection::TO_SERVER, "SetFlag", &ReadWith<GameMessages::SetFlag> },
 		{ MessageType::Game::NOTIFY_CLIENT_FLAG_CHANGE, eDirection::TO_CLIENT, "NotifyClientFlagChange", &ReadWith<GameMessages::NotifyClientFlagChange> },
 		{ MessageType::Game::HAS_BEEN_COLLECTED, eDirection::TO_SERVER, "HasBeenCollected", &ReadWith<GameMessages::HasBeenCollected> },
+		{ MessageType::Game::SET_MISSION_TYPE_STATE, eDirection::TO_SERVER, "SetMissionTypeState", &ReadWith<GameMessages::SetMissionTypeState> },
 		{ MessageType::Game::NOTIFY_LEVEL_REWARDS, eDirection::TO_CLIENT, "NotifyLevelRewards", &ReadWith<GameMessages::NotifyLevelRewards> },
 		{ MessageType::Game::TELEPORT, eDirection::TO_CLIENT, "Teleport", &ReadWith<GameMessages::Teleport> },
 		{ MessageType::Game::START_PATHING, eDirection::TO_CLIENT, "StartPathing", &ReadWith<GameMessages::StartPathing> },
@@ -2727,6 +2743,7 @@ const std::vector<Entry>& Entries() {
 		{ MessageType::Game::UPDATE_PROPERTY_PERFORMANCE_COST, eDirection::TO_SERVER, "UpdatePropertyPerformanceCost", &ReadWith<GameMessages::UpdatePropertyPerformanceCost> },
 		{ MessageType::Game::REPORT_OFFENSIVE_MODEL, eDirection::TO_SERVER, "ReportOffensiveModel", &ReadWith<GameMessages::ReportOffensiveModel> },
 		{ MessageType::Game::REPORT_OFFENSIVE_PROPERTY, eDirection::TO_SERVER, "ReportOffensiveProperty", &ReadWith<GameMessages::ReportOffensiveProperty> },
+		{ MessageType::Game::CONTROL_BEHAVIORS, eDirection::TO_SERVER, "ControlBehaviors", &ReadWith<GameMessages::ControlBehaviors> },
 		{ MessageType::Game::GET_HOT_PROPERTY_DATA, eDirection::TO_SERVER, "GetHotPropertyData", &ReadWith<GameMessages::GetHotPropertyData> },
 		{ MessageType::Game::SEND_HOT_PROPERTY_DATA, eDirection::TO_CLIENT, "NewsSendHotPropertiesInfoToClient", &ReadWith<GameMessages::NewsSendHotPropertiesInfoToClient> },
 		{ MessageType::Game::PLAY_BEHAVIOR_SOUND, eDirection::TO_CLIENT, "PlayBehaviorSound", &ReadWith<GameMessages::PlayBehaviorSound> },

docs/CaptureReplay.md

@@ -90,6 +90,53 @@ A busy world sends a few thousand packets a second, so capturing everything ther
 Appending to a file is about 80 times cheaper than a database row per packet, so packets go to files and the
 database keeps only the session row (the game message inspector keeps its rows as before).
 
+## Decoding
+
+The viewer shows every packet with named fields and values, and its bytes below. Nothing is parsed twice: packets are
+read with the server's own structs and their `Deserialize`.
+
+| Packets | How |
+|---|---|
+| LU packets (auth, chat, client, master, world, common) | `dNet/PacketDecoder.cpp`: the struct for its (service, ID). An ID with a struct per direction is read with the first that reads the whole packet, the direction's own first. |
+| Game messages, both directions | `dGame/dGameMessages/GameMessageDecoder.cpp`: the `NetGameMsg` struct for its ID (the one the server reads a client's message with, else the one it sends). |
+| Replica constructions, serializations, destructions | `dGame/dUtilities/ReplicaDecoder.cpp`, below. |
+
+The members of each struct are listed by `tools/gen_game_message_fields.py`, which writes `dNet/PacketFields.inc` and
+`dGame/dGameMessages/GameMessageFields.inc` from the struct definitions; run it after changing a packet struct (the
+`PacketFieldsUpToDate` test fails until then). Every member type needs a `ToJson` overload (`dNet/PacketJson.h`), or
+the build fails. Members holding secrets (passwords, session and user keys) are never listed. Whole bytes a struct
+leaves unread are shown as `(unread bits)`.
+
+What isn't decoded shows its name (from the `MessageType` enums) and its bytes: LU packet IDs this server never sends
+or handles (listed in `PacketDecoderCoverageTests`, which fails for any other ID without a struct), game message IDs
+the server has no struct for (live-only messages), and the client's `MAIL` packet (its sub-messages are read by the
+game's mail code). Every game message the server sends or reads has a struct.
+
+### Replica packets
+
+Replica packets depend on what came before, so a capture is read once in timeline order, on a dashboard worker
+thread, and the result is kept with the loaded capture:
+
+- **Destruction:** the network ID, and the object it was when the capture had its construction.
+- **Construction:** the object header as `Entity::WriteBaseReplicaData` writes it (network ID, object ID, LOT, name,
+  age, config, trigger, spawner, spawner node, scale, world state, GM level, parent and children), then each
+  component's data in the order the client reads them (`Entity::WriteComponents`). Which components an object has
+  comes from the ComponentsRegistry rows of its LOT, read from the CDClient at startup, with the ones the server adds
+  itself (a destroyable for collectibles, quick builds and models, the character's parts, a buff with a destroyable).
+- **Serialization:** the object's LOT and components from its construction earlier on the same world instance, then
+  each component's update. An object constructed before the capture started shows "object not constructed in this
+  capture" and the bytes.
+
+Each component reader mirrors the component's `Serialize(bIsInitialUpdate)` (`ReplicaDecoderTests` writes real
+components with the server's serializers and reads them back). Some objects have components their LOT doesn't list
+(a smashable, moving platform or script set up by the zone file): when the registry's list doesn't read the packet
+exactly (to the last whole byte, padding zero), those variants are tried. When none fits, what read is shown with
+`(layout did not match)` and the rest as bits; parts the server never writes (only live did, such as local space
+info) stop the reader with `(... present, not read)`. Nothing is guessed.
+
+The capture tool reads game messages the same way (it links the game), so replays compare their fields, not only their
+size.
+
 ## The bundle format
 
 One format for the dashboard's capture files, exported bundles and converted live captures (`dNet/CaptureBundle.h`):
@@ -147,7 +194,7 @@ CaptureTool replay <bundle>... --client <game client folder> [--cdserver <CDServ
                                [--speed 4] [--port 41000] [--sandbox-root <dir>] [--keep | --keep-on-failure] [--report <file.json>]
 CaptureTool import-live <folder> <out-dir>      convert live captures
 CaptureTool anonymise <in> <out>                make a fixture
-CaptureTool info|decode <bundle>                look inside
+CaptureTool info|decode <bundle> [--cdserver <CDServer.sqlite>]   look inside (replica packets too with --cdserver)
 ```
 
 ### The sandbox
@@ -224,4 +271,7 @@ decoder registry knows and checks it writes back to the same bytes; without fixt
 - Capture a character that changes worlds (a rocket or a portal): the capture page's slider has a mark at each change
   named after the new zone; in World 3D, following the character switches the scene at the mark and the camera stays
   on them, forward and when seeking back.
+- In a capture of a zone load: every packet shows fields; constructions list their components (no
+  `(layout did not match)` on players, enemies, smashables, NPCs); a later `ID_REPLICA_MANAGER_SERIALIZE` of an enemy
+  hit in the capture shows its new health; game messages both ways (a skill, an emote, a vendor purchase) show fields.
 - Export a bundle, replay it with `CaptureTool replay`, and open a kept sandbox's logs.

tests/dGameTests/CMakeLists.txt

@@ -66,3 +66,10 @@ target_link_libraries(dGameTests ${COMMON_LIBRARIES} sqlite3 GTest::gtest_main
 
 # Discover the tests
 gtest_discover_tests(dGameTests)
+
+# The member lists the capture viewer shows packets with are generated from the packet structs: fail when a struct
+# changed without running the generator
+find_package(Python3 COMPONENTS Interpreter)
+if(Python3_Interpreter_FOUND)
+	add_test(NAME PacketFieldsUpToDate COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/tools/gen_game_message_fields.py --check)
+endif()

tests/dGameTests/dComponentsTests/CMakeLists.txt

@@ -17,6 +17,7 @@ set(DCOMPONENTS_TESTS
 	"QuickBuildCompleteTests.cpp"
 	"RocketLaunchTests.cpp"
 	"ReplicaConstructionTests.cpp"
+	"ReplicaDecoderTests.cpp"
 	"ItemRemovalTests.cpp"
 )
 

tests/dGameTests/dComponentsTests/ReplicaDecoderTests.cpp

@@ -0,0 +1,192 @@
+// The capture viewer's replica reader (ReplicaDecoder) against the server's own writers: objects are built with real
+// components, written with Entity::WriteBaseReplicaData and WriteComponents as a construction or serialization packet,
+// and read back. A component whose Serialize changes without its reader fails here.
+#include "GameDependencies.h"
+#include <gtest/gtest.h>
+
+#include "BitStream.h"
+#include "BuffComponent.h"
+#include "Character.h"
+#include "CharacterComponent.h"
+#include "ControllablePhysicsComponent.h"
+#include "DestroyableComponent.h"
+#include "Entity.h"
+#include "InventoryComponent.h"
+#include "ItemComponent.h"
+#include "LevelProgressionComponent.h"
+#include "MessageIdentifiers.h"
+#include "ModelComponent.h"
+#include "PlayerForcedMovementComponent.h"
+#include "PossessorComponent.h"
+#include "ReplicaDecoder.h"
+#include "SimplePhysicsComponent.h"
+#include "SkillComponent.h"
+#include "User.h"
+#include "eReplicaComponentType.h"
+#include "eReplicaPacketType.h"
+
+namespace {
+	using json = nlohmann::json;
+	using enum eReplicaComponentType;
+
+	std::string Bytes(const RakNet::BitStream& stream) { return std::string(reinterpret_cast<const char*>(stream.GetData()), stream.GetNumberOfBytesUsed()); }
+
+	std::string Construction(Entity& entity, uint16_t network) {
+		RakNet::BitStream stream;
+		stream.Write<uint8_t>(ID_REPLICA_MANAGER_CONSTRUCTION);
+		stream.Write1();
+		stream.Write(network);
+		entity.WriteBaseReplicaData(stream, eReplicaPacketType::CONSTRUCTION);
+		entity.WriteComponents(stream, eReplicaPacketType::CONSTRUCTION);
+		return Bytes(stream);
+	}
+
+	std::string Serialization(Entity& entity, uint16_t network) {
+		RakNet::BitStream stream;
+		stream.Write<uint8_t>(ID_REPLICA_MANAGER_SERIALIZE);
+		stream.Write(network);
+		entity.WriteBaseReplicaData(stream, eReplicaPacketType::SERIALIZATION);
+		entity.WriteComponents(stream, eReplicaPacketType::SERIALIZATION);
+		return Bytes(stream);
+	}
+
+	const json* FieldsOf(const json& fields, const std::string& name) {
+		for (const auto& component : fields["components"]) {
+			if (component["component"] == name) return &component["fields"];
+		}
+		return nullptr;
+	}
+
+	std::string Names(const json& fields) {
+		std::string names;
+		for (const auto& component : fields["components"]) names += component["component"].get<std::string>() + " ";
+		return names;
+	}
+}
+
+class ReplicaDecoderTest : public GameDependenciesTest {
+protected:
+	void SetUp() override { SetUpDependencies(); }
+	void TearDown() override { TearDownDependencies(); }
+};
+
+TEST_F(ReplicaDecoderTest, EnemyConstructionAndSerializationReadBack) {
+	info.lot = 4712;
+	Entity enemy(288300744895900100, info);
+	auto* const physics = enemy.AddComponent<ControllablePhysicsComponent>(-1);
+	auto* const destroyable = enemy.AddComponent<DestroyableComponent>(-1);
+	destroyable->SetMaxHealth(8.0f);
+	destroyable->SetHealth(8);
+	physics->SetPosition(NiPoint3(10.0f, 20.0f, 30.0f));
+
+	// A smashable (is_smashable in the zone file): its destroyable isn't in the registry and is written last, which the
+	// reader finds by trying that layout when the registry's doesn't read exactly
+	const ReplicaDecoder::ComponentTable table{ { 4712, { CONTROLLABLE_PHYSICS } } };
+	ReplicaDecoder::Session session(table);
+	const auto constructed = session.Decode(Construction(enemy, 7), 1);
+	ASSERT_TRUE(constructed);
+	EXPECT_FALSE(constructed->contains("(layout did not match)")) << constructed->dump();
+	EXPECT_EQ((*constructed)["networkID"], 7);
+	EXPECT_EQ((*constructed)["objectID"], "288300744895900100");
+	EXPECT_EQ((*constructed)["lot"], 4712);
+	EXPECT_EQ(Names(*constructed), "CONTROLLABLE_PHYSICS DESTROYABLE ");
+	const auto* health = FieldsOf(*constructed, "DESTROYABLE");
+	ASSERT_TRUE(health);
+	EXPECT_EQ((*health)["health"], 8);
+	EXPECT_EQ((*FieldsOf(*constructed, "CONTROLLABLE_PHYSICS"))["position"], json::array({ 10.0f, 20.0f, 30.0f }));
+
+	// An update: only what changed, read with the components the construction had
+	destroyable->SetHealth(3);
+	const auto updated = session.Decode(Serialization(enemy, 7), 1);
+	ASSERT_TRUE(updated);
+	EXPECT_FALSE(updated->contains("(layout did not match)")) << updated->dump();
+	EXPECT_EQ((*updated)["lot"], 4712);
+	EXPECT_EQ((*FieldsOf(*updated, "DESTROYABLE"))["health"], 3);
+
+	// Another connection never saw the construction
+	const auto unknown = session.Decode(Serialization(enemy, 7), 2);
+	ASSERT_TRUE(unknown);
+	EXPECT_TRUE(unknown->contains("(object not constructed in this capture)"));
+
+	// Destruction: the network ID, and the object it was
+	RakNet::BitStream destruction;
+	destruction.Write<uint8_t>(ID_REPLICA_MANAGER_DESTRUCTION);
+	destruction.Write<uint16_t>(7);
+	const auto destroyed = session.Decode(Bytes(destruction), 1);
+	ASSERT_TRUE(destroyed);
+	EXPECT_EQ((*destroyed)["networkID"], 7);
+	EXPECT_EQ((*destroyed)["lot"], 4712);
+	EXPECT_TRUE(session.Decode(Serialization(enemy, 7), 1)->contains("(object not constructed in this capture)"));
+}
+
+TEST_F(ReplicaDecoderTest, PlayerConstructionReadsTheCharacterParts) {
+	User user(UNASSIGNED_SYSTEM_ADDRESS, "tester", "key");
+	Character character(1, &user);
+	info.lot = 1;
+	Entity player(1152921506064087003, info);
+	player.SetCharacter(&character);
+	character.SetEntity(&player);
+	player.AddComponent<ControllablePhysicsComponent>(-1);
+	player.AddComponent<PossessorComponent>(-1);
+	player.AddComponent<LevelProgressionComponent>(-1);
+	player.AddComponent<PlayerForcedMovementComponent>(-1);
+	player.AddComponent<CharacterComponent>(-1, &character, UNASSIGNED_SYSTEM_ADDRESS)->InitializeStatisticsFromString("");
+
+	const ReplicaDecoder::ComponentTable table{ { 1, { CONTROLLABLE_PHYSICS, CHARACTER } } };
+	ReplicaDecoder::Session session(table);
+	const auto constructed = session.Decode(Construction(player, 1), 1);
+	ASSERT_TRUE(constructed);
+	EXPECT_FALSE(constructed->contains("(layout did not match)")) << constructed->dump();
+	EXPECT_EQ(Names(*constructed), "CONTROLLABLE_PHYSICS POSSESSOR LEVEL_PROGRESSION PLAYER_FORCED_MOVEMENT CHARACTER ");
+	EXPECT_EQ((*FieldsOf(*constructed, "CHARACTER"))["statistics"].size(), 27u);
+}
+
+// A model's destroyable is not in its registry rows: it is written after the other components
+TEST_F(ReplicaDecoderTest, ModelWithTheDestroyableTheRegistryDoesNotList) {
+	info.lot = 6000;
+	Entity model(288300744895900200, info);
+	model.AddComponent<SimplePhysicsComponent>(-1);
+	model.AddComponent<ModelComponent>(-1)->LoadBehaviors();
+	model.AddComponent<ItemComponent>(-1);
+	auto* const destroyable = model.AddComponent<DestroyableComponent>(-1);
+	destroyable->SetIsSmashable(true);
+
+	const ReplicaDecoder::ComponentTable table{ { 6000, { SIMPLE_PHYSICS, MODEL, ITEM } } };
+	ReplicaDecoder::Session session(table);
+	const auto constructed = session.Decode(Construction(model, 3), 1);
+	ASSERT_TRUE(constructed);
+	EXPECT_FALSE(constructed->contains("(layout did not match)")) << constructed->dump();
+	EXPECT_EQ(Names(*constructed), "SIMPLE_PHYSICS ITEM MODEL DESTROYABLE ");
+	EXPECT_EQ((*FieldsOf(*constructed, "DESTROYABLE"))["smashable"], true);
+}
+
+// When no layout reads the packet exactly, what read is shown with the rest as bits, never a guess
+TEST_F(ReplicaDecoderTest, UnknownLayoutShowsTheRest) {
+	info.lot = 4712;
+	Entity enemy(288300744895900100, info);
+	enemy.AddComponent<ControllablePhysicsComponent>(-1);
+	auto packet = Construction(enemy, 9);
+	packet += std::string("\x12\x34\x56", 3);
+
+	const ReplicaDecoder::ComponentTable table{ { 4712, { CONTROLLABLE_PHYSICS } } };
+	ReplicaDecoder::Session session(table);
+	const auto constructed = session.Decode(packet, 1);
+	ASSERT_TRUE(constructed);
+	EXPECT_TRUE(constructed->contains("(layout did not match)"));
+	EXPECT_FALSE((*constructed)["(rest)"].get<std::string>().empty());
+}
+
+// A live construction (network ID 11, LOT 13006, no components, parent/child info present but empty): the object
+// header reads the same as the server writes it
+TEST_F(ReplicaDecoderTest, LiveConstructionHeader) {
+	const unsigned char live[] = { 0x24, 0x85, 0x80, 0x7f, 0x7f, 0xff, 0xff, 0xff, 0x9f, 0x80, 0x00, 0x67, 0x19, 0x00, 0x00, 0x00, 0x5f, 0x99, 0x9d, 0x00, 0x00, 0x80 };
+	const ReplicaDecoder::ComponentTable table{ { 13006, {} } };
+	ReplicaDecoder::Session session(table);
+	const auto constructed = session.Decode(std::string(reinterpret_cast<const char*>(live), sizeof(live)), 1);
+	ASSERT_TRUE(constructed);
+	EXPECT_FALSE(constructed->contains("(layout did not match)")) << constructed->dump();
+	EXPECT_EQ((*constructed)["networkID"], 11);
+	EXPECT_EQ((*constructed)["objectID"], "70368744177662");
+	EXPECT_EQ((*constructed)["lot"], 13006);
+	EXPECT_EQ((*constructed)["timeSinceCreatedMs"], 3814335);
+}

tests/dGameTests/dGameMessagesTests/GameMessageDecoderTests.cpp

@@ -39,28 +39,31 @@ TEST(GameMessageDecoderTest, ReadsWhatTheServerWrites) {
 	EXPECT_EQ((*fields)["skillID"], 42);
 	EXPECT_EQ((*fields)["optionalOriginatorID"], "1234");
 	EXPECT_EQ((*fields)["optionalTargetID"], "77");
-	EXPECT_EQ((*fields)["sBitStream"], "0102");
+	EXPECT_EQ((*fields)["sBitStream"], nlohmann::json({ {"hex", "0102"} }));
 }
 
-TEST(GameMessageDecoderTest, DirectionMatters) {
+// A message only sent one way is read with its struct in either direction (the layout is the same)
+TEST(GameMessageDecoderTest, SentMessagesDecode) {
 	GameMessages::EchoSyncSkill sent;
 	sent.bDone = true;
 	sent.uiSkillHandle = 5;
 	RakNet::BitStream stream;
 	sent.Serialize(stream);
 
-	EXPECT_FALSE(GameMessageDecoder::CanDecode(MessageType::Game::ECHO_SYNC_SKILL, true));
+	EXPECT_TRUE(GameMessageDecoder::CanDecode(MessageType::Game::ECHO_SYNC_SKILL, false));
 	const auto fields = GameMessageDecoder::Decode(MessageType::Game::ECHO_SYNC_SKILL, false, stream);
 	ASSERT_TRUE(fields);
 	EXPECT_EQ((*fields)["bDone"], true);
 	EXPECT_EQ((*fields)["uiSkillHandle"], 5);
 }
 
-TEST(GameMessageDecoderTest, UntypedMessagesAreNotDecoded) {
+// A message the server has no struct for (live only) isn't decoded
+TEST(GameMessageDecoderTest, MessagesWithoutAStructAreNotDecoded) {
 	RakNet::BitStream payload;
 	payload.Write<uint32_t>(1);
-	EXPECT_FALSE(GameMessageDecoder::CanDecode(MessageType::Game::PLAY_EMOTE, true));
+	EXPECT_FALSE(GameMessageDecoder::HasStruct(MessageType::Game::SET_PVP_STATUS));
-	EXPECT_FALSE(GameMessageDecoder::Decode(MessageType::Game::PLAY_EMOTE, true, payload));
+	EXPECT_FALSE(GameMessageDecoder::CanDecode(MessageType::Game::SET_PVP_STATUS, true));
+	EXPECT_FALSE(GameMessageDecoder::Decode(MessageType::Game::SET_PVP_STATUS, true, payload));
 }
 
 TEST(GameMessageDecoderTest, ShortPayloadsFail) {

tests/dGameTests/dNetTests/CMakeLists.txt

@@ -5,6 +5,7 @@ SET(DNET_TESTS
 	"MasterPacketsTests.cpp"
 	"MatchmakingPacketsTests.cpp"
 	"PacketCaptureTests.cpp"
+	"PacketDecoderCoverageTests.cpp"
 	"ServerTrafficTests.cpp"
 	"WorldPacketsTests.cpp")
 

tests/dGameTests/dNetTests/PacketDecoderCoverageTests.cpp

@@ -0,0 +1,195 @@
+// The capture viewer decodes every packet the server sends or reads: every LU message ID has a struct (or is on the
+// list below of IDs this server never sends or handles), every game message struct reads, and samples of each family
+// come back with the values they were written with.
+#include <gtest/gtest.h>
+
+#include <set>
+#include <string>
+
+#include "AuthPackets.h"
+#include "ChatPackets.h"
+#include "ClientPackets.h"
+#include "CommonPackets.h"
+#include "EffectsMessages.h"
+#include "GameMessageDecoder.h"
+#include "GameMessageHandler.h"
+#include "MasterPackets.h"
+#include "MessageType/Auth.h"
+#include "MessageType/Chat.h"
+#include "MessageType/Client.h"
+#include "MessageType/Master.h"
+#include "MessageType/Server.h"
+#include "MessageType/World.h"
+#include "PacketDecoder.h"
+#include "ServiceType.h"
+#include "SkillMessages.h"
+#include "TradeMessages.h"
+#include "WorldPackets.h"
+#include "magic_enum.hpp"
+
+namespace {
+	using json = nlohmann::json;
+
+	std::string Bytes(const RakNet::BitStream& stream) { return std::string(reinterpret_cast<const char*>(stream.GetData()), stream.GetNumberOfBytesUsed()); }
+
+	template<typename T>
+	PacketDecoder::Decoded Written(const T& packet, bool fromClient) {
+		RakNet::BitStream stream;
+		packet.WritePacket(stream);
+		return PacketDecoder::Decode(Bytes(stream), fromClient);
+	}
+
+	// IDs this server never sends and never handles: no struct, so the viewer shows their name and bytes
+	const std::set<std::string> NOT_USED{
+		"AUTH LOGOUT_REQUEST", "AUTH CREATE_NEW_ACCOUNT_REQUEST", "AUTH LEGOINTERFACE_AUTH_RESPONSE", "AUTH SESSIONKEY_RECEIVED_CONFIRM",
+		"CHAT USER_CHANNEL_CHAT_MESSAGE", "CHAT WORLD_DISCONNECT_REQUEST", "CHAT WORLD_PROXIMITY_RESPONSE", "CHAT WORLD_PARCEL_RESPONSE",
+		"CHAT TEAM_MISSED_INVITE_CHECK", "CHAT BLUEPRINT_MODERATED", "CHAT BLUEPRINT_MODEL_READY", "CHAT PROPERTY_READY_FOR_APPROVAL",
+		"CHAT PROPERTY_MODERATION_CHANGED", "CHAT PROPERTY_BUILDMODE_CHANGED", "CHAT PROPERTY_BUILDMODE_CHANGED_REPORT",
+		"CHAT WORLD_INSTANCE_LOCATION_REQUEST", "CHAT REPUTATION_UPDATE", "CHAT SEND_CANNED_TEXT", "CHAT CHARACTER_NAME_CHANGE_REQUEST",
+		"CHAT CSR_REQUEST", "CHAT CSR_REPLY", "CHAT GM_KICK", "CHAT ACTIVITY_UPDATE", "CHAT GET_ZONE_POPULATIONS", "CHAT MATCH_REQUEST",
+		"CHAT UGCMANIFEST_REPORT_MISSING_FILE", "CHAT UGCMANIFEST_REPORT_DONE_FILE", "CHAT UGCMANIFEST_REPORT_DONE_BLUEPRINT",
+		"CHAT UGCC_REQUEST", "CHAT WORLD_PLAYERS_PET_MODERATED_ACKNOWLEDGE", "CHAT GM_CLOSE_PRIVATE_CHAT_WINDOW", "CHAT PLAYER_READY",
+		"CHAT GET_DONATION_TOTAL", "CHAT UPDATE_DONATION", "CHAT PRG_CSR_COMMAND", "CHAT HEARTBEAT_REQUEST_FROM_WORLD",
+		"CHAT UPDATE_FREE_TRIAL_STATUS",
+		"CLIENT LOGOUT_RESPONSE", "CLIENT CREATE_OBJECT", "CLIENT CREATE_CHARACTER_EXTENDED", "CLIENT CHAT_CONNECT_RESPONSE",
+		"CLIENT AUTH_ACCOUNT_CREATE_RESPONSE", "CLIENT CONNECT_CHAT", "CLIENT IMPENDING_RELOAD_NOTIFY", "CLIENT SLASH_PUSH_MAP_RESPONSE",
+		"CLIENT SLASH_PULL_MAP_RESPONSE", "CLIENT SLASH_LOCK_MAP_RESPONSE", "CLIENT BLUEPRINT_LUP_SAVE_RESPONSE",
+		"CLIENT BLUEPRINT_GET_ALL_DATA_RESPONSE", "CLIENT MODEL_INSTANTIATE_RESPONSE", "CLIENT GUILD_GET_STATUS_RESPONSE",
+		"CLIENT GUILD_RANK_CHANGE", "CLIENT GUILD_STATUS", "CLIENT DB_PROXY_RESULT", "CLIENT UPDATE_CHARACTER_NAME",
+		"CLIENT SET_NETWORK_SIMULATOR", "CLIENT INVALID_CHAT_MESSAGE", "CLIENT IN_LOGIN_QUEUE", "CLIENT GM_CLOSE_TARGET_CHAT_WINDOW",
+		"CLIENT GENERAL_TEXT_FOR_LOCALIZATION", "CLIENT UPDATE_FREE_TRIAL_STATUS",
+		"MASTER SHUTDOWN_IMMEDIATE", "AUTH RUNTIME_CONFIG", "CLIENT UGC_DOWNLOAD_FAILED",
+		"WORLD HAPPY_FLOWER_MODE_NOTIFY", "WORLD SLASH_RELOAD_MAP", "WORLD SLASH_PUSH_MAP_REQUEST", "WORLD SLASH_PUSH_MAP",
+		"WORLD SLASH_PULL_MAP", "WORLD LOCK_MAP_REQUEST", "WORLD HTTP_MONITOR_INFO_REQUEST", "WORLD SLASH_DEBUG_SCRIPTS",
+		"WORLD MODELS_CLEAR", "WORLD EXHIBIT_INSERT_MODEL", "WORLD WORD_CHECK", "WORLD GET_PLAYERS_IN_ZONE",
+		"WORLD BLUEPRINT_GET_ALL_DATA_REQUEST", "WORLD CANCEL_MAP_QUEUE", "WORLD FAKE_PRG_CSR_MESSAGE",
+		"WORLD REQUEST_FREE_TRIAL_REFRESH", "WORLD GM_SET_FREE_TRIAL_STATUS",
+	};
+
+	// Decoded some other way: game messages (GameMessageDecoder) and mail (a sub-header of its own, read by the game)
+	const std::set<std::string> ELSEWHERE{ "CLIENT GAME_MSG", "CLIENT MAIL" };
+
+	template<typename E>
+	void ExpectEveryIdDecodes(ServiceType service, std::set<std::string>& missing) {
+		for (const auto id : magic_enum::enum_values<E>()) {
+			const auto name = std::string(magic_enum::enum_name(service)) + " " + std::string(magic_enum::enum_name(id));
+			if (PacketDecoder::HasFields(service, static_cast<uint32_t>(id))) {
+				EXPECT_FALSE(NOT_USED.contains(name)) << name << " has a struct now: take it off the list";
+				continue;
+			}
+			if (!NOT_USED.contains(name) && !ELSEWHERE.contains(name)) missing.insert(name);
+		}
+	}
+}
+
+TEST(PacketDecoderCoverageTest, EveryMessageTypeHasAStruct) {
+	std::set<std::string> missing;
+	ExpectEveryIdDecodes<MessageType::Server>(ServiceType::COMMON, missing);
+	ExpectEveryIdDecodes<MessageType::Auth>(ServiceType::AUTH, missing);
+	ExpectEveryIdDecodes<MessageType::Chat>(ServiceType::CHAT, missing);
+	ExpectEveryIdDecodes<MessageType::Client>(ServiceType::CLIENT, missing);
+	ExpectEveryIdDecodes<MessageType::Master>(ServiceType::MASTER, missing);
+	ExpectEveryIdDecodes<MessageType::World>(ServiceType::WORLD, missing);
+	std::string list;
+	for (const auto& name : missing) list += name + "\n";
+	EXPECT_TRUE(missing.empty()) << "No struct decodes these (add one, or list them as not used):\n" << list;
+}
+
+// Every game message the server has a struct for reads, and so does every message it handles from a client
+TEST(PacketDecoderCoverageTest, EveryGameMessageStructReads) {
+	const auto decodable = GameMessageDecoder::Decodable();
+	EXPECT_GT(decodable.size(), 290u);
+	for (const auto id : magic_enum::enum_values<MessageType::Game>()) {
+		if (!GameMessageHandler::CreateReceived(id)) continue;
+		EXPECT_TRUE(GameMessageDecoder::CanDecode(id, true)) << magic_enum::enum_name(id);
+	}
+	for (const auto id : decodable) {
+		EXPECT_TRUE(GameMessageDecoder::CanDecode(id, true) && GameMessageDecoder::CanDecode(id, false)) << magic_enum::enum_name(id);
+	}
+}
+
+TEST(PacketDecoderCoverageTest, AuthLoginRequestNeverShowsThePassword) {
+	AuthPackets::LoginRequest login;
+	login.username.string = u"tester";
+	login.password.string = u"hunter2";
+	login.numberOfProcessors = 8;
+	const auto decoded = Written(login, true);
+	ASSERT_TRUE(decoded.fields);
+	EXPECT_EQ((*decoded.fields)["username"], "tester");
+	EXPECT_EQ((*decoded.fields)["numberOfProcessors"], 8);
+	EXPECT_FALSE(decoded.fields->contains("password"));
+	EXPECT_EQ(decoded.fields->dump().find("hunter2"), std::string::npos);
+}
+
+TEST(PacketDecoderCoverageTest, SamplesOfEachFamilyReadBack) {
+	CommonPackets::ServerVersionConfirm version;
+	version.netVersion = 171022;
+	auto decoded = Written(version, false);
+	ASSERT_TRUE(decoded.fields);
+	EXPECT_EQ((*decoded.fields)["netVersion"], 171022);
+
+	ChatPackets::GMMute mute;
+	mute.playerID = 1152921504606846999LL;
+	mute.expire = 60;
+	decoded = Written(mute, false);
+	ASSERT_TRUE(decoded.fields) << decoded.name;
+	EXPECT_EQ((*decoded.fields)["playerID"], "1152921504606846999");
+	EXPECT_EQ((*decoded.fields)["expire"], "60");
+
+	MasterPackets::PlayerAdded added;
+	added.zoneID = 1200;
+	added.instanceID = 3;
+	decoded = Written(added, false);
+	ASSERT_TRUE(decoded.fields);
+	EXPECT_EQ((*decoded.fields)["zoneID"], 1200);
+	EXPECT_EQ((*decoded.fields)["instanceID"], 3);
+
+	ClientPackets::LoadStaticZone zone;
+	zone.mapID = 1100;
+	zone.mapChecksum = 0x49525511;
+	zone.playerPosition = NiPoint3(1.0f, 2.0f, 3.0f);
+	decoded = Written(zone, false);
+	ASSERT_TRUE(decoded.fields);
+	EXPECT_EQ((*decoded.fields)["mapID"], 1100);
+	EXPECT_EQ((*decoded.fields)["mapChecksum"], 0x49525511);
+	EXPECT_EQ((*decoded.fields)["playerPosition"], json::array({ 1.0f, 2.0f, 3.0f }));
+
+	WorldPackets::LevelLoadComplete loaded;
+	loaded.mapID = 1100;
+	decoded = Written(loaded, true);
+	ASSERT_TRUE(decoded.fields);
+	EXPECT_EQ((*decoded.fields)["mapID"], 1100);
+}
+
+// Game messages the other way round from the ones the server reads: what it sends to clients
+TEST(PacketDecoderCoverageTest, SentGameMessagesReadBack) {
+	GameMessages::DisplayTooltip tooltip;
+	tooltip.show = true;
+	tooltip.time = 5000;
+	tooltip.id = u"tip";
+	tooltip.text = u"Press E";
+	tooltip.localizeParams.Insert(u"count", 3);
+	RakNet::BitStream payload;
+	tooltip.Serialize(payload);
+	const auto fields = GameMessageDecoder::Decode(MessageType::Game::DISPLAY_TOOLTIP, false, payload);
+	ASSERT_TRUE(fields);
+	EXPECT_EQ((*fields)["show"], true);
+	EXPECT_EQ((*fields)["time"], 5000);
+	EXPECT_EQ((*fields)["text"], "Press E");
+	EXPECT_EQ((*fields)["localizeParams"], json::array({ "count=1:3" }));
+
+	GameMessages::ServerTradeUpdate update;
+	GameMessages::TradeItemEntry item;
+	item.key = 5;
+	item.itemID = 5;
+	item.templateID = 1727;
+	item.count = 2u;
+	update.inventoryMap.push_back(item);
+	RakNet::BitStream tradePayload;
+	update.Serialize(tradePayload);
+	const auto trade = GameMessageDecoder::Decode(MessageType::Game::SERVER_TRADE_UPDATE, false, tradePayload);
+	ASSERT_TRUE(trade);
+	ASSERT_EQ((*trade)["inventoryMap"].size(), 1u);
+	EXPECT_EQ((*trade)["inventoryMap"][0]["templateID"], 1727);
+	EXPECT_EQ((*trade)["inventoryMap"][0]["count"], 2);
+}

tools/gen_game_message_fields.py

@@ -134,6 +134,13 @@ def game_messages():
 		if path.name == "GameMessages.h":
 			continue
 		structs += [s for s in parse(path) if s["base"] != "GameMsg"]
+	# Constructors defined in the .cpp
+	for s in structs:
+		source = directory / s["file"].replace(".h", ".cpp")
+		if s["base"] == "NetGameMsg" and not s["id"] and source.exists():
+			own = s["short"].split("::")[-1]
+			if (mid := re.search(rf"\b{own}::{own}\s*\(\s*\)\s*:\s*NetGameMsg\(\s*MessageType::Game::(\w+)", source.read_text())):
+				s["enum"], s["id"] = "Game", mid.group(1)
 	wire = [s for s in structs if s["enum"] == "Game" and s["id"]]
 	lines = member_lines(reachable(structs, wire))
 	lines.append("// Wire messages: (ID, direction, reader). A struct with Handle is what the server reads from a client.")