feat(capture): decode every LU packet with its own struct

The capture viewer's LU packet fields come from each struct's Deserialize and a member list generated from the
struct definitions (tools/gen_game_message_fields.py -> dNet/PacketFields.inc), replacing the hand-written list of
25. Secret members are never listed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dNet/PacketDecoder.cpp

@@ -1,6 +1,11 @@
 #include "PacketDecoder.h"
 
+#include <algorithm>
+#include <array>
 #include <cstring>
+#include <deque>
+#include <set>
+#include <unordered_map>
 #include <map>
 #include <utility>
 
@@ -21,16 +26,30 @@
 #include "WorldPackets.h"
 #include "WorldRoutePacket.h"
 #include "magic_enum.hpp"
+#include "PacketJson.h"
+#include "master/CDClientReload.h"
+#include "master/DashboardMessages.h"
+#include "master/DataChanged.h"
+#include "master/InstanceMigration.h"
+#include "master/LiveUpdate.h"
+#include "master/MessageCapture.h"
+#include "master/PlayerAction.h"
+#include "master/Profiling.h"
+#include "master/ServerTraffic.h"
+#include "master/UgcModelsMade.h"
+#include "master/WorldFiles.h"
+#include "Profiler.h"
+#include "TrafficStats.h"
+#include "ZoneFileLog.h"
 
 namespace {
 	using json = nlohmann::json;
-	using Fields = std::function<std::optional<json>(RakNet::BitStream&, bool fromClient)>;
 	using Redactor = std::function<bool(std::string&)>;
 
 	using Scrubber = std::function<bool(std::string&, bool anonymise)>;
 
-	struct Entry {
+	// Packets with secrets or text to scrub (the fields of every packet come from PacketFields.inc)
-		Fields fields;
+	struct Protection {
 		Redactor redact; // set for structs with secret fields
 		Scrubber scrub;  // set for structs with account names, character names or typed text
 	};
@@ -73,19 +92,124 @@ namespace {
 		};
 	}
 
-	std::string Id(LWOOBJID id) { return std::to_string(id); }
+	// The fields of every packet: its struct's own Deserialize, then its members by name (PacketFields.inc, written by
-	json Point(const NiPoint3& p) { return json::array({ p.x, p.y, p.z }); }
+	// tools/gen_game_message_fields.py from the structs)
-	json Rotation(const NiQuaternion& q) { return json::array({ q.x, q.y, q.z, q.w }); }
+	using PacketJson::ToJson;
 
-	template<typename T>
+	json ToJson(const TrafficStats::Histogram& histogram) {
-	Fields Make(std::function<void(const T&, json&)> fill) {
+		json buckets = json::array();
-		return [fill](RakNet::BitStream& stream, bool) -> std::optional<json> {
+		for (const auto& [bucket, count] : histogram.Sparse()) buckets.push_back(json::array({ bucket, count }));
-			T packet;
+		return json{ {"count", std::to_string(histogram.Count())}, {"sumUs", std::to_string(histogram.Sum())}, {"buckets", buckets} };
-			if (!packet.Deserialize(stream)) return std::nullopt;
+	}
-			json out = json::object();
+
-			fill(packet, out);
+	template<typename T> json ToJson(const std::optional<T>& value);
+	template<typename T> json ToJson(const std::vector<T>& values);
+	template<typename K, typename V> json ToJson(const std::map<K, V>& values);
+	template<typename A, typename B> json ToJson(const std::pair<A, B>& value);
+	template<typename T, size_t N> json ToJson(const std::array<T, N>& values);
+	template<typename T> json ToJson(const std::set<T>& values);
+	template<typename T> json ToJson(const std::deque<T>& values);
+	template<typename K, typename V> json ToJson(const std::unordered_map<K, V>& values);
+
+	struct Read {
+		std::optional<json> fields;
+		uint32_t unreadBits{};
+	};
+	using Reader = Read(*)(RakNet::BitStream&);
+
+	struct Entry {
+		ServiceType service;
+		uint32_t id;
+		const char* structName;
+		bool received; // read by the server from its client (has a Handle)
+		Reader read;
+	};
+
+	template<typename T> Read ReadWith(RakNet::BitStream& stream);
+
+#include "PacketFields.inc"
+
+	template<typename T> json ToJson(const std::optional<T>& value) { return value ? ToJson(*value) : json(nullptr); }
+	template<typename T> json ToJson(const std::vector<T>& values) {
+		if constexpr (std::is_same_v<T, uint8_t> || std::is_same_v<T, char>) {
+			return json{ {"hex", PacketJson::Hex(std::string_view(reinterpret_cast<const char*>(values.data()), values.size()))} };
+		} else {
+			json out = json::array();
+			for (const auto& value : values) out.push_back(ToJson(value));
 			return out;
-		};
+		}
+	}
+	template<typename K, typename V> json ToJson(const std::map<K, V>& values) {
+		json out = json::array();
+		for (const auto& [key, value] : values) out.push_back(json::array({ ToJson(key), ToJson(value) }));
+		return out;
+	}
+	template<typename A, typename B> json ToJson(const std::pair<A, B>& value) { return json::array({ ToJson(value.first), ToJson(value.second) }); }
+	template<typename T, size_t N> json ToJson(const std::array<T, N>& values) {
+		json out = json::array();
+		for (const auto& value : values) out.push_back(ToJson(value));
+		return out;
+	}
+
+	template<typename T> json ToJson(const std::set<T>& values) {
+		json out = json::array();
+		for (const auto& value : values) out.push_back(ToJson(value));
+		return out;
+	}
+	template<typename T> json ToJson(const std::deque<T>& values) {
+		json out = json::array();
+		for (const auto& value : values) out.push_back(ToJson(value));
+		return out;
+	}
+	template<typename K, typename V> json ToJson(const std::unordered_map<K, V>& values) {
+		json out = json::array();
+		for (const auto& [key, value] : values) out.push_back(json::array({ ToJson(key), ToJson(value) }));
+		return out;
+	}
+
+	template<typename T> Read ReadWith(RakNet::BitStream& stream) {
+		T packet;
+		if (!packet.Deserialize(stream)) return {};
+		return { ToJson(packet), stream.GetNumberOfUnreadBits() };
+	}
+
+	// (service, ID) -> its structs, the ones the server reads from a client first
+	const std::map<std::pair<ServiceType, uint32_t>, std::vector<const Entry*>>& FieldIndex() {
+		static const auto index = [] {
+			std::map<std::pair<ServiceType, uint32_t>, std::vector<const Entry*>> out;
+			for (const auto& entry : Entries()) out[{ entry.service, entry.id }].push_back(&entry);
+			return out;
+		}();
+		return index;
+	}
+
+	/**
+	 * The packet's fields from the first of its structs that reads the whole of it (a struct per direction can share
+	 * an ID: the one for this direction is tried first). A struct that reads but leaves whole bytes unread is shown
+	 * with "(unread bits)" when no struct reads it all.
+	 */
+	std::optional<json> ReadFields(ServiceType service, uint32_t id, bool fromClient, RakNet::BitStream& stream, bool& failed) {
+		const auto it = FieldIndex().find({ service, id });
+		if (it == FieldIndex().end()) return std::nullopt;
+		auto candidates = it->second;
+		std::stable_sort(candidates.begin(), candidates.end(), [fromClient](const Entry* a, const Entry* b) {
+			return (a->received == fromClient) > (b->received == fromClient);
+		});
+		const auto start = stream.GetReadOffset();
+		std::optional<json> partial;
+		for (const auto* entry : candidates) {
+			if (!entry->read) continue;
+			stream.SetReadOffset(start);
+			auto read = entry->read(stream);
+			if (!read.fields) continue;
+			if (read.unreadBits < 8) return read.fields;
+			if (!partial) {
+				partial = std::move(read.fields);
+				(*partial)["(unread bits)"] = read.unreadBits;
+			}
+		}
+		failed = !partial;
+		return partial;
 	}
 
 	// Reads T from a whole packet, lets `blank` clear its secrets, and writes it back in place
@@ -106,141 +230,23 @@ namespace {
 	using Key = std::pair<ServiceType, uint32_t>;
 	template<typename E> Key K(ServiceType service, E id) { return { service, static_cast<uint32_t>(id) }; }
 
-	const std::map<Key, Entry>& Registry() {
+	const std::map<Key, Protection>& Registry() {
 		using S = ServiceType;
-		static const std::map<Key, Entry> registry{
+		static const std::map<Key, Protection> registry{
-			// Handshake: one ID, a struct per direction
+			{ K(S::AUTH, MessageType::Auth::LOGIN_REQUEST), { Blank<AuthPackets::LoginRequest>([](auto& p) { p.username.string.clear(); p.password.string.clear(); }), Scrub<AuthPackets::LoginRequest>([](auto& p) { X(p.username); }) } },
-			{ K(S::COMMON, MessageType::Server::VERSION_CONFIRM), { [](RakNet::BitStream& stream, bool fromClient) -> std::optional<json> {
+			{ K(S::CLIENT, MessageType::Client::LOGIN_RESPONSE), { Blank<ClientPackets::LoginResponse>([](auto& p) { p.userKey.string.clear(); p.cdnKey.string.clear(); }), nullptr } },
-				if (fromClient) {
+			{ K(S::WORLD, MessageType::World::VALIDATION), { Blank<WorldPackets::Validation>([](auto& p) { p.sessionKey.string.clear(); }), Scrub<WorldPackets::Validation>([](auto& p) { X(p.username); }) } },
-					CommonPackets::ClientVersionConfirm packet;
+			{ K(S::WORLD, MessageType::World::CHARACTER_CREATE_REQUEST), { nullptr, Scrub<WorldPackets::CharacterCreateRequest>(nullptr, [](auto& p) { X(p.name); }) } },
-					if (!packet.Deserialize(stream)) return std::nullopt;
+			{ K(S::WORLD, MessageType::World::CHARACTER_RENAME_REQUEST), { nullptr, Scrub<WorldPackets::CharacterRenameRequest>(nullptr, [](auto& p) { X(p.name); }) } },
-					return json{ {"netVersion", packet.netVersion}, {"serviceType", static_cast<int>(packet.serviceType)}, {"processID", packet.processID}, {"port", packet.port} };
+			{ K(S::WORLD, MessageType::World::GENERAL_CHAT_MESSAGE), { nullptr, Scrub<WorldPackets::GeneralChatMessage>(nullptr, [](auto& p) { X(p.message); }) } },
-				}
+			{ K(S::CLIENT, MessageType::Client::CHARACTER_LIST_RESPONSE), { nullptr, Scrub<ClientPackets::CharacterListResponse>(nullptr, [](auto& p) { for (auto& c : p.characters) { X(c.name); X(c.unapprovedName); } }) } },
-				CommonPackets::ServerVersionConfirm packet;
+			{ K(S::CLIENT, MessageType::Client::CREATE_CHARACTER), { nullptr, Scrub<ClientPackets::CreateCharacter>(nullptr, [](auto& p) { X(p.name); }) } },
-				if (!packet.Deserialize(stream)) return std::nullopt;
+			{ K(S::CHAT, MessageType::Chat::GENERAL_CHAT_MESSAGE), { nullptr, Scrub<ChatPackets::GeneralChatMessage>(nullptr, [](auto& p) { X(p.senderName); X(p.message); }) } },
-				return json{ {"netVersion", packet.netVersion}, {"serviceType", static_cast<int>(packet.serviceType)} };
+			{ K(S::CHAT, MessageType::Chat::PRIVATE_CHAT_MESSAGE), { nullptr, Scrub<ChatPackets::PrivateChatMessage>(nullptr, [](auto& p) { X(p.senderName); X(p.receiverName); X(p.message); }) } },
-			} } },
+			{ K(S::MASTER, MessageType::Master::REQUEST_SESSION_KEY), { nullptr, Scrub<MasterPackets::RequestSessionKey>([](auto& p) { X(p.username); }) } },
-
+			{ K(S::MASTER, MessageType::Master::SESSION_KEY_RESPONSE), { Blank<MasterPackets::SessionKeyResponse>([](auto& p) { p.sessionKey = 0; }), Scrub<MasterPackets::SessionKeyResponse>([](auto& p) { X(p.username); }) } },
-			// Auth: the username and password are blanked when recorded
+			{ K(S::MASTER, MessageType::Master::SET_SESSION_KEY), { Blank<MasterPackets::SetSessionKey>([](auto& p) { p.sessionKey = 0; }), Scrub<MasterPackets::SetSessionKey>([](auto& p) { X(p.username); }) } },
-			{ K(S::AUTH, MessageType::Auth::LOGIN_REQUEST), {
+			{ K(S::MASTER, MessageType::Master::NEW_SESSION_ALERT), { Blank<MasterPackets::NewSessionAlert>([](auto& p) { p.sessionKey = 0; }), Scrub<MasterPackets::NewSessionAlert>([](auto& p) { X(p.username); }) } },
-				Make<AuthPackets::LoginRequest>([](const auto& p, json& j) {
-					j = { {"username", p.username.GetAsString()}, {"localeID", static_cast<int>(p.localeID)}, {"clientOS", static_cast<int>(p.clientOS)},
-						{"memoryStats", p.memoryStats.GetAsString()}, {"videoCard", p.videoCard.GetAsString()} };
-				}),
-				Blank<AuthPackets::LoginRequest>([](auto& p) { p.username.string.clear(); p.password.string.clear(); }),
-				Scrub<AuthPackets::LoginRequest>([](auto& p) { X(p.username); }) } },
-			{ K(S::CLIENT, MessageType::Client::LOGIN_RESPONSE), {
-				Make<ClientPackets::LoginResponse>([](const auto& p, json& j) {
-					json stamps = json::array();
-					for (const auto& stamp : p.stamps.list) stamps.push_back({ {"type", static_cast<int>(stamp.type)}, {"value", stamp.value}, {"timestamp", stamp.timestamp} });
-					j = { {"responseCode", static_cast<int>(p.responseCode)}, {"worldServerIP", p.worldServerIP.string}, {"worldServerPort", p.worldServerPort},
-						{"errorMessage", p.errorMessage}, {"stamps", stamps} };
-				}),
-				Blank<ClientPackets::LoginResponse>([](auto& p) { p.userKey.string.clear(); p.cdnKey.string.clear(); }) } },
-
-			// World
-			{ K(S::WORLD, MessageType::World::VALIDATION), {
-				Make<WorldPackets::Validation>([](const auto& p, json& j) { j = { {"username", p.username.GetAsString()}, {"fdbChecksum", p.fdbChecksum.string} }; }),
-				Blank<WorldPackets::Validation>([](auto& p) { p.sessionKey.string.clear(); }),
-				Scrub<WorldPackets::Validation>([](auto& p) { X(p.username); }) } },
-			{ K(S::WORLD, MessageType::World::CHARACTER_CREATE_REQUEST), { Make<WorldPackets::CharacterCreateRequest>([](const auto& p, json& j) {
-				j = { {"name", p.name.GetAsString()}, {"firstNameIndex", p.firstNameIndex}, {"middleNameIndex", p.middleNameIndex}, {"lastNameIndex", p.lastNameIndex},
-					{"shirtColor", p.shirtColor}, {"shirtStyle", p.shirtStyle}, {"pantsColor", p.pantsColor}, {"hairStyle", p.hairStyle}, {"hairColor", p.hairColor},
-					{"eyebrows", p.eyebrows}, {"eyes", p.eyes}, {"mouth", p.mouth} };
-			}),
-				nullptr, Scrub<WorldPackets::CharacterCreateRequest>(nullptr, [](auto& p) { X(p.name); }) } },
-			{ K(S::WORLD, MessageType::World::LOGIN_REQUEST), { Make<WorldPackets::CharacterLoginRequest>([](const auto& p, json& j) { j = { {"playerID", Id(p.playerID)} }; }) } },
-			{ K(S::WORLD, MessageType::World::CHARACTER_DELETE_REQUEST), { Make<WorldPackets::CharacterDeleteRequest>([](const auto& p, json& j) { j = { {"objectID", Id(p.objectID)} }; }) } },
-			{ K(S::WORLD, MessageType::World::CHARACTER_RENAME_REQUEST), { Make<WorldPackets::CharacterRenameRequest>([](const auto& p, json& j) {
-				j = { {"objectID", Id(p.objectID)}, {"name", p.name.GetAsString()} };
-			}),
-				nullptr, Scrub<WorldPackets::CharacterRenameRequest>(nullptr, [](auto& p) { X(p.name); }) } },
-			{ K(S::WORLD, MessageType::World::LEVEL_LOAD_COMPLETE), { Make<WorldPackets::LevelLoadComplete>([](const auto& p, json& j) {
-				j = { {"mapID", p.mapID}, {"instanceID", p.instanceID}, {"cloneID", p.cloneID} };
-			}) } },
-			{ K(S::WORLD, MessageType::World::POSITION_UPDATE), { Make<WorldPackets::PositionUpdate>([](const auto& p, json& j) {
-				j = { {"position", Point(p.update.position)}, {"rotation", Rotation(p.update.rotation)}, {"onGround", p.update.onGround}, {"onRail", p.update.onRail} };
-				if (p.hasVelocity) j["velocity"] = Point(p.update.velocity);
-				if (p.hasLocalSpaceInfo) j["platform"] = Id(p.update.localSpaceInfo.objectId);
-			}) } },
-			{ K(S::WORLD, MessageType::World::GENERAL_CHAT_MESSAGE), { Make<WorldPackets::GeneralChatMessage>([](const auto& p, json& j) {
-				j = { {"chatChannel", p.chatChannel}, {"message", GeneralUtils::UTF16ToWTF8(p.message)} };
-			}),
-				nullptr, Scrub<WorldPackets::GeneralChatMessage>(nullptr, [](auto& p) { X(p.message); }) } },
-			{ K(S::WORLD, MessageType::World::ROUTE_PACKET), { Make<WorldPackets::RoutePacket>([](const auto& p, json& j) {
-				j = { {"routed", PacketDecoder::Name(p.routedService, p.routedMessageID)}, {"size", p.size} };
-			}) } },
-
-			// To the client
-			{ K(S::CLIENT, MessageType::Client::LOAD_STATIC_ZONE), { Make<ClientPackets::LoadStaticZone>([](const auto& p, json& j) {
-				j = { {"mapID", p.mapID}, {"instanceID", p.instanceID}, {"cloneID", p.cloneID}, {"mapChecksum", p.mapChecksum}, {"playerPosition", Point(p.playerPosition)},
-					{"instanceType", p.instanceType} };
-			}) } },
-			{ K(S::CLIENT, MessageType::Client::CHARACTER_LIST_RESPONSE), { Make<ClientPackets::CharacterListResponse>([](const auto& p, json& j) {
-				json characters = json::array();
-				for (const auto& c : p.characters) {
-					characters.push_back({ {"objectID", Id(c.objectID)}, {"name", c.name.GetAsString()}, {"zoneID", c.zoneID}, {"equippedItems", c.equippedItems} });
-				}
-				j = { {"selectedCharacterIndex", p.selectedCharacterIndex}, {"characters", characters} };
-			}),
-				nullptr, Scrub<ClientPackets::CharacterListResponse>(nullptr, [](auto& p) { for (auto& c : p.characters) { X(c.name); X(c.unapprovedName); } }) } },
-			{ K(S::CLIENT, MessageType::Client::CREATE_CHARACTER), { Make<ClientPackets::CreateCharacter>([](const auto& p, json& j) {
-				j = { {"objectID", Id(p.objectID)}, {"templateID", p.templateID}, {"name", GeneralUtils::UTF16ToWTF8(p.name)}, {"gmLevel", static_cast<int>(p.gmLevel)},
-					{"reputation", p.reputation}, {"propertyCloneID", p.propertyCloneID}, {"xmlBytes", p.xmlData.size()} };
-			}),
-				nullptr, Scrub<ClientPackets::CreateCharacter>(nullptr, [](auto& p) { X(p.name); }) } },
-			{ K(S::CLIENT, MessageType::Client::CHARACTER_CREATE_RESPONSE), { Make<ClientPackets::CharacterCreateResponse>([](const auto& p, json& j) {
-				j = { {"response", static_cast<int>(p.response)} };
-			}) } },
-			{ K(S::CLIENT, MessageType::Client::TRANSFER_TO_WORLD), { Make<ClientPackets::TransferToWorld>([](const auto& p, json& j) {
-				j = { {"serverIP", p.serverIP.string}, {"serverPort", p.serverPort}, {"mythranShift", p.mythranShift} };
-			}) } },
-
-			// Chat
-			{ K(S::CHAT, MessageType::Chat::GENERAL_CHAT_MESSAGE), { Make<ChatPackets::GeneralChatMessage>([](const auto& p, json& j) {
-				j = { {"playerID", Id(p.playerID)}, {"chatChannel", static_cast<int>(p.chatChannel)}, {"message", p.message.GetAsString()} };
-			}),
-				nullptr, Scrub<ChatPackets::GeneralChatMessage>(nullptr, [](auto& p) { X(p.senderName); X(p.message); }) } },
-			{ K(S::CHAT, MessageType::Chat::PRIVATE_CHAT_MESSAGE), { Make<ChatPackets::PrivateChatMessage>([](const auto& p, json& j) {
-				j = { {"playerID", Id(p.playerID)}, {"senderName", p.senderName.GetAsString()}, {"receiverName", p.receiverName.GetAsString()},
-					{"responseCode", p.responseCode}, {"message", p.message.GetAsString()} };
-			}),
-				nullptr, Scrub<ChatPackets::PrivateChatMessage>(nullptr, [](auto& p) { X(p.senderName); X(p.receiverName); X(p.message); }) } },
-			{ K(S::CHAT, MessageType::Chat::WORLD_ROUTE_PACKET), { Make<ChatPackets::WorldRoutePacket>([](const auto& p, json& j) {
-				j = { {"targetID", Id(p.targetID)}, {"bytes", p.routedData.size()} };
-				if (p.routedData.size() >= 8 && p.routedData[0] == ID_USER_PACKET_ENUM) {
-					uint16_t service;
-					uint32_t id;
-					std::memcpy(&service, p.routedData.data() + 1, sizeof(service));
-					std::memcpy(&id, p.routedData.data() + 3, sizeof(id));
-					j["routed"] = PacketDecoder::Name(static_cast<ServiceType>(service), id);
-				}
-			}) } },
-
-			// Between servers: session keys are blanked when recorded
-			{ K(S::MASTER, MessageType::Master::REQUEST_SESSION_KEY), { Make<MasterPackets::RequestSessionKey>([](const auto& p, json& j) { j = { {"username", p.username.GetAsString()} }; }),
-				nullptr, Scrub<MasterPackets::RequestSessionKey>([](auto& p) { X(p.username); }) } },
-			{ K(S::MASTER, MessageType::Master::SESSION_KEY_RESPONSE), {
-				Make<MasterPackets::SessionKeyResponse>([](const auto& p, json& j) { j = { {"username", p.username.GetAsString()} }; }),
-				Blank<MasterPackets::SessionKeyResponse>([](auto& p) { p.sessionKey = 0; }),
-				Scrub<MasterPackets::SessionKeyResponse>([](auto& p) { X(p.username); }) } },
-			{ K(S::MASTER, MessageType::Master::SET_SESSION_KEY), {
-				Make<MasterPackets::SetSessionKey>([](const auto& p, json& j) { j = { {"username", p.username.string} }; }),
-				Blank<MasterPackets::SetSessionKey>([](auto& p) { p.sessionKey = 0; }),
-				Scrub<MasterPackets::SetSessionKey>([](auto& p) { X(p.username); }) } },
-			{ K(S::MASTER, MessageType::Master::NEW_SESSION_ALERT), {
-				Make<MasterPackets::NewSessionAlert>([](const auto& p, json& j) { j = { {"username", p.username.string} }; }),
-				Blank<MasterPackets::NewSessionAlert>([](auto& p) { p.sessionKey = 0; }),
-				Scrub<MasterPackets::NewSessionAlert>([](auto& p) { X(p.username); }) } },
-			{ K(S::MASTER, MessageType::Master::REQUEST_ZONE_TRANSFER), { Make<MasterPackets::RequestZoneTransfer>([](const auto& p, json& j) {
-				j = { {"requestID", Id(p.requestID)}, {"zoneID", p.zoneID}, {"cloneID", p.cloneID}, {"mythranShift", p.mythranShift}, {"stamps", p.stamps.size()} };
-			}) } },
-			{ K(S::MASTER, MessageType::Master::REQUEST_ZONE_TRANSFER_RESPONSE), { Make<MasterPackets::RequestZoneTransferResponse>([](const auto& p, json& j) {
-				j = { {"requestID", Id(p.requestID)}, {"zoneID", p.zoneID}, {"zoneInstance", p.zoneInstance}, {"zoneClone", p.zoneClone}, {"serverPort", p.serverPort},
-					{"stamps", p.stamps.size()} };
-			}) } },
-			{ K(S::MASTER, MessageType::Master::PLAYER_ADDED), { Make<MasterPackets::PlayerAdded>([](const auto& p, json& j) { j = { {"zoneID", p.zoneID}, {"instanceID", p.instanceID} }; }) } },
-			{ K(S::MASTER, MessageType::Master::PLAYER_REMOVED), { Make<MasterPackets::PlayerRemoved>([](const auto& p, json& j) { j = { {"zoneID", p.zoneID}, {"instanceID", p.instanceID} }; }) } },
 		};
 		return registry;
 	}
@@ -312,6 +318,27 @@ namespace {
 	}
 }
 
+namespace {
+	// Routed packets: the name of the packet they carry
+	void AddRoutedName(ServiceType service, uint32_t id, std::string_view bytes, json& fields) {
+		RakNet::BitStream stream(reinterpret_cast<unsigned char*>(const_cast<char*>(bytes.data())), static_cast<unsigned int>(bytes.size()), false);
+		stream.IgnoreBytes(8);
+		if (service == ServiceType::WORLD && id == static_cast<uint32_t>(MessageType::World::ROUTE_PACKET)) {
+			WorldPackets::RoutePacket packet;
+			if (packet.Deserialize(stream)) fields["routed"] = PacketDecoder::Name(packet.routedService, packet.routedMessageID);
+		} else if (service == ServiceType::CHAT && id == static_cast<uint32_t>(MessageType::Chat::WORLD_ROUTE_PACKET)) {
+			ChatPackets::WorldRoutePacket packet;
+			if (packet.Deserialize(stream) && packet.routedData.size() >= 8 && packet.routedData[0] == ID_USER_PACKET_ENUM) {
+				uint16_t routedService;
+				uint32_t routedId;
+				std::memcpy(&routedService, packet.routedData.data() + 1, sizeof(routedService));
+				std::memcpy(&routedId, packet.routedData.data() + 3, sizeof(routedId));
+				fields["routed"] = PacketDecoder::Name(static_cast<ServiceType>(routedService), routedId);
+			}
+		}
+	}
+}
+
 namespace PacketDecoder {
 	std::string Name(ServiceType service, uint32_t messageId) {
 		switch (service) {
@@ -344,7 +371,7 @@ namespace PacketDecoder {
 				int32_t lot{};
 				if (stream.Read(flag) && stream.Read(network) && stream.Read(object) && stream.Read(lot)) {
 					out.objectId = object;
-					out.fields = json{ {"networkID", network}, {"objectID", Id(object)}, {"lot", lot} };
+					out.fields = json{ {"networkID", network}, {"objectID", std::to_string(object)}, {"lot", lot} };
 				}
 			}
 			return out;
@@ -380,10 +407,8 @@ namespace PacketDecoder {
 			return out;
 		}
 
-		const auto it = Registry().find({ service, id });
+		out.fields = ReadFields(service, id, fromClient, stream, out.failed);
-		if (it == Registry().end()) return out;
+		if (out.fields) AddRoutedName(service, id, bytes, *out.fields);
-		out.fields = it->second.fields(stream, fromClient);
-		out.failed = !out.fields.has_value();
 		return out;
 	}
 
@@ -434,5 +459,7 @@ namespace PacketDecoder {
 		return it != Registry().end() && it->second.redact;
 	}
 
-	size_t RegisteredCount() { return Registry().size() + 1; }
+	size_t RegisteredCount() { return FieldIndex().size(); }
+
+	bool HasFields(ServiceType service, uint32_t messageId) { return FieldIndex().contains({ service, messageId }); }
 }

dNet/PacketDecoder.h

@@ -74,6 +74,9 @@ namespace PacketDecoder {
 
 	// How many (service, message) pairs have typed decoders, for tests
 	size_t RegisteredCount();
+
+	// Whether a (service, message) has a struct its fields are read with
+	bool HasFields(ServiceType service, uint32_t messageId);
 }
 
 #endif  //!__PACKETDECODER__H__

dNet/PacketJson.h

@@ -0,0 +1,118 @@
+#ifndef __PACKETJSON__H__
+#define __PACKETJSON__H__
+
+#include <chrono>
+#include <cstdint>
+#include <string>
+#include <type_traits>
+
+#include "BitStream.h"
+#include "BitStreamUtils.h"
+#include "dCommonVars.h"
+#include "Stamps.h"
+#include "GeneralUtils.h"
+#include "json.hpp"
+#include "LDFFormat.h"
+#include "magic_enum.hpp"
+#include "NiPoint3.h"
+#include "NiQuaternion.h"
+
+/**
+ * How a packet struct's members are shown in the capture viewer: one ToJson overload per member type. The member
+ * lists are generated from the structs (tools/gen_game_message_fields.py); a member type without an overload here (or
+ * in the file that includes the generated list) fails the build. Bring them in with `using PacketJson::ToJson;`.
+ */
+namespace PacketJson {
+	using json = nlohmann::json;
+
+	inline std::string Hex(std::string_view bytes) {
+		static constexpr char digits[] = "0123456789abcdef";
+		std::string out;
+		out.reserve(bytes.size() * 2);
+		for (const unsigned char c : bytes) {
+			out += digits[c >> 4];
+			out += digits[c & 15];
+		}
+		return out;
+	}
+
+	inline json ToJson(bool value) { return value; }
+	inline json ToJson(float value) { return value; }
+	inline json ToJson(double value) { return value; }
+	// 64-bit values (object IDs) as strings: they don't fit in a JavaScript number
+	inline json ToJson(int64_t value) { return std::to_string(value); }
+	inline json ToJson(uint64_t value) { return std::to_string(value); }
+	inline json ToJson(int32_t value) { return value; }
+	inline json ToJson(uint32_t value) { return value; }
+	inline json ToJson(int16_t value) { return value; }
+	inline json ToJson(uint16_t value) { return value; }
+	inline json ToJson(uint8_t value) { return value; }
+	inline json ToJson(int8_t value) { return value; }
+	inline json ToJson(char value) { return static_cast<int>(value); }
+	inline json ToJson(const NiPoint3& point) { return json::array({ point.x, point.y, point.z }); }
+	inline json ToJson(const NiQuaternion& rotation) { return json::array({ rotation.x, rotation.y, rotation.z, rotation.w }); }
+	inline json ToJson(const std::u16string& text) { return GeneralUtils::UTF16ToWTF8(text); }
+
+	// Text, or its bytes in hex when it isn't text (skill behavior streams are std::string too)
+	inline json ToJson(const std::string& text) {
+		for (const unsigned char c : text) {
+			if (c < 0x20 && c != '\n' && c != '\r' && c != '\t') return json{ {"hex", Hex(text)} };
+		}
+		try {
+			(void)json(text).dump(); // throws when it isn't UTF-8
+		} catch (const json::exception&) {
+			return json{ {"hex", Hex(text)} };
+		}
+		return text;
+	}
+
+	// Fixed-size strings: the text up to the first null
+	// (Templates, so nothing converts to them: they have implicit constructors from a size)
+	template<typename T> requires std::is_same_v<T, LUString>
+	json ToJson(const T& text) { return ToJson(text.string.substr(0, text.string.find('\0'))); }
+	template<typename T> requires std::is_same_v<T, LUWString>
+	json ToJson(const T& text) { return ToJson(text.string.substr(0, text.string.find(u'\0'))); }
+
+	inline json ToJson(const RakNet::BitStream& stream) {
+		return json{ {"hex", Hex(std::string_view(reinterpret_cast<const char*>(stream.GetData()), stream.GetNumberOfBytesUsed()))} };
+	}
+
+	inline json ToJson(const LwoNameValue& config) {
+		json out = json::array();
+		for (const auto& [key, value] : config.values) {
+			if (value) out.push_back(value->GetString());
+		}
+		return out;
+	}
+
+	inline json ToJson(const json& value) { return value; }
+
+	inline json ToJson(const LWOZONEID& zone) {
+		return json{ {"mapID", zone.GetMapID()}, {"instanceID", zone.GetInstanceID()}, {"cloneID", zone.GetCloneID()} };
+	}
+
+	inline json ToJson(const Stamps& stamps) {
+		json out = json::array();
+		for (const auto& stamp : stamps.list) {
+			const auto name = magic_enum::enum_name(stamp.type);
+			out.push_back({ {"type", name.empty() ? std::to_string(static_cast<int>(stamp.type)) : std::string(name)}, {"value", stamp.value}, {"timestamp", std::to_string(stamp.timestamp)} });
+		}
+		return out;
+	}
+
+	template<typename Clock, typename Duration>
+	json ToJson(const std::chrono::time_point<Clock, Duration>& time) {
+		return std::to_string(std::chrono::duration_cast<std::chrono::milliseconds>(time.time_since_epoch()).count());
+	}
+
+	// Enums by name, with the number when it has none
+	template<typename E> requires std::is_enum_v<E>
+	json ToJson(E value) {
+		const auto name = magic_enum::enum_name(value);
+		const auto number = static_cast<std::underlying_type_t<E>>(value);
+		if (name.empty()) return number;
+		return std::string(name) + " (" + std::to_string(number) + ")";
+	}
+}
+
+#endif  //!__PACKETJSON__H__

tools/gen_game_message_fields.py

@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+"""
+Writes the member lists the capture viewer shows packets with, from the packet structs themselves:
+
+  dGame/dGameMessages/GameMessageFields.inc   from dGame/dGameMessages/*Messages.h (game messages, NetGameMsg)
+  dNet/PacketFields.inc                       from dNet/*Packets.h (LU packets, LUBitStream)
+
+For every struct: its members by name, in declaration order (what is shown after the struct's own Deserialize read
+the packet); for every wire struct: its message ID, direction and whether it can be read. The lists come from the
+struct definitions, so a member added to a struct shows up in the viewer, and the build fails if a member's type has
+no ToJson overload (GameMessageDecoder.cpp, PacketDecoder.cpp). Members that hold secrets are never listed.
+
+  gen_game_message_fields.py            rewrite the files
+  gen_game_message_fields.py --check    exit 1 if a file is out of date (the PacketFieldsUpToDate test)
+"""
+import pathlib
+import re
+import sys
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+
+NAMESPACE = re.compile(r"^\s*namespace\s+(\w+)\s*\{")
+STRUCT = re.compile(r"^(\s*)struct\s+(\w+)\s*(?:final\s*)?(?::\s*public\s+(\w+))?\s*\{")
+MEMBER = re.compile(r"^\s*((?:const\s+)?[\w:]+(?:<.*>)?\s*\*?)\s+(\w+)\s*(?:\{.*\}|=.*)?;\s*$")
+SERVICES = {"Auth": "AUTH", "Chat": "CHAT", "Client": "CLIENT", "Master": "MASTER", "Server": "COMMON", "World": "WORLD"}
+SKIP_TYPES = ("static", "using", "friend", "return", "virtual", "typedef", "enum", "constexpr", "inline")
+# Never shown, even blanked
+SECRETS = {"password", "sessionKey", "userKey", "cdnKey", "passwordHash"}
+# Set by the server, not read from the wire
+NOT_WIRE = {"sysAddr"}
+# Have their own ToJson (private members)
+SKIP_STRUCTS = {"LWOZONEID"}
+
+
+def strip_comment(line):
+	line = re.sub(r"/\*.*?\*/", "", line)
+	return line.split("//", 1)[0]
+
+
+def parse(path):
+	structs = []
+	stack = []  # (kind, value, depth): kind "ns" or "struct"
+	depth = 0
+	in_block_comment = False
+	for raw in path.read_text().splitlines():
+		line = raw
+		if in_block_comment:
+			if "*/" not in line:
+				continue
+			line = line.split("*/", 1)[1]
+			in_block_comment = False
+		line = strip_comment(line)
+		if "/*" in line:
+			line = line.split("/*", 1)[0]
+			in_block_comment = True
+		namespaces = [v for k, v, _ in stack if k == "ns"]
+		parents = [v for k, v, _ in stack if k == "struct"]
+		if (n := NAMESPACE.match(line)) and not parents:
+			depth += line.count("{") - line.count("}")
+			stack.append(("ns", n.group(1), depth))
+			continue
+		m = STRUCT.match(line)
+		if m and not line.rstrip().endswith(";"):
+			name = m.group(2)
+			qualified = "::".join(namespaces + [p["short"] for p in parents] + [name])
+			s = {"name": qualified, "short": "::".join([p["short"] for p in parents] + [name]), "base": m.group(3),
+				 "members": [], "types": [], "id": None, "enum": None, "handle": False, "deserialize": False, "file": path.name}
+			structs.append(s)
+			depth += line.count("{") - line.count("}")
+			stack.append(("struct", s, depth))
+			continue
+		current = parents[-1] if parents else None
+		if current is not None and stack[-1][0] == "struct" and depth == stack[-1][2]:
+			own = current["short"].split("::")[-1]
+			if (mid := re.search(rf"\b{own}\s*\(\s*\)\s*:.*MessageType::(\w+)::(\w+)", line)):
+				current["enum"], current["id"] = mid.group(1), mid.group(2)
+			if re.search(r"\bvoid\s+Handle\s*\(", line):
+				current["handle"] = True
+			if re.search(r"\bbool\s+Deserialize\s*\(", line):
+				current["deserialize"] = True
+			mm = MEMBER.match(line)
+			if mm and "(" not in line.split("{")[0].split("=")[0] and ")" not in line:
+				typ, name = mm.group(1).strip(), mm.group(2)
+				if typ.split()[0] not in SKIP_TYPES and not typ.endswith("*"):
+					current["members"].append(name)
+					current["types"].append(typ)
+		depth += line.count("{") - line.count("}")
+		while stack and depth < stack[-1][2]:
+			stack.pop()
+	return structs
+
+
+def reachable(structs, wire):
+	"""The wire structs, their bases and the structs their members hold: the only ones shown"""
+	by_short = {}
+	for s in structs:
+		by_short.setdefault(s["name"].split("::")[-1], []).append(s)
+	keep, todo = [], list(wire)
+	while todo:
+		s = todo.pop()
+		if s in keep:
+			continue
+		keep.append(s)
+		words = re.findall(r"\w+", " ".join(s["types"])) + [s["base"] or ""]
+		for word in words:
+			todo += by_short.get(word, [])
+	return [s for s in structs if s in keep]
+
+
+def member_lines(structs):
+	known = {s["name"].split("::")[-1]: s["name"] for s in structs}
+	lines = ["// Every struct's members, in declaration order (a base struct's first)"]
+	for s in structs:
+		lines.append(f"json ToJson(const {s['name']}& m);")
+	lines.append("")
+	for s in structs:
+		lines.append(f"json ToJson(const {s['name']}& m) {{")
+		base = known.get(s["base"] or "")
+		lines.append(f"\tjson j = ToJson(static_cast<const {base}&>(m));" if base else "\tjson j = json::object();")
+		for member in s["members"]:
+			if member in SECRETS or member in NOT_WIRE:
+				continue
+			lines.append(f"\tj[\"{member}\"] = ToJson(m.{member});")
+		lines.append("\treturn j;")
+		lines.append("}")
+	lines.append("")
+	return lines
+
+
+def game_messages():
+	directory = ROOT / "dGame" / "dGameMessages"
+	structs = []
+	for path in sorted(directory.glob("*Messages.h")):
+		if path.name == "GameMessages.h":
+			continue
+		structs += [s for s in parse(path) if s["base"] != "GameMsg"]
+	wire = [s for s in structs if s["enum"] == "Game" and s["id"]]
+	lines = member_lines(reachable(structs, wire))
+	lines.append("// Wire messages: (ID, direction, reader). A struct with Handle is what the server reads from a client.")
+	lines.append("// Structs with members but no Deserialize can't be read.")
+	lines.append("const std::vector<Entry>& Entries() {")
+	lines.append("\tstatic const std::vector<Entry> entries{")
+	for s in wire:
+		direction = "eDirection::TO_SERVER" if s["handle"] else "eDirection::TO_CLIENT"
+		readable = s["deserialize"] or not s["members"]
+		reader = f"&ReadWith<{s['name']}>" if readable else "nullptr"
+		lines.append(f"\t\t{{ MessageType::Game::{s['id']}, {direction}, \"{s['short']}\", {reader} }},")
+	lines.append("\t};")
+	lines.append("\treturn entries;")
+	lines.append("}")
+	unreadable = [s["name"] for s in wire if s["members"] and not s["deserialize"]]
+	return directory / "GameMessageFields.inc", lines, f"{len(wire)} game messages, unreadable: {unreadable}"
+
+
+def packets():
+	directory = ROOT / "dNet"
+	structs = []
+	for path in sorted(directory.glob("*Packets.h")) + [directory / "WorldRoutePacket.h"] + sorted((directory / "master").glob("*.h")) + [ROOT / "dCommon" / name for name in ("PositionUpdate.h", "Profiler.h", "TrafficStats.h", "ZoneFileLog.h", "dEnums/dCommonVars.h")]:
+		structs += [s for s in parse(path) if s["name"] not in SKIP_STRUCTS]
+	wire = [s for s in structs if s["enum"] in SERVICES and s["id"]]
+	lines = member_lines(reachable(structs, wire))
+	lines.append("// Wire packets: (service, ID, struct, reader). An ID can have more than one struct (one per direction):")
+	lines.append("// the first that reads the whole packet is shown.")
+	lines.append("const std::vector<Entry>& Entries() {")
+	lines.append("\tstatic const std::vector<Entry> entries{")
+	for s in wire:
+		enum, service = s["enum"], SERVICES[s["enum"]]
+		readable = s["deserialize"] or not s["members"]
+		reader = f"&ReadWith<{s['name']}>" if readable else "nullptr"
+		lines.append(f"\t\t{{ ServiceType::{service}, static_cast<uint32_t>(MessageType::{enum}::{s['id']}), \"{s['short']}\", {s['handle'] and 'true' or 'false'}, {reader} }},")
+	lines.append("\t};")
+	lines.append("\treturn entries;")
+	lines.append("}")
+	unreadable = [s["name"] for s in wire if s["members"] and not s["deserialize"]]
+	return directory / "PacketFields.inc", lines, f"{len(wire)} packets, unreadable: {unreadable}"
+
+
+def main():
+	status = 0
+	for out, lines, summary in (game_messages(), packets()):
+		header = [
+			"// Generated by tools/gen_game_message_fields.py from the packet structs. Do not edit: run the script.",
+			"",
+		]
+		text = "\n".join(header + lines) + "\n"
+		if "--check" in sys.argv:
+			if not out.exists() or out.read_text() != text:
+				print(f"{out.relative_to(ROOT)} is out of date: run tools/gen_game_message_fields.py", file=sys.stderr)
+				status = 1
+			continue
+		out.write_text(text)
+		print(f"{out.relative_to(ROOT)}: {summary}")
+	return status
+
+
+if __name__ == "__main__":
+	sys.exit(main())