diff --git a/dDashboardServer/CMakeLists.txt b/dDashboardServer/CMakeLists.txt index c5c1b7a3f..cfb98ef24 100644 --- a/dDashboardServer/CMakeLists.txt +++ b/dDashboardServer/CMakeLists.txt @@ -30,7 +30,9 @@ target_include_directories(DashboardServer PRIVATE ) target_compile_definitions(DashboardServer PRIVATE PROJECT_VERSION="\"${PROJECT_VERSION}\"") -target_link_libraries(DashboardServer ${COMMON_LIBRARIES} dWeb dServer bcrypt OpenSSL::Crypto CURL::libcurl DashboardRoutes DashboardAuth) +# The game's libraries: the capture viewer reads game messages and replica packets with the game's own structs +target_link_libraries(DashboardServer ${COMMON_LIBRARIES} dWeb dServer bcrypt OpenSSL::Crypto CURL::libcurl DashboardRoutes DashboardAuth + dScripts dGameBase dComponents dUtilities dGameMessages dInventory dGame dChatFilter dZoneManager dPhysics Detour Recast tinyxml2 dWorldServer dNavigation) # Copy static files and templates to the build directory on every build, so editing only a template or diff --git a/dDashboardServer/DashboardServer.cpp b/dDashboardServer/DashboardServer.cpp index 0710815cb..a5c60b274 100644 --- a/dDashboardServer/DashboardServer.cpp +++ b/dDashboardServer/DashboardServer.cpp @@ -80,6 +80,8 @@ #include "CDClientBrowser.h" #include "master/MessageCapture.h" #include "CaptureReplay.h" +#include "GameMessageDecoder.h" +#include "PacketDecoder.h" #include "PublicRoutes.h" #include "Showcase.h" #include "ContrabandRoutes.h" @@ -127,6 +129,15 @@ namespace Game { dConfig* config = nullptr; Game::signal_t lastSignal = 0; std::mt19937 randomEngine; + // Also defined by every program that links the game's libraries (it decodes game messages with the game's own + // message structs); the dashboard runs no game, so they stay empty + dChatFilter* chatFilter = nullptr; + AssetManager* assetManager = nullptr; + RakPeerInterface* chatServer = nullptr; + SystemAddress chatSysAddr; + EntityManager* entityManager = nullptr; + dZoneManager* zoneManager = nullptr; + std::string projectVersion = PROJECT_VERSION; } // Define global server state @@ -476,6 +487,10 @@ int main(int argc, char** argv) { PreloadZoneData(); Scenery::Preload(); WorldView::Preload(); + // The capture viewer decodes game messages with the game's own structs and replica packets with the + // components of each LOT + PacketDecoder::SetGameMessageDecoder(GameMessageDecoder::Decode); + CaptureReplay::PreloadDecoding(); LOG("Read the client data for the 3D views in %lld ms", static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count())); } Workers::Start(); diff --git a/dDashboardServer/routes/CaptureReplay.cpp b/dDashboardServer/routes/CaptureReplay.cpp index e0b3a7acb..c68d792ef 100644 --- a/dDashboardServer/routes/CaptureReplay.cpp +++ b/dDashboardServer/routes/CaptureReplay.cpp @@ -1,11 +1,14 @@ #include "CaptureReplay.h" #include "MasterPackets.h" +#include #include #include #include #include #include +#include +#include #include #include @@ -20,6 +23,8 @@ #include "Logger.h" #include "master/MessageCapture.h" #include "PacketDecoder.h" +#include "ReplicaDecoder.h" +#include "Workers.h" #include "Permissions.h" #include "RouteUtils.h" #include "Web.h" @@ -76,14 +81,39 @@ namespace { std::map g_Captures; Clock::time_point g_NextPush{}; + // LOT -> components, read from the CDClient at startup (PreloadDecoding) and only read after: the replica pass runs + // on worker threads, which never query the CDClient + ReplicaDecoder::ComponentTable g_Components; + // The last bundle read for the viewer (a running capture's file grows: read again when its size changed) struct Cached { uint64_t id{}; uintmax_t size{}; CaptureBundle::Bundle bundle; int64_t startUs{}; + + // Replica packets' fields, read once per capture in timeline order (a serialization needs the construction + // before it), on a worker thread + std::once_flag decodeOnce; + std::atomic decoded{}; + std::vector> replica; + + void Decode() { + std::call_once(decodeOnce, [this] { + ReplicaDecoder::Session session(g_Components); + replica.resize(bundle.records.size()); + for (size_t i = 0; i < bundle.records.size(); i++) { + const auto& record = bundle.records[i]; + if (record.header.flags & PacketRecordFlags::GAP || CaptureTools::FromClient(record.header)) continue; + replica[i] = session.Decode(record.bytes, CaptureTools::ReplicaConnection(record.header)); + } + decoded = true; + }); + } + + const nlohmann::json* ReplicaFields(size_t i) const { return i < replica.size() && replica[i] ? &*replica[i] : nullptr; } }; - Cached g_Cache; + std::shared_ptr g_Cache; bool Connected() { return Game::server && Game::server->GetIsConnectedToMaster(); } @@ -235,7 +265,7 @@ namespace { } // The capture's records on one timeline (read again when the file grew) - const Cached* Load(const Session& session, HTTPReply& reply) { + std::shared_ptr Load(const Session& session, HTTPReply& reply) { const auto path = CaptureReplay::FileOf(session.id); std::error_code ec; const auto size = fs::file_size(path, ec); @@ -243,8 +273,9 @@ namespace { JsonError(reply, eHTTPStatusCode::NOT_FOUND, "The capture's file is gone (deleted, or kept on another machine)"); return nullptr; } - if (g_Cache.id == session.id && g_Cache.size == size) return &g_Cache; - Cached cached; + if (g_Cache && g_Cache->id == session.id && g_Cache->size == size) return g_Cache; + auto loaded = std::make_shared(); + auto& cached = *loaded; std::string error; if (!CaptureBundle::Load(path, cached.bundle, error)) { JsonError(reply, eHTTPStatusCode::INTERNAL_SERVER_ERROR, error); @@ -254,8 +285,8 @@ namespace { cached.id = session.id; cached.size = size; cached.startUs = cached.bundle.records.empty() ? session.startedAt * 1000000 : std::min(session.startedAt * 1000000, cached.bundle.records.front().header.timeUs); - g_Cache = std::move(cached); - return &g_Cache; + g_Cache = std::move(loaded); + return g_Cache; } std::string CharacterName(LWOOBJID id) { @@ -442,6 +473,11 @@ namespace CaptureReplay { } } + void PreloadDecoding() { + const auto lots = ReplicaDecoder::LoadComponentTable(g_Components); + LOG("Read the components of %zu LOTs for decoding replica packets", lots); + } + void RegisterRoutes() { Game::web.RegisterWSSubscription(TOPIC, std::function([] { return Permissions::Level(PERMISSION); })); @@ -574,7 +610,7 @@ namespace CaptureReplay { [](HTTPReply& reply, const HTTPContext& context) { const auto session = FindSession(context, reply); if (!session) return; - const auto* cached = Load(*session, reply); + const auto cached = Load(*session, reply); if (!cached) return; const auto offset = GeneralUtils::TryParse(QueryValue(context.queryString, "offset")).value_or(0); const auto limit = std::clamp(GeneralUtils::TryParse(QueryValue(context.queryString, "limit")).value_or(PAGE_DEFAULT), 1, PAGE_MAX); @@ -582,39 +618,48 @@ namespace CaptureReplay { for (auto& ch : q) ch = static_cast(std::toupper(static_cast(ch))); const auto source = QueryValue(context.queryString, "source"); const bool fields = QueryValue(context.queryString, "fields") != "0"; - json records = json::array(); - size_t matched = 0; - const auto& all = cached->bundle.records; - for (size_t i = 0; i < all.size(); i++) { - if (q.empty() && source.empty()) { - if (i < offset) continue; - if (records.size() >= limit) break; - records.push_back(CaptureTools::RecordJson(all[i], i, cached->startUs, fields)); - continue; + // Read on the main thread (the session JSON reads live capture state); the packets are decoded on a + // worker the first time, since replica packets need a pass over the whole capture + auto capture = StoredSessionJson(*session); + Workers::Reply(reply, context, cached->decoded, [cached, capture = std::move(capture), offset, limit, q, source, fields](HTTPReply& out) { + if (fields) cached->Decode(); + json records = json::array(); + size_t matched = 0; + const auto& all = cached->bundle.records; + for (size_t i = 0; i < all.size(); i++) { + if (q.empty() && source.empty()) { + if (i < offset) continue; + if (records.size() >= limit) break; + records.push_back(CaptureTools::RecordJson(all[i], i, cached->startUs, fields, cached->ReplicaFields(i))); + continue; + } + auto record = CaptureTools::RecordJson(all[i], i, cached->startUs, false); + if (!source.empty() && record.value("source", std::string{}) != source) continue; + if (!q.empty() && record.value("name", std::string{}).find(q) == std::string::npos) continue; + if (matched++ < offset || records.size() >= limit) continue; + records.push_back(fields ? CaptureTools::RecordJson(all[i], i, cached->startUs, true, cached->ReplicaFields(i)) : record); } - auto record = CaptureTools::RecordJson(all[i], i, cached->startUs, false); - if (!source.empty() && record.value("source", std::string{}) != source) continue; - if (!q.empty() && record.value("name", std::string{}).find(q) == std::string::npos) continue; - if (matched++ < offset || records.size() >= limit) continue; - records.push_back(fields ? CaptureTools::RecordJson(all[i], i, cached->startUs, true) : record); - } - JsonSuccess(reply, { {"capture", StoredSessionJson(*session)}, {"total", all.size()}, {"matched", q.empty() && source.empty() ? all.size() : matched}, - {"start", cached->startUs / 1000}, {"duration", all.empty() ? 0.0 : static_cast(all.back().header.timeUs - cached->startUs) / 1000.0}, - {"records", records} }); + JsonSuccess(out, { {"capture", capture}, {"total", all.size()}, {"matched", q.empty() && source.empty() ? all.size() : matched}, + {"start", cached->startUs / 1000}, {"duration", all.empty() ? 0.0 : static_cast(all.back().header.timeUs - cached->startUs) / 1000.0}, + {"records", records} }); + }); }); Route(eHTTPMethod::GET, "/api/inspector/sessions/:id/packets/:index", Perm(PERMISSION), "One captured packet with its bytes", [](HTTPReply& reply, const HTTPContext& context) { const auto session = FindSession(context, reply); if (!session) return; - const auto* cached = Load(*session, reply); + const auto cached = Load(*session, reply); if (!cached) return; const auto index = PathId(context.path, 5); if (!index || *index >= cached->bundle.records.size()) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No such packet"); - const auto& record = cached->bundle.records[*index]; - auto out = CaptureTools::RecordJson(record, *index, cached->startUs, true); - out["hex"] = MessageCapture::ToHex(record.bytes); - JsonSuccess(reply, { {"record", out} }); + Workers::Reply(reply, context, cached->decoded, [cached, index = *index](HTTPReply& out) { + cached->Decode(); + const auto& record = cached->bundle.records[index]; + auto json = CaptureTools::RecordJson(record, index, cached->startUs, true, cached->ReplicaFields(index)); + json["hex"] = MessageCapture::ToHex(record.bytes); + JsonSuccess(out, { {"record", json} }); + }); }); Route(eHTTPMethod::GET, "/api/inspector/sessions/:id/positions", Perm(PERMISSION), @@ -622,7 +667,7 @@ namespace CaptureReplay { [](HTTPReply& reply, const HTTPContext& context) { const auto session = FindSession(context, reply); if (!session) return; - const auto* cached = Load(*session, reply); + const auto cached = Load(*session, reply); if (!cached) return; const auto tracks = CaptureTools::Tracks(cached->bundle.records, cached->startUs); std::set zones; @@ -656,7 +701,7 @@ namespace CaptureReplay { [](HTTPReply& reply, const HTTPContext& context) { const auto session = FindSession(context, reply); if (!session) return; - const auto* cached = Load(*session, reply); + const auto cached = Load(*session, reply); if (!cached) return; const bool anonymise = QueryValue(context.queryString, "anonymise") == "1"; auto bundle = cached->bundle; diff --git a/dDashboardServer/routes/CaptureReplay.h b/dDashboardServer/routes/CaptureReplay.h index 076e02d63..15658850f 100644 --- a/dDashboardServer/routes/CaptureReplay.h +++ b/dDashboardServer/routes/CaptureReplay.h @@ -22,6 +22,9 @@ namespace CaptureReplay { void RegisterRoutes(); + // Reads what decoding captured replica packets needs from the CDClient (main thread, before the workers start) + void PreloadDecoding(); + // MESSAGE_CAPTURE_DATA with status PACKETS, via master void HandleData(const MessageCaptureData& data); diff --git a/dDashboardServer/static/js/packet-captures.js b/dDashboardServer/static/js/packet-captures.js index fae4d88f3..f050708ad 100644 --- a/dDashboardServer/static/js/packet-captures.js +++ b/dDashboardServer/static/js/packet-captures.js @@ -302,7 +302,7 @@ byId('detailMeta').textContent = '#' + r.i + ' at ' + timeText(r.t) + ' · ' + (r.from || '') + ' → ' + (r.to || '') + ' · ' + r.source + (r.zone ? ' ' + r.zone + ':' + r.instance + (r.clone ? ':' + r.clone : '') : '') + ' · ' + nf.format(r.bytes) + ' bytes' + (r.account ? ' · account ' + r.account : '') + (r.character !== '0' ? ' · character ' + r.character : '') + (r.peer ? ' · ' + r.peer : ''); - byId('detailFields').textContent = r.fields ? JSON.stringify(r.fields, null, 2) : (r.gap ? r.gap + ' packets were lost here (a server buffer was full)' : 'No decoder for this packet; its bytes are below.'); + byId('detailFields').textContent = r.fields ? JSON.stringify(r.fields, null, 2) : (r.gap ? r.gap + ' packets were lost here (a server buffer was full)' : 'Not decoded: no struct reads this packet (the server never sends or handles it). Its bytes are below.'); byId('detailHex').textContent = hexdump(r.hex || ''); }).catch(function () {}); });