From 6b9310e513321227f5917736c0dcd173334f8c28 Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Sat, 26 Sep 2026 18:55:03 -0500 Subject: [PATCH] perf(dashboard): economy reports on a large ledger - Leave staff out of the currency and U-score reports by excluding the few staff characters, instead of joining every ledger row to its character and account. - Top earners: total per character first and look up names for the top rows only. - The places list and the activity report read map events for every kind in one query (GetMapZonesAllKinds) instead of one query per kind. With about 1M currency rows and 650k map event rows (90 days) on MariaDB: currency 1.8 s to 0.8 s, top earners 2.4 s to 0.75 s, places 2.6 s to 0.5 s, activity 2.6 s to 1.7 s. Co-Authored-By: Claude Opus 5.5 --- dDashboardServer/routes/EconomyPlaces.cpp | 7 +++-- dDashboardServer/routes/ReportRoutes.cpp | 8 +++--- .../GameDatabase/ITables/IEconomyLedger.h | 4 +++ dDatabase/GameDatabase/MySQL/MySQLDatabase.h | 1 + .../GameDatabase/MySQL/Tables/Economy.cpp | 26 ++++++++++++++----- .../GameDatabase/SQLite/SQLiteDatabase.h | 1 + .../GameDatabase/SQLite/Tables/Economy.cpp | 11 ++++++++ .../GameDatabase/TestSQL/TestSQLDatabase.h | 1 + tests/dDatabaseTests/DatabaseParityTests.cpp | 1 + 9 files changed, 45 insertions(+), 15 deletions(-) diff --git a/dDashboardServer/routes/EconomyPlaces.cpp b/dDashboardServer/routes/EconomyPlaces.cpp index 82d25fba0..0160fa214 100644 --- a/dDashboardServer/routes/EconomyPlaces.cpp +++ b/dDashboardServer/routes/EconomyPlaces.cpp @@ -223,10 +223,9 @@ namespace EconomyPlaces { // Every (zone, clone) with map events or statistics in the range std::map, PlaceTotals> PlacesWithData(uint32_t from, uint32_t to) { std::map, PlaceTotals> places; - for (const auto kind : magic_enum::enum_values()) { - for (const auto& row : Database::Get()->GetMapZones(kind, from, to)) { - places[{ row.value("zone", 0u), row.value("clone", 0u) }].events += row.value("events", int64_t{ 0 }); - } + for (const auto& row : Database::Get()->GetMapZonesAllKinds(from, to)) { + if (!magic_enum::enum_cast(row.value("kind", 0))) continue; + places[{ row.value("zone", 0u), row.value("clone", 0u) }].events += row.value("events", int64_t{ 0 }); } for (const auto& row : Database::Get()->GetPlayerStatsPerZone(from, to, false)) { places[{ row.value("zone", 0u), row.value("clone", 0u) }].stats += row.value("amount", int64_t{ 0 }); diff --git a/dDashboardServer/routes/ReportRoutes.cpp b/dDashboardServer/routes/ReportRoutes.cpp index 8e68b7cdd..6f623993a 100644 --- a/dDashboardServer/routes/ReportRoutes.cpp +++ b/dDashboardServer/routes/ReportRoutes.cpp @@ -1025,11 +1025,9 @@ void RegisterReportRoutes() { auto statRows = Database::Get()->GetPlayerStatsPerZone(from, to, excludeStaff); nlohmann::json mapRows = nlohmann::json::array(); - for (const auto kind : magic_enum::enum_values()) { - for (auto& row : Database::Get()->GetMapZones(kind, from, to)) { - row["kind"] = static_cast(kind); - mapRows.push_back(std::move(row)); - } + // One pass over the range for every kind (a query per kind read it once each) + for (auto& row : Database::Get()->GetMapZonesAllKinds(from, to)) { + if (magic_enum::enum_cast(row.value("kind", 0))) mapRows.push_back(std::move(row)); } std::set clones; for (const auto* rows : { &statRows, &mapRows }) for (const auto& row : *rows) clones.insert(row.value("clone", 0u)); diff --git a/dDatabase/GameDatabase/ITables/IEconomyLedger.h b/dDatabase/GameDatabase/ITables/IEconomyLedger.h index 17cfa2c6a..fabd64381 100644 --- a/dDatabase/GameDatabase/ITables/IEconomyLedger.h +++ b/dDatabase/GameDatabase/ITables/IEconomyLedger.h @@ -160,6 +160,10 @@ public: // Zones and clones with map events of a kind in the range: {zone, clone, events, quantity} virtual nlohmann::json GetMapZones(eMapEvent kind, uint32_t fromDay, uint32_t toDay) = 0; + // GetMapZones for every kind at once, in one pass over the range: {kind, zone, clone, events, quantity}, by kind and then + // in GetMapZones' order + virtual nlohmann::json GetMapZonesAllKinds(uint32_t fromDay, uint32_t toDay) = 0; + // LOTs with the most events of a kind in a zone (one clone, or every instance): {lot, events, quantity} virtual nlohmann::json GetMapLots(uint32_t zone, std::optional clone, eMapEvent kind, uint32_t fromDay, uint32_t toDay, uint32_t limit) = 0; diff --git a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h index 7924ebc28..cbe5f3ed3 100644 --- a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h +++ b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h @@ -274,6 +274,7 @@ public: uint32_t GetMaxBugReportId() override; nlohmann::json GetTransfersForCharacters(const std::vector& characterIds, uint32_t start, uint32_t length) override; nlohmann::json GetMapZones(eMapEvent kind, uint32_t fromDay, uint32_t toDay) override; + nlohmann::json GetMapZonesAllKinds(uint32_t fromDay, uint32_t toDay) override; nlohmann::json GetMapLots(uint32_t zone, std::optional clone, eMapEvent kind, uint32_t fromDay, uint32_t toDay, uint32_t limit) override; nlohmann::json GetMapCells(uint32_t zone, std::optional clone, eMapEvent kind, uint32_t fromDay, uint32_t toDay, LOT lot) override; nlohmann::json GetMapEventsPerDay(uint32_t fromDay, uint32_t toDay, const PlaceFilter& place) override; diff --git a/dDatabase/GameDatabase/MySQL/Tables/Economy.cpp b/dDatabase/GameDatabase/MySQL/Tables/Economy.cpp index 876c2329b..9965e9d19 100644 --- a/dDatabase/GameDatabase/MySQL/Tables/Economy.cpp +++ b/dDatabase/GameDatabase/MySQL/Tables/Economy.cpp @@ -2,8 +2,10 @@ namespace { // Staff (GM 3+) are excluded from player reports, matching NexusDashboard. Deleted characters still count. - constexpr const char* STAFF_FILTER = - " LEFT JOIN charinfo c ON c.id = f.character_id LEFT JOIN accounts a ON a.id = c.account_id WHERE COALESCE(a.gm_level, 0) < 3 AND "; + // The few staff characters are looked up once; joining every ledger row to its character and account took several + // times as long on a large ledger. + constexpr const char* STAFF_CHARACTERS = "SELECT c.id FROM charinfo c JOIN accounts a ON a.id = c.account_id WHERE a.gm_level >= 3"; + const std::string STAFF_FILTER = std::string(" WHERE f.character_id NOT IN (") + STAFF_CHARACTERS + ") AND "; constexpr const char* NO_FILTER = " WHERE "; } @@ -89,11 +91,12 @@ nlohmann::json MySQLDatabase::GetItemFlows(uint32_t fromDay, uint32_t toDay, LOT nlohmann::json MySQLDatabase::GetTopEarners(uint32_t fromDay, uint32_t toDay, uint32_t limit, bool excludeStaff) { // Trades and mail move coins between players rather than creating them, so they are left out of "income" + // Totals per character first, names for the top ones only auto result = ExecuteSelect(std::string( - "SELECT f.character_id, COALESCE(c.name, '') AS name, SUM(f.gained) AS gained, SUM(f.spent) AS spent FROM economy_currency_daily f " - "LEFT JOIN charinfo c ON c.id = f.character_id LEFT JOIN accounts a ON a.id = c.account_id " - "WHERE f.day BETWEEN ? AND ? AND f.source NOT IN (3, 6)") + (excludeStaff ? " AND COALESCE(a.gm_level, 0) < 3" : "") + - " GROUP BY f.character_id ORDER BY gained DESC LIMIT ?;", fromDay, toDay, limit); + "SELECT t.character_id, COALESCE(c.name, '') AS name, t.gained, t.spent FROM (SELECT f.character_id, SUM(f.gained) AS gained, " + "SUM(f.spent) AS spent FROM economy_currency_daily f WHERE f.day BETWEEN ? AND ? AND f.source NOT IN (3, 6)") + + (excludeStaff ? std::string(" AND f.character_id NOT IN (") + STAFF_CHARACTERS + ")" : std::string()) + + " GROUP BY f.character_id ORDER BY gained DESC LIMIT ?) t LEFT JOIN charinfo c ON c.id = t.character_id ORDER BY t.gained DESC;", fromDay, toDay, limit); nlohmann::json rows = nlohmann::json::array(); while (result->next()) { rows.push_back({ {"character_id", std::to_string(result->getInt64("character_id"))}, {"name", std::string(result->getString("name").c_str())}, @@ -206,6 +209,17 @@ nlohmann::json MySQLDatabase::GetMapZones(eMapEvent kind, uint32_t fromDay, uint return rows; } +nlohmann::json MySQLDatabase::GetMapZonesAllKinds(uint32_t fromDay, uint32_t toDay) { + auto result = ExecuteSelect("SELECT kind, zone, clone_id, SUM(events) AS events, SUM(quantity) AS quantity FROM map_events_daily " + "WHERE day BETWEEN ? AND ? GROUP BY kind, zone, clone_id ORDER BY kind, events DESC, zone, clone_id;", fromDay, toDay); + nlohmann::json rows = nlohmann::json::array(); + while (result->next()) { + rows.push_back({ {"kind", result->getInt("kind")}, {"zone", result->getInt("zone")}, {"clone", result->getUInt("clone_id")}, + {"events", result->getInt64("events")}, {"quantity", result->getInt64("quantity")} }); + } + return rows; +} + nlohmann::json MySQLDatabase::GetMapLots(uint32_t zone, std::optional clone, eMapEvent kind, uint32_t fromDay, uint32_t toDay, uint32_t limit) { auto result = ExecuteSelect("SELECT lot, SUM(events) AS events, SUM(quantity) AS quantity FROM map_events_daily " "WHERE zone = ? AND kind = ? AND day BETWEEN ? AND ?" + CloneSql(clone) + " GROUP BY lot ORDER BY events DESC, lot LIMIT ?;", zone, static_cast(kind), fromDay, toDay, limit); diff --git a/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h b/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h index 6ae00b1aa..5af62bc1f 100644 --- a/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h +++ b/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h @@ -258,6 +258,7 @@ public: uint32_t GetMaxBugReportId() override; nlohmann::json GetTransfersForCharacters(const std::vector& characterIds, uint32_t start, uint32_t length) override; nlohmann::json GetMapZones(eMapEvent kind, uint32_t fromDay, uint32_t toDay) override; + nlohmann::json GetMapZonesAllKinds(uint32_t fromDay, uint32_t toDay) override; nlohmann::json GetMapLots(uint32_t zone, std::optional clone, eMapEvent kind, uint32_t fromDay, uint32_t toDay, uint32_t limit) override; nlohmann::json GetMapCells(uint32_t zone, std::optional clone, eMapEvent kind, uint32_t fromDay, uint32_t toDay, LOT lot) override; nlohmann::json GetMapEventsPerDay(uint32_t fromDay, uint32_t toDay, const PlaceFilter& place) override; diff --git a/dDatabase/GameDatabase/SQLite/Tables/Economy.cpp b/dDatabase/GameDatabase/SQLite/Tables/Economy.cpp index 9653d21cf..5346fd9e0 100644 --- a/dDatabase/GameDatabase/SQLite/Tables/Economy.cpp +++ b/dDatabase/GameDatabase/SQLite/Tables/Economy.cpp @@ -214,6 +214,17 @@ nlohmann::json SQLiteDatabase::GetMapZones(eMapEvent kind, uint32_t fromDay, uin return rows; } +nlohmann::json SQLiteDatabase::GetMapZonesAllKinds(uint32_t fromDay, uint32_t toDay) { + auto [_, result] = ExecuteSelect("SELECT kind, zone, clone_id, SUM(events) AS events, SUM(quantity) AS quantity FROM map_events_daily " + "WHERE day BETWEEN ? AND ? GROUP BY kind, zone, clone_id ORDER BY kind, events DESC, zone, clone_id;", fromDay, toDay); + nlohmann::json rows = nlohmann::json::array(); + for (; !result.eof(); result.nextRow()) { + rows.push_back({ {"kind", result.getIntField("kind")}, {"zone", result.getIntField("zone")}, {"clone", static_cast(result.getInt64Field("clone_id"))}, + {"events", result.getInt64Field("events")}, {"quantity", result.getInt64Field("quantity")} }); + } + return rows; +} + nlohmann::json SQLiteDatabase::GetMapLots(uint32_t zone, std::optional clone, eMapEvent kind, uint32_t fromDay, uint32_t toDay, uint32_t limit) { auto [_, result] = ExecuteSelect("SELECT lot, SUM(events) AS events, SUM(quantity) AS quantity FROM map_events_daily " "WHERE zone = ? AND kind = ? AND day BETWEEN ? AND ?" + CloneSql(clone) + " GROUP BY lot ORDER BY events DESC, lot LIMIT ?;", zone, static_cast(kind), fromDay, toDay, limit); diff --git a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h index 5622bc4f0..32adcbba8 100644 --- a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h +++ b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h @@ -304,6 +304,7 @@ class TestSQLDatabase : public GameDatabase { uint32_t GetMaxBugReportId() override { return 0; } nlohmann::json GetTransfersForCharacters(const std::vector& characterIds, uint32_t start, uint32_t length) override { return nlohmann::json::array(); } nlohmann::json GetMapZones(eMapEvent kind, uint32_t fromDay, uint32_t toDay) override { return nlohmann::json::array(); }; + nlohmann::json GetMapZonesAllKinds(uint32_t fromDay, uint32_t toDay) override { return nlohmann::json::array(); }; nlohmann::json GetMapLots(uint32_t zone, std::optional clone, eMapEvent kind, uint32_t fromDay, uint32_t toDay, uint32_t limit) override { return nlohmann::json::array(); }; nlohmann::json GetMapCells(uint32_t zone, std::optional clone, eMapEvent kind, uint32_t fromDay, uint32_t toDay, LOT lot) override { return nlohmann::json::array(); }; nlohmann::json GetMapEventsPerDay(uint32_t fromDay, uint32_t toDay, const PlaceFilter& place) override { return nlohmann::json::array(); }; diff --git a/tests/dDatabaseTests/DatabaseParityTests.cpp b/tests/dDatabaseTests/DatabaseParityTests.cpp index c8f666dc2..e9c248a88 100644 --- a/tests/dDatabaseTests/DatabaseParityTests.cpp +++ b/tests/dDatabaseTests/DatabaseParityTests.cpp @@ -819,6 +819,7 @@ TEST_F(ParitySeeded, Economy) { Both("GetTransfers without inventory moves", [](GameDatabase& db) { return json{ db.GetTransfers(0, 10, 0, 0), db.GetTransfers(0, 10, CHAR_ALICE2, 0) }; }); Both("GetTransfersForCharacters", [](GameDatabase& db) { return json{ db.GetTransfersForCharacters({ CHAR_ALICE, CHAR_ALICE2 }, 0, 10), db.GetTransfersForCharacters({}, 0, 10) }; }); Both("GetMapZones", [&](GameDatabase& db) { return json{ db.GetMapZones(IEconomyLedger::eMapEvent::ENEMY_KILLS, day, day + 1), db.GetMapZones(IEconomyLedger::eMapEvent::POWERUP_DROPS, day, day + 1) }; }); + Both("GetMapZonesAllKinds", [&](GameDatabase& db) { return json{ db.GetMapZonesAllKinds(day, day + 1), db.GetMapZonesAllKinds(day + 5, day + 6) }; }); Both("GetMapLots", [&](GameDatabase& db) { return json{ db.GetMapLots(1100, std::nullopt, IEconomyLedger::eMapEvent::ENEMY_KILLS, day, day + 1, 10), db.GetMapLots(1150, std::nullopt, IEconomyLedger::eMapEvent::ENEMY_KILLS, day, day + 1, 10), db.GetMapLots(1150, 3u, IEconomyLedger::eMapEvent::ENEMY_KILLS, day, day + 1, 10), db.GetMapLots(1150, 0u, IEconomyLedger::eMapEvent::ENEMY_KILLS, day, day + 1, 10) };