diff --git a/dDashboardServer/routes/EconomyPlaces.cpp b/dDashboardServer/routes/EconomyPlaces.cpp index 82d25fba0..0160fa214 100644 --- a/dDashboardServer/routes/EconomyPlaces.cpp +++ b/dDashboardServer/routes/EconomyPlaces.cpp @@ -223,10 +223,9 @@ namespace EconomyPlaces { // Every (zone, clone) with map events or statistics in the range std::map, PlaceTotals> PlacesWithData(uint32_t from, uint32_t to) { std::map, PlaceTotals> places; - for (const auto kind : magic_enum::enum_values()) { - for (const auto& row : Database::Get()->GetMapZones(kind, from, to)) { - places[{ row.value("zone", 0u), row.value("clone", 0u) }].events += row.value("events", int64_t{ 0 }); - } + for (const auto& row : Database::Get()->GetMapZonesAllKinds(from, to)) { + if (!magic_enum::enum_cast(row.value("kind", 0))) continue; + places[{ row.value("zone", 0u), row.value("clone", 0u) }].events += row.value("events", int64_t{ 0 }); } for (const auto& row : Database::Get()->GetPlayerStatsPerZone(from, to, false)) { places[{ row.value("zone", 0u), row.value("clone", 0u) }].stats += row.value("amount", int64_t{ 0 }); diff --git a/dDashboardServer/routes/ReportRoutes.cpp b/dDashboardServer/routes/ReportRoutes.cpp index 8e68b7cdd..6f623993a 100644 --- a/dDashboardServer/routes/ReportRoutes.cpp +++ b/dDashboardServer/routes/ReportRoutes.cpp @@ -1025,11 +1025,9 @@ void RegisterReportRoutes() { auto statRows = Database::Get()->GetPlayerStatsPerZone(from, to, excludeStaff); nlohmann::json mapRows = nlohmann::json::array(); - for (const auto kind : magic_enum::enum_values()) { - for (auto& row : Database::Get()->GetMapZones(kind, from, to)) { - row["kind"] = static_cast(kind); - mapRows.push_back(std::move(row)); - } + // One pass over the range for every kind (a query per kind read it once each) + for (auto& row : Database::Get()->GetMapZonesAllKinds(from, to)) { + if (magic_enum::enum_cast(row.value("kind", 0))) mapRows.push_back(std::move(row)); } std::set clones; for (const auto* rows : { &statRows, &mapRows }) for (const auto& row : *rows) clones.insert(row.value("clone", 0u)); diff --git a/dDatabase/GameDatabase/ITables/IEconomyLedger.h b/dDatabase/GameDatabase/ITables/IEconomyLedger.h index 17cfa2c6a..fabd64381 100644 --- a/dDatabase/GameDatabase/ITables/IEconomyLedger.h +++ b/dDatabase/GameDatabase/ITables/IEconomyLedger.h @@ -160,6 +160,10 @@ public: // Zones and clones with map events of a kind in the range: {zone, clone, events, quantity} virtual nlohmann::json GetMapZones(eMapEvent kind, uint32_t fromDay, uint32_t toDay) = 0; + // GetMapZones for every kind at once, in one pass over the range: {kind, zone, clone, events, quantity}, by kind and then + // in GetMapZones' order + virtual nlohmann::json GetMapZonesAllKinds(uint32_t fromDay, uint32_t toDay) = 0; + // LOTs with the most events of a kind in a zone (one clone, or every instance): {lot, events, quantity} virtual nlohmann::json GetMapLots(uint32_t zone, std::optional clone, eMapEvent kind, uint32_t fromDay, uint32_t toDay, uint32_t limit) = 0; diff --git a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h index 7924ebc28..cbe5f3ed3 100644 --- a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h +++ b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h @@ -274,6 +274,7 @@ public: uint32_t GetMaxBugReportId() override; nlohmann::json GetTransfersForCharacters(const std::vector& characterIds, uint32_t start, uint32_t length) override; nlohmann::json GetMapZones(eMapEvent kind, uint32_t fromDay, uint32_t toDay) override; + nlohmann::json GetMapZonesAllKinds(uint32_t fromDay, uint32_t toDay) override; nlohmann::json GetMapLots(uint32_t zone, std::optional clone, eMapEvent kind, uint32_t fromDay, uint32_t toDay, uint32_t limit) override; nlohmann::json GetMapCells(uint32_t zone, std::optional clone, eMapEvent kind, uint32_t fromDay, uint32_t toDay, LOT lot) override; nlohmann::json GetMapEventsPerDay(uint32_t fromDay, uint32_t toDay, const PlaceFilter& place) override; diff --git a/dDatabase/GameDatabase/MySQL/Tables/Economy.cpp b/dDatabase/GameDatabase/MySQL/Tables/Economy.cpp index 876c2329b..9965e9d19 100644 --- a/dDatabase/GameDatabase/MySQL/Tables/Economy.cpp +++ b/dDatabase/GameDatabase/MySQL/Tables/Economy.cpp @@ -2,8 +2,10 @@ namespace { // Staff (GM 3+) are excluded from player reports, matching NexusDashboard. Deleted characters still count. - constexpr const char* STAFF_FILTER = - " LEFT JOIN charinfo c ON c.id = f.character_id LEFT JOIN accounts a ON a.id = c.account_id WHERE COALESCE(a.gm_level, 0) < 3 AND "; + // The few staff characters are looked up once; joining every ledger row to its character and account took several + // times as long on a large ledger. + constexpr const char* STAFF_CHARACTERS = "SELECT c.id FROM charinfo c JOIN accounts a ON a.id = c.account_id WHERE a.gm_level >= 3"; + const std::string STAFF_FILTER = std::string(" WHERE f.character_id NOT IN (") + STAFF_CHARACTERS + ") AND "; constexpr const char* NO_FILTER = " WHERE "; } @@ -89,11 +91,12 @@ nlohmann::json MySQLDatabase::GetItemFlows(uint32_t fromDay, uint32_t toDay, LOT nlohmann::json MySQLDatabase::GetTopEarners(uint32_t fromDay, uint32_t toDay, uint32_t limit, bool excludeStaff) { // Trades and mail move coins between players rather than creating them, so they are left out of "income" + // Totals per character first, names for the top ones only auto result = ExecuteSelect(std::string( - "SELECT f.character_id, COALESCE(c.name, '') AS name, SUM(f.gained) AS gained, SUM(f.spent) AS spent FROM economy_currency_daily f " - "LEFT JOIN charinfo c ON c.id = f.character_id LEFT JOIN accounts a ON a.id = c.account_id " - "WHERE f.day BETWEEN ? AND ? AND f.source NOT IN (3, 6)") + (excludeStaff ? " AND COALESCE(a.gm_level, 0) < 3" : "") + - " GROUP BY f.character_id ORDER BY gained DESC LIMIT ?;", fromDay, toDay, limit); + "SELECT t.character_id, COALESCE(c.name, '') AS name, t.gained, t.spent FROM (SELECT f.character_id, SUM(f.gained) AS gained, " + "SUM(f.spent) AS spent FROM economy_currency_daily f WHERE f.day BETWEEN ? AND ? AND f.source NOT IN (3, 6)") + + (excludeStaff ? std::string(" AND f.character_id NOT IN (") + STAFF_CHARACTERS + ")" : std::string()) + + " GROUP BY f.character_id ORDER BY gained DESC LIMIT ?) t LEFT JOIN charinfo c ON c.id = t.character_id ORDER BY t.gained DESC;", fromDay, toDay, limit); nlohmann::json rows = nlohmann::json::array(); while (result->next()) { rows.push_back({ {"character_id", std::to_string(result->getInt64("character_id"))}, {"name", std::string(result->getString("name").c_str())}, @@ -206,6 +209,17 @@ nlohmann::json MySQLDatabase::GetMapZones(eMapEvent kind, uint32_t fromDay, uint return rows; } +nlohmann::json MySQLDatabase::GetMapZonesAllKinds(uint32_t fromDay, uint32_t toDay) { + auto result = ExecuteSelect("SELECT kind, zone, clone_id, SUM(events) AS events, SUM(quantity) AS quantity FROM map_events_daily " + "WHERE day BETWEEN ? AND ? GROUP BY kind, zone, clone_id ORDER BY kind, events DESC, zone, clone_id;", fromDay, toDay); + nlohmann::json rows = nlohmann::json::array(); + while (result->next()) { + rows.push_back({ {"kind", result->getInt("kind")}, {"zone", result->getInt("zone")}, {"clone", result->getUInt("clone_id")}, + {"events", result->getInt64("events")}, {"quantity", result->getInt64("quantity")} }); + } + return rows; +} + nlohmann::json MySQLDatabase::GetMapLots(uint32_t zone, std::optional clone, eMapEvent kind, uint32_t fromDay, uint32_t toDay, uint32_t limit) { auto result = ExecuteSelect("SELECT lot, SUM(events) AS events, SUM(quantity) AS quantity FROM map_events_daily " "WHERE zone = ? AND kind = ? AND day BETWEEN ? AND ?" + CloneSql(clone) + " GROUP BY lot ORDER BY events DESC, lot LIMIT ?;", zone, static_cast(kind), fromDay, toDay, limit); diff --git a/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h b/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h index 6ae00b1aa..5af62bc1f 100644 --- a/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h +++ b/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h @@ -258,6 +258,7 @@ public: uint32_t GetMaxBugReportId() override; nlohmann::json GetTransfersForCharacters(const std::vector& characterIds, uint32_t start, uint32_t length) override; nlohmann::json GetMapZones(eMapEvent kind, uint32_t fromDay, uint32_t toDay) override; + nlohmann::json GetMapZonesAllKinds(uint32_t fromDay, uint32_t toDay) override; nlohmann::json GetMapLots(uint32_t zone, std::optional clone, eMapEvent kind, uint32_t fromDay, uint32_t toDay, uint32_t limit) override; nlohmann::json GetMapCells(uint32_t zone, std::optional clone, eMapEvent kind, uint32_t fromDay, uint32_t toDay, LOT lot) override; nlohmann::json GetMapEventsPerDay(uint32_t fromDay, uint32_t toDay, const PlaceFilter& place) override; diff --git a/dDatabase/GameDatabase/SQLite/Tables/Economy.cpp b/dDatabase/GameDatabase/SQLite/Tables/Economy.cpp index 9653d21cf..5346fd9e0 100644 --- a/dDatabase/GameDatabase/SQLite/Tables/Economy.cpp +++ b/dDatabase/GameDatabase/SQLite/Tables/Economy.cpp @@ -214,6 +214,17 @@ nlohmann::json SQLiteDatabase::GetMapZones(eMapEvent kind, uint32_t fromDay, uin return rows; } +nlohmann::json SQLiteDatabase::GetMapZonesAllKinds(uint32_t fromDay, uint32_t toDay) { + auto [_, result] = ExecuteSelect("SELECT kind, zone, clone_id, SUM(events) AS events, SUM(quantity) AS quantity FROM map_events_daily " + "WHERE day BETWEEN ? AND ? GROUP BY kind, zone, clone_id ORDER BY kind, events DESC, zone, clone_id;", fromDay, toDay); + nlohmann::json rows = nlohmann::json::array(); + for (; !result.eof(); result.nextRow()) { + rows.push_back({ {"kind", result.getIntField("kind")}, {"zone", result.getIntField("zone")}, {"clone", static_cast(result.getInt64Field("clone_id"))}, + {"events", result.getInt64Field("events")}, {"quantity", result.getInt64Field("quantity")} }); + } + return rows; +} + nlohmann::json SQLiteDatabase::GetMapLots(uint32_t zone, std::optional clone, eMapEvent kind, uint32_t fromDay, uint32_t toDay, uint32_t limit) { auto [_, result] = ExecuteSelect("SELECT lot, SUM(events) AS events, SUM(quantity) AS quantity FROM map_events_daily " "WHERE zone = ? AND kind = ? AND day BETWEEN ? AND ?" + CloneSql(clone) + " GROUP BY lot ORDER BY events DESC, lot LIMIT ?;", zone, static_cast(kind), fromDay, toDay, limit); diff --git a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h index 5622bc4f0..32adcbba8 100644 --- a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h +++ b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h @@ -304,6 +304,7 @@ class TestSQLDatabase : public GameDatabase { uint32_t GetMaxBugReportId() override { return 0; } nlohmann::json GetTransfersForCharacters(const std::vector& characterIds, uint32_t start, uint32_t length) override { return nlohmann::json::array(); } nlohmann::json GetMapZones(eMapEvent kind, uint32_t fromDay, uint32_t toDay) override { return nlohmann::json::array(); }; + nlohmann::json GetMapZonesAllKinds(uint32_t fromDay, uint32_t toDay) override { return nlohmann::json::array(); }; nlohmann::json GetMapLots(uint32_t zone, std::optional clone, eMapEvent kind, uint32_t fromDay, uint32_t toDay, uint32_t limit) override { return nlohmann::json::array(); }; nlohmann::json GetMapCells(uint32_t zone, std::optional clone, eMapEvent kind, uint32_t fromDay, uint32_t toDay, LOT lot) override { return nlohmann::json::array(); }; nlohmann::json GetMapEventsPerDay(uint32_t fromDay, uint32_t toDay, const PlaceFilter& place) override { return nlohmann::json::array(); }; diff --git a/tests/dDatabaseTests/DatabaseParityTests.cpp b/tests/dDatabaseTests/DatabaseParityTests.cpp index c8f666dc2..e9c248a88 100644 --- a/tests/dDatabaseTests/DatabaseParityTests.cpp +++ b/tests/dDatabaseTests/DatabaseParityTests.cpp @@ -819,6 +819,7 @@ TEST_F(ParitySeeded, Economy) { Both("GetTransfers without inventory moves", [](GameDatabase& db) { return json{ db.GetTransfers(0, 10, 0, 0), db.GetTransfers(0, 10, CHAR_ALICE2, 0) }; }); Both("GetTransfersForCharacters", [](GameDatabase& db) { return json{ db.GetTransfersForCharacters({ CHAR_ALICE, CHAR_ALICE2 }, 0, 10), db.GetTransfersForCharacters({}, 0, 10) }; }); Both("GetMapZones", [&](GameDatabase& db) { return json{ db.GetMapZones(IEconomyLedger::eMapEvent::ENEMY_KILLS, day, day + 1), db.GetMapZones(IEconomyLedger::eMapEvent::POWERUP_DROPS, day, day + 1) }; }); + Both("GetMapZonesAllKinds", [&](GameDatabase& db) { return json{ db.GetMapZonesAllKinds(day, day + 1), db.GetMapZonesAllKinds(day + 5, day + 6) }; }); Both("GetMapLots", [&](GameDatabase& db) { return json{ db.GetMapLots(1100, std::nullopt, IEconomyLedger::eMapEvent::ENEMY_KILLS, day, day + 1, 10), db.GetMapLots(1150, std::nullopt, IEconomyLedger::eMapEvent::ENEMY_KILLS, day, day + 1, 10), db.GetMapLots(1150, 3u, IEconomyLedger::eMapEvent::ENEMY_KILLS, day, day + 1, 10), db.GetMapLots(1150, 0u, IEconomyLedger::eMapEvent::ENEMY_KILLS, day, day + 1, 10) };