From a4fa7eb30ae6ffece1bde24ea5af3f0dd55e75db Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Tue, 29 Sep 2026 21:56:32 -0500 Subject: [PATCH] perf(cdclient): read ComponentsRegistry, ItemComponent and Objects from the fdb The world and master servers pass the client's res/cdclient.fdb to CDClientManager. When it opens, these three tables, all looked up by their first column, find rows through the fdb's buckets instead of each process caching the whole table: ComponentsRegistry keeps nothing, ItemComponent and Objects keep only the entries asked for (their API returns references). Ids whose rows CDServer.sqlite changes are loaded from SQLite at startup and win. Without an fdb (or with one whose columns don't match) the tables load from CDServer.sqlite as before. ItemComponent and Objects now fill entries from one template for both sources instead of copies of the same field list. Tests cover the SQLite changes on top of the fdb, the no-fdb and unmapped paths, and, when DLU_CLIENT_RES points at a client, every id of the three tables through the fdb against CDServer.sqlite. Co-Authored-By: Claude Opus 5.5 --- .../CDClientDatabase/CDClientManager.cpp | 22 +- dDatabase/CDClientDatabase/CDClientManager.h | 9 +- .../CDComponentsRegistryTable.cpp | 56 +++- .../CDComponentsRegistryTable.h | 14 + .../CDClientTables/CDItemComponentTable.cpp | 209 ++++++------ .../CDClientTables/CDItemComponentTable.h | 11 + .../CDClientTables/CDObjectsTable.cpp | 117 ++++--- .../CDClientTables/CDObjectsTable.h | 12 + dMasterServer/MasterServer.cpp | 2 +- dWorldServer/WorldServer.cpp | 3 +- tests/dGameTests/CDFdbTests.cpp | 304 ++++++++++++++++++ tests/dGameTests/CMakeLists.txt | 1 + 12 files changed, 581 insertions(+), 179 deletions(-) create mode 100644 tests/dGameTests/CDFdbTests.cpp diff --git a/dDatabase/CDClientDatabase/CDClientManager.cpp b/dDatabase/CDClientDatabase/CDClientManager.cpp index af1d8c596..f98134455 100644 --- a/dDatabase/CDClientDatabase/CDClientManager.cpp +++ b/dDatabase/CDClientDatabase/CDClientManager.cpp @@ -1,4 +1,6 @@ #include "CDClientManager.h" +#include "CDFdb.h" +#include "Logger.h" #include "CDActivityRewardsTable.h" #include "CDAnimationsTable.h" #include "CDBehaviorParameterTable.h" @@ -111,11 +113,20 @@ DEFINE_TABLE_STORAGE(CDSkillBehaviorTable); DEFINE_TABLE_STORAGE(CDTamingBuildPuzzleTable); DEFINE_TABLE_STORAGE(CDVendorComponentTable); -void CDClientManager::LoadValuesFromDatabase() { +void CDClientManager::LoadValuesFromDatabase(const std::filesystem::path& fdbPath) { if (!CDClientDatabase::isConnected) { throw std::runtime_error{ "CDClientDatabase is not connected!" }; } + if (!fdbPath.empty()) { + if (CDFdb::Open(fdbPath)) { + LOG("Reading CDClient rows by id from %s (%s)", fdbPath.string().c_str(), + CDFdb::Get()->IsMapped() ? "mapped, shared between processes" : "could not map it, read into memory"); + } else { + LOG("Could not open %s, reading CDClient from CDServer.sqlite only", fdbPath.string().c_str()); + } + } + CDActivityRewardsTable::Instance().LoadValuesFromDatabase(); CDActivitiesTable::Instance().LoadValuesFromDatabase(); CDCLIENT_DONT_CACHE_TABLE(CDAnimationsTable::Instance().LoadValuesFromDatabase()); @@ -123,10 +134,11 @@ void CDClientManager::LoadValuesFromDatabase() { CDBehaviorTemplateTable::Instance().LoadValuesFromDatabase(); CDBrickIDTableTable::Instance().LoadValuesFromDatabase(); CDCollectibleComponentTable::Instance().LoadValuesFromDatabase(); - // Always in memory: every entity and every inventory item asks for its components and item data, and looking up an + // ComponentsRegistry, ItemComponent and Objects read rows from the fdb when it is open. Without it, the first two are + // always in memory: every entity and every inventory item asks for its components and item data, and looking up an // id not seen yet in the unindexed CDClient scans the whole table (a character holding 3000 different items took a // minute to load) - CDComponentsRegistryTable::Instance().LoadValuesFromDatabase(); + if (!CDComponentsRegistryTable::Instance().LoadFromFdb()) CDComponentsRegistryTable::Instance().LoadValuesFromDatabase(); CDCurrencyTableTable::Instance().LoadValuesFromDatabase(); CDDeletionRestrictionsTable::Instance().LoadValuesFromDatabase(); CDDestructibleComponentTable::Instance().LoadValuesFromDatabase(); @@ -134,7 +146,7 @@ void CDClientManager::LoadValuesFromDatabase() { CDEventGatingTable::Instance().LoadValuesFromDatabase(); CDFeatureGatingTable::Instance().LoadValuesFromDatabase(); CDInventoryComponentTable::Instance().LoadValuesFromDatabase(); - CDItemComponentTable::Instance().LoadValuesFromDatabase(); + if (!CDItemComponentTable::Instance().LoadFromFdb()) CDItemComponentTable::Instance().LoadValuesFromDatabase(); CDItemSetSkillsTable::Instance().LoadValuesFromDatabase(); CDItemSetsTable::Instance().LoadValuesFromDatabase(); CDLevelProgressionLookupTable::Instance().LoadValuesFromDatabase(); @@ -146,7 +158,7 @@ void CDClientManager::LoadValuesFromDatabase() { CDMissionsTable::Instance().LoadValuesFromDatabase(); CDMovementAIComponentTable::Instance().LoadValuesFromDatabase(); CDObjectSkillsTable::Instance().LoadValuesFromDatabase(); - CDCLIENT_DONT_CACHE_TABLE(CDObjectsTable::Instance().LoadValuesFromDatabase()); + if (!CDObjectsTable::Instance().LoadFromFdb()) CDCLIENT_DONT_CACHE_TABLE(CDObjectsTable::Instance().LoadValuesFromDatabase()); CDPhysicsComponentTable::Instance().LoadValuesFromDatabase(); CDPackageComponentTable::Instance().LoadValuesFromDatabase(); CDPetComponentTable::Instance().LoadValuesFromDatabase(); diff --git a/dDatabase/CDClientDatabase/CDClientManager.h b/dDatabase/CDClientDatabase/CDClientManager.h index c1c4443d7..2d4162539 100644 --- a/dDatabase/CDClientDatabase/CDClientManager.h +++ b/dDatabase/CDClientDatabase/CDClientManager.h @@ -3,11 +3,18 @@ #define UNUSED_TABLE(v) +#include + /** * Initialize the CDClient tables so they are all loaded into memory. */ namespace CDClientManager { - void LoadValuesFromDatabase(); + /** + * @param fdbPath The client's cdclient.fdb. When it opens, the tables looked up by their first + * column read their rows from it (shared between processes) instead of caching them. Empty, or + * a file that doesn't open, keeps every table on CDServer.sqlite. + */ + void LoadValuesFromDatabase(const std::filesystem::path& fdbPath = {}); void LoadValuesFromDefaults(); /** diff --git a/dDatabase/CDClientDatabase/CDClientTables/CDComponentsRegistryTable.cpp b/dDatabase/CDClientDatabase/CDClientTables/CDComponentsRegistryTable.cpp index 4944c13ba..1a8e4181f 100644 --- a/dDatabase/CDClientDatabase/CDClientTables/CDComponentsRegistryTable.cpp +++ b/dDatabase/CDClientDatabase/CDClientTables/CDComponentsRegistryTable.cpp @@ -1,4 +1,6 @@ #include "CDComponentsRegistryTable.h" +#include "CDFdb.h" +#include "Logger.h" #include "eReplicaComponentType.h" void CDComponentsRegistryTable::LoadValuesFromDatabase() { @@ -20,15 +22,30 @@ void CDComponentsRegistryTable::LoadValuesFromDatabase() { tableData.finalize(); } -int32_t CDComponentsRegistryTable::GetByIDAndType(uint32_t id, eReplicaComponentType componentType, int32_t defaultValue) { - auto& entries = GetEntriesMutable(); - auto exists = entries.find(id); - if (exists != entries.end()) { - auto iter = entries.find(static_cast(componentType) << 32 | static_cast(id)); - return iter == entries.end() ? defaultValue : iter->second; - } +bool CDComponentsRegistryTable::LoadFromFdb() { + m_FdbTable = nullptr; + const auto* table = CDFdb::GetTable("ComponentsRegistry"); + if (!table) return false; - // Now get the data. Get all components of this entity so we dont do a query for each component + m_TypeColumn = table->GetColumnIndex("component_type"); + m_ComponentIdColumn = table->GetColumnIndex("component_id"); + if (m_TypeColumn < 0 || m_ComponentIdColumn < 0) return false; + + const auto changed = CDFdb::FindChangedKeys(*table); + if (!changed) return false; + + // Ids whose rows CDServer.sqlite changes are read from it once and kept; everything else comes from the fdb + for (const auto id : *changed) LoadFromSqlite(static_cast(id)); + LOG("ComponentsRegistry: reading from the fdb, %zu ids differ in CDServer.sqlite and are kept in memory", changed->size()); + + m_FdbTable = table; + return true; +} + +void CDComponentsRegistryTable::LoadFromSqlite(uint32_t id) { + auto& entries = GetEntriesMutable(); + + // Get all components of this entity so we dont do a query for each component auto query = CDClientDatabase::CreatePreppedStmt("SELECT * FROM ComponentsRegistry WHERE id = ?;"); query.bind(1, static_cast(id)); @@ -46,6 +63,29 @@ int32_t CDComponentsRegistryTable::GetByIDAndType(uint32_t id, eReplicaComponent } entries.insert_or_assign(id, 0); +} + +int32_t CDComponentsRegistryTable::GetByIDAndType(uint32_t id, eReplicaComponentType componentType, int32_t defaultValue) { + auto& entries = GetEntriesMutable(); + auto exists = entries.find(id); + if (exists != entries.end()) { + auto iter = entries.find(static_cast(componentType) << 32 | static_cast(id)); + return iter == entries.end() ? defaultValue : iter->second; + } + + if (m_FdbTable) { + // The last matching row wins, as when the rows are loaded into the map in file order + int32_t result = defaultValue; + const auto type = static_cast(componentType); + m_FdbTable->ForEachRowWithKey(static_cast(id), [&](const FdbReader::Row& row) { + if (row.GetInt(static_cast(m_TypeColumn), 0) == type) { + result = row.GetInt(static_cast(m_ComponentIdColumn), -1); + } + }); + return result; + } + + LoadFromSqlite(id); auto iter = entries.find(static_cast(componentType) << 32 | static_cast(id)); diff --git a/dDatabase/CDClientDatabase/CDClientTables/CDComponentsRegistryTable.h b/dDatabase/CDClientDatabase/CDClientTables/CDComponentsRegistryTable.h index 2165f907b..d1d581c40 100644 --- a/dDatabase/CDClientDatabase/CDClientTables/CDComponentsRegistryTable.h +++ b/dDatabase/CDClientDatabase/CDClientTables/CDComponentsRegistryTable.h @@ -5,6 +5,8 @@ #include +#include "FdbReader.h" + enum class eReplicaComponentType : uint32_t; struct CDComponentsRegistry { uint32_t id; //!< The LOT is used as the ID @@ -16,5 +18,17 @@ struct CDComponentsRegistry { class CDComponentsRegistryTable : public CDTable> { public: void LoadValuesFromDatabase(); + + // Reads rows from the client's fdb from now on, if it is open; false keeps the table on CDServer.sqlite + bool LoadFromFdb(); + int32_t GetByIDAndType(uint32_t id, eReplicaComponentType componentType, int32_t defaultValue = 0); + +private: + // Caches every row of one id from CDServer.sqlite + void LoadFromSqlite(uint32_t id); + + const FdbReader::Table* m_FdbTable = nullptr; + int32_t m_TypeColumn = -1; + int32_t m_ComponentIdColumn = -1; }; diff --git a/dDatabase/CDClientDatabase/CDClientTables/CDItemComponentTable.cpp b/dDatabase/CDClientDatabase/CDClientTables/CDItemComponentTable.cpp index 8b69e2620..68f1c98b5 100644 --- a/dDatabase/CDClientDatabase/CDClientTables/CDItemComponentTable.cpp +++ b/dDatabase/CDClientDatabase/CDClientTables/CDItemComponentTable.cpp @@ -1,68 +1,67 @@ #include "CDItemComponentTable.h" #include "GeneralUtils.h" +#include "CDFdb.h" +#include "Logger.h" CDItemComponent CDItemComponentTable::Default = {}; -void CDItemComponentTable::LoadValuesFromDatabase() { - // First, get the size of the table - uint32_t size = 0; - auto tableSize = CDClientDatabase::ExecuteQuery("SELECT COUNT(*) FROM ItemComponent"); - while (!tableSize.eof()) { - size = tableSize.getIntField(0, 0); - - tableSize.nextRow(); +namespace { + // Fills an entry from a CDServer.sqlite row or an fdb row (CDFdb::RowFields), which read alike + template + CDItemComponent ReadEntry(Row& row) { + CDItemComponent entry; + entry.id = row.getIntField("id", -1); + entry.equipLocation = row.getStringField("equipLocation", ""); + entry.baseValue = row.getIntField("baseValue", -1); + entry.isKitPiece = row.getIntField("isKitPiece", -1) == 1 ? true : false; + entry.rarity = row.getIntField("rarity", 0); + entry.itemType = row.getIntField("itemType", -1); + entry.itemInfo = row.getInt64Field("itemInfo", -1); + entry.inLootTable = row.getIntField("inLootTable", -1) == 1 ? true : false; + entry.inVendor = row.getIntField("inVendor", -1) == 1 ? true : false; + entry.isUnique = row.getIntField("isUnique", -1) == 1 ? true : false; + entry.isBOP = row.getIntField("isBOP", -1) == 1 ? true : false; + entry.isBOE = row.getIntField("isBOE", -1) == 1 ? true : false; + entry.reqFlagID = row.getIntField("reqFlagID", -1); + entry.reqSpecialtyID = row.getIntField("reqSpecialtyID", -1); + entry.reqSpecRank = row.getIntField("reqSpecRank", -1); + entry.reqAchievementID = row.getIntField("reqAchievementID", -1); + entry.stackSize = row.getIntField("stackSize", -1); + entry.color1 = row.getIntField("color1", -1); + entry.decal = row.getIntField("decal", -1); + entry.offsetGroupID = row.getIntField("offsetGroupID", -1); + entry.buildTypes = row.getIntField("buildTypes", -1); + entry.reqPrecondition = row.getStringField("reqPrecondition", ""); + entry.animationFlag = row.getIntField("animationFlag", 0); + entry.equipEffects = row.getIntField("equipEffects", -1); + entry.readyForQA = row.getIntField("readyForQA", -1) == 1 ? true : false; + entry.itemRating = row.getIntField("itemRating", -1); + entry.isTwoHanded = row.getIntField("isTwoHanded", -1) == 1 ? true : false; + entry.minNumRequired = row.getIntField("minNumRequired", -1); + entry.delResIndex = row.getIntField("delResIndex", -1); + entry.currencyLOT = row.getIntField("currencyLOT", -1); + entry.altCurrencyCost = row.getIntField("altCurrencyCost", -1); + entry.subItems = row.getStringField("subItems", ""); + UNUSED_COLUMN(entry.audioEventUse = row.getStringField("audioEventUse", "")); + entry.noEquipAnimation = row.getIntField("noEquipAnimation", -1) == 1 ? true : false; + entry.commendationLOT = row.getIntField("commendationLOT", -1); + entry.commendationCost = row.getIntField("commendationCost", -1); + UNUSED_COLUMN(entry.audioEquipMetaEventSet = row.getStringField("audioEquipMetaEventSet", "")); + entry.currencyCosts = row.getStringField("currencyCosts", ""); + UNUSED_COLUMN(entry.ingredientInfo = row.getStringField("ingredientInfo", "")); + entry.locStatus = row.getIntField("locStatus", -1); + entry.forgeType = row.getIntField("forgeType", -1); + entry.SellMultiplier = row.getFloatField("SellMultiplier", -1.0f); + return entry; } +} - tableSize.finalize(); - +void CDItemComponentTable::LoadValuesFromDatabase() { // Now get the data auto tableData = CDClientDatabase::ExecuteQuery("SELECT * FROM ItemComponent"); auto& entries = GetEntriesMutable(); while (!tableData.eof()) { - CDItemComponent entry; - entry.id = tableData.getIntField("id", -1); - entry.equipLocation = tableData.getStringField("equipLocation", ""); - entry.baseValue = tableData.getIntField("baseValue", -1); - entry.isKitPiece = tableData.getIntField("isKitPiece", -1) == 1 ? true : false; - entry.rarity = tableData.getIntField("rarity", 0); - entry.itemType = tableData.getIntField("itemType", -1); - entry.itemInfo = tableData.getInt64Field("itemInfo", -1); - entry.inLootTable = tableData.getIntField("inLootTable", -1) == 1 ? true : false; - entry.inVendor = tableData.getIntField("inVendor", -1) == 1 ? true : false; - entry.isUnique = tableData.getIntField("isUnique", -1) == 1 ? true : false; - entry.isBOP = tableData.getIntField("isBOP", -1) == 1 ? true : false; - entry.isBOE = tableData.getIntField("isBOE", -1) == 1 ? true : false; - entry.reqFlagID = tableData.getIntField("reqFlagID", -1); - entry.reqSpecialtyID = tableData.getIntField("reqSpecialtyID", -1); - entry.reqSpecRank = tableData.getIntField("reqSpecRank", -1); - entry.reqAchievementID = tableData.getIntField("reqAchievementID", -1); - entry.stackSize = tableData.getIntField("stackSize", -1); - entry.color1 = tableData.getIntField("color1", -1); - entry.decal = tableData.getIntField("decal", -1); - entry.offsetGroupID = tableData.getIntField("offsetGroupID", -1); - entry.buildTypes = tableData.getIntField("buildTypes", -1); - entry.reqPrecondition = tableData.getStringField("reqPrecondition", ""); - entry.animationFlag = tableData.getIntField("animationFlag", 0); - entry.equipEffects = tableData.getIntField("equipEffects", -1); - entry.readyForQA = tableData.getIntField("readyForQA", -1) == 1 ? true : false; - entry.itemRating = tableData.getIntField("itemRating", -1); - entry.isTwoHanded = tableData.getIntField("isTwoHanded", -1) == 1 ? true : false; - entry.minNumRequired = tableData.getIntField("minNumRequired", -1); - entry.delResIndex = tableData.getIntField("delResIndex", -1); - entry.currencyLOT = tableData.getIntField("currencyLOT", -1); - entry.altCurrencyCost = tableData.getIntField("altCurrencyCost", -1); - entry.subItems = tableData.getStringField("subItems", ""); - UNUSED_COLUMN(entry.audioEventUse = tableData.getStringField("audioEventUse", "")); - entry.noEquipAnimation = tableData.getIntField("noEquipAnimation", -1) == 1 ? true : false; - entry.commendationLOT = tableData.getIntField("commendationLOT", -1); - entry.commendationCost = tableData.getIntField("commendationCost", -1); - UNUSED_COLUMN(entry.audioEquipMetaEventSet = tableData.getStringField("audioEquipMetaEventSet", "")); - entry.currencyCosts = tableData.getStringField("currencyCosts", ""); - UNUSED_COLUMN(entry.ingredientInfo = tableData.getStringField("ingredientInfo", "")); - entry.locStatus = tableData.getIntField("locStatus", -1); - entry.forgeType = tableData.getIntField("forgeType", -1); - entry.SellMultiplier = tableData.getFloatField("SellMultiplier", -1.0f); - + CDItemComponent entry = ReadEntry(tableData); entries.insert(std::make_pair(entry.id, entry)); tableData.nextRow(); } @@ -70,6 +69,43 @@ void CDItemComponentTable::LoadValuesFromDatabase() { tableData.finalize(); } +bool CDItemComponentTable::LoadFromFdb() { + m_FdbTable = nullptr; + const auto* table = CDFdb::GetTable("ItemComponent"); + if (!table) return false; + + const auto changed = CDFdb::FindChangedKeys(*table); + if (!changed) return false; + + // Ids whose rows CDServer.sqlite changes are read from it once and kept; everything else comes from the fdb + for (const auto id : *changed) LoadFromSqlite(static_cast(id)); + LOG("ItemComponent: reading from the fdb, %zu ids differ in CDServer.sqlite and are kept in memory", changed->size()); + + m_FdbTable = table; + return true; +} + +const CDItemComponent& CDItemComponentTable::LoadFromSqlite(uint32_t id) { + auto& entries = GetEntriesMutable(); + auto query = CDClientDatabase::CreatePreppedStmt("SELECT * FROM ItemComponent WHERE id = ?;"); + query.bind(1, static_cast(id)); + + auto tableData = query.execQuery(); + if (tableData.eof()) { + entries.insert(std::make_pair(id, Default)); + return Default; + } + + while (!tableData.eof()) { + CDItemComponent entry = ReadEntry(tableData); + entries.insert(std::make_pair(entry.id, entry)); + tableData.nextRow(); + } + + const auto& it = entries.find(id); + return it != entries.end() ? it->second : Default; +} + const CDItemComponent& CDItemComponentTable::GetItemComponentByID(uint32_t skillID) { auto& entries = GetEntriesMutable(); const auto& it = entries.find(skillID); @@ -77,70 +113,17 @@ const CDItemComponent& CDItemComponentTable::GetItemComponentByID(uint32_t skill return it->second; } - auto query = CDClientDatabase::CreatePreppedStmt("SELECT * FROM ItemComponent WHERE id = ?;"); - query.bind(1, static_cast(skillID)); + if (!m_FdbTable) return LoadFromSqlite(skillID); - auto tableData = query.execQuery(); - if (tableData.eof()) { + // Only the items asked for are kept in memory; the first row of an id wins, as in the SQLite path + const auto row = m_FdbTable->FindFirst(static_cast(skillID)); + if (!row) { entries.insert(std::make_pair(skillID, Default)); return Default; } - while (!tableData.eof()) { - CDItemComponent entry; - entry.id = tableData.getIntField("id", -1); - entry.equipLocation = tableData.getStringField("equipLocation", ""); - entry.baseValue = tableData.getIntField("baseValue", -1); - entry.isKitPiece = tableData.getIntField("isKitPiece", -1) == 1 ? true : false; - entry.rarity = tableData.getIntField("rarity", 0); - entry.itemType = tableData.getIntField("itemType", -1); - entry.itemInfo = tableData.getInt64Field("itemInfo", -1); - entry.inLootTable = tableData.getIntField("inLootTable", -1) == 1 ? true : false; - entry.inVendor = tableData.getIntField("inVendor", -1) == 1 ? true : false; - entry.isUnique = tableData.getIntField("isUnique", -1) == 1 ? true : false; - entry.isBOP = tableData.getIntField("isBOP", -1) == 1 ? true : false; - entry.isBOE = tableData.getIntField("isBOE", -1) == 1 ? true : false; - entry.reqFlagID = tableData.getIntField("reqFlagID", -1); - entry.reqSpecialtyID = tableData.getIntField("reqSpecialtyID", -1); - entry.reqSpecRank = tableData.getIntField("reqSpecRank", -1); - entry.reqAchievementID = tableData.getIntField("reqAchievementID", -1); - entry.stackSize = tableData.getIntField("stackSize", -1); - entry.color1 = tableData.getIntField("color1", -1); - entry.decal = tableData.getIntField("decal", -1); - entry.offsetGroupID = tableData.getIntField("offsetGroupID", -1); - entry.buildTypes = tableData.getIntField("buildTypes", -1); - entry.reqPrecondition = tableData.getStringField("reqPrecondition", ""); - entry.animationFlag = tableData.getIntField("animationFlag", 0); - entry.equipEffects = tableData.getIntField("equipEffects", -1); - entry.readyForQA = tableData.getIntField("readyForQA", -1) == 1 ? true : false; - entry.itemRating = tableData.getIntField("itemRating", -1); - entry.isTwoHanded = tableData.getIntField("isTwoHanded", -1) == 1 ? true : false; - entry.minNumRequired = tableData.getIntField("minNumRequired", -1); - entry.delResIndex = tableData.getIntField("delResIndex", -1); - entry.currencyLOT = tableData.getIntField("currencyLOT", -1); - entry.altCurrencyCost = tableData.getIntField("altCurrencyCost", -1); - entry.subItems = tableData.getStringField("subItems", ""); - UNUSED(entry.audioEventUse = tableData.getStringField("audioEventUse", "")); - entry.noEquipAnimation = tableData.getIntField("noEquipAnimation", -1) == 1 ? true : false; - entry.commendationLOT = tableData.getIntField("commendationLOT", -1); - entry.commendationCost = tableData.getIntField("commendationCost", -1); - UNUSED(entry.audioEquipMetaEventSet = tableData.getStringField("audioEquipMetaEventSet", "")); - entry.currencyCosts = tableData.getStringField("currencyCosts", ""); - UNUSED(entry.ingredientInfo = tableData.getStringField("ingredientInfo", "")); - entry.locStatus = tableData.getIntField("locStatus", -1); - entry.forgeType = tableData.getIntField("forgeType", -1); - entry.SellMultiplier = tableData.getFloatField("SellMultiplier", -1.0f); - - entries.insert(std::make_pair(entry.id, entry)); - tableData.nextRow(); - } - - const auto& it2 = entries.find(skillID); - if (it2 != entries.end()) { - return it2->second; - } - - return Default; + const CDFdb::RowFields fields(*m_FdbTable, *row); + return entries.insert(std::make_pair(skillID, ReadEntry(fields))).first->second; } std::map CDItemComponentTable::ParseCraftingCurrencies(const CDItemComponent& itemComponent) { diff --git a/dDatabase/CDClientDatabase/CDClientTables/CDItemComponentTable.h b/dDatabase/CDClientDatabase/CDClientTables/CDItemComponentTable.h index 73683d66f..92b600544 100644 --- a/dDatabase/CDClientDatabase/CDClientTables/CDItemComponentTable.h +++ b/dDatabase/CDClientDatabase/CDClientTables/CDItemComponentTable.h @@ -3,6 +3,7 @@ // Custom Classes #include "CDTable.h" #include "dCommonVars.h" +#include "FdbReader.h" struct CDItemComponent { uint32_t id; //!< The Component ID @@ -52,10 +53,20 @@ struct CDItemComponent { class CDItemComponentTable : public CDTable> { public: void LoadValuesFromDatabase(); + + // Reads rows from the client's fdb from now on, if it is open; false keeps the table on CDServer.sqlite + bool LoadFromFdb(); + static std::map ParseCraftingCurrencies(const CDItemComponent& itemComponent); // Gets an entry by ID const CDItemComponent& GetItemComponentByID(uint32_t skillID); static CDItemComponent Default; + +private: + // Caches the entry of one id from CDServer.sqlite (Default when there is none) and returns it + const CDItemComponent& LoadFromSqlite(uint32_t id); + + const FdbReader::Table* m_FdbTable = nullptr; }; diff --git a/dDatabase/CDClientDatabase/CDClientTables/CDObjectsTable.cpp b/dDatabase/CDClientDatabase/CDClientTables/CDObjectsTable.cpp index ede9dad7d..54a8fbd00 100644 --- a/dDatabase/CDClientDatabase/CDClientTables/CDObjectsTable.cpp +++ b/dDatabase/CDClientDatabase/CDClientTables/CDObjectsTable.cpp @@ -1,21 +1,30 @@ #include "CDObjectsTable.h" +#include "CDFdb.h" +#include "Logger.h" namespace { CDObjects ObjDefault; + + // Fills an entry from a CDServer.sqlite row or an fdb row (CDFdb::RowFields), which read alike + template + void ReadEntry(Row& row, CDObjects& entry) { + entry.name = row.getStringField("name", ""); + UNUSED_COLUMN(entry.placeable = row.getIntField("placeable", -1);) + entry.type = row.getStringField("type", ""); + UNUSED_COLUMN(entry.description = row.getStringField("description", "");) + UNUSED_COLUMN(entry.localize = row.getIntField("localize", -1);) + UNUSED_COLUMN(entry.npcTemplateID = row.getIntField("npcTemplateID", -1);) + UNUSED_COLUMN(entry.displayName = row.getStringField("displayName", "");) + entry.interactionDistance = row.getFloatField("interactionDistance", -1.0f); + UNUSED_COLUMN(entry.nametag = row.getIntField("nametag", -1);) + UNUSED_COLUMN(entry._internalNotes = row.getStringField("_internalNotes", "");) + UNUSED_COLUMN(entry.locStatus = row.getIntField("locStatus", -1);) + UNUSED_COLUMN(entry.gate_version = row.getStringField("gate_version", "");) + UNUSED_COLUMN(entry.HQ_valid = row.getIntField("HQ_valid", -1);) + } }; void CDObjectsTable::LoadValuesFromDatabase() { - // First, get the size of the table - uint32_t size = 0; - auto tableSize = CDClientDatabase::ExecuteQuery("SELECT COUNT(*) FROM Objects"); - while (!tableSize.eof()) { - size = tableSize.getIntField(0, 0); - - tableSize.nextRow(); - } - - tableSize.finalize(); - // Now get the data auto tableData = CDClientDatabase::ExecuteQuery("SELECT * FROM Objects"); auto& entries = GetEntriesMutable(); @@ -24,19 +33,7 @@ void CDObjectsTable::LoadValuesFromDatabase() { auto& entry = entries[lot]; entry.id = lot; - entry.name = tableData.getStringField("name", ""); - UNUSED_COLUMN(entry.placeable = tableData.getIntField("placeable", -1);) - entry.type = tableData.getStringField("type", ""); - UNUSED_COLUMN(entry.description = tableData.getStringField("description", "");) - UNUSED_COLUMN(entry.localize = tableData.getIntField("localize", -1);) - UNUSED_COLUMN(entry.npcTemplateID = tableData.getIntField("npcTemplateID", -1);) - UNUSED_COLUMN(entry.displayName = tableData.getStringField("displayName", "");) - entry.interactionDistance = tableData.getFloatField("interactionDistance", -1.0f); - UNUSED_COLUMN(entry.nametag = tableData.getIntField("nametag", -1);) - UNUSED_COLUMN(entry._internalNotes = tableData.getStringField("_internalNotes", "");) - UNUSED_COLUMN(entry.locStatus = tableData.getIntField("locStatus", -1);) - UNUSED_COLUMN(entry.gate_version = tableData.getStringField("gate_version", "");) - UNUSED_COLUMN(entry.HQ_valid = tableData.getIntField("HQ_valid", -1);) + ReadEntry(tableData, entry); tableData.nextRow(); } @@ -44,13 +41,24 @@ void CDObjectsTable::LoadValuesFromDatabase() { ObjDefault.id = 0; } -const CDObjects& CDObjectsTable::GetByID(const uint32_t lot) { - auto& entries = GetEntriesMutable(); - const auto& it = entries.find(lot); - if (it != entries.end()) { - return it->second; - } +bool CDObjectsTable::LoadFromFdb() { + m_FdbTable = nullptr; + const auto* table = CDFdb::GetTable("Objects"); + if (!table) return false; + const auto changed = CDFdb::FindChangedKeys(*table); + if (!changed) return false; + + // Ids whose rows CDServer.sqlite changes are read from it once and kept; everything else comes from the fdb + for (const auto id : *changed) LoadFromSqlite(static_cast(id)); + LOG("Objects: reading from the fdb, %zu ids differ in CDServer.sqlite and are kept in memory", changed->size()); + + m_FdbTable = table; + return true; +} + +const CDObjects& CDObjectsTable::LoadFromSqlite(const uint32_t lot) { + auto& entries = GetEntriesMutable(); auto query = CDClientDatabase::CreatePreppedStmt("SELECT * FROM Objects WHERE id = ?;"); query.bind(1, static_cast(lot)); @@ -62,33 +70,42 @@ const CDObjects& CDObjectsTable::GetByID(const uint32_t lot) { // Now get the data while (!tableData.eof()) { - const uint32_t lot = tableData.getIntField("id", 0); + const uint32_t rowLot = tableData.getIntField("id", 0); - auto& entry = entries[lot]; - entry.id = lot; - entry.name = tableData.getStringField("name", ""); - UNUSED(entry.placeable = tableData.getIntField("placeable", -1)); - entry.type = tableData.getStringField("type", ""); - UNUSED(entry.description = tableData.getStringField(4, "")); - UNUSED(entry.localize = tableData.getIntField("localize", -1)); - UNUSED(entry.npcTemplateID = tableData.getIntField("npcTemplateID", -1)); - UNUSED(entry.displayName = tableData.getStringField("displayName", "")); - entry.interactionDistance = tableData.getFloatField("interactionDistance", -1.0f); - UNUSED(entry.nametag = tableData.getIntField("nametag", -1)); - UNUSED(entry._internalNotes = tableData.getStringField("_internalNotes", "")); - UNUSED(entry.locStatus = tableData.getIntField("locStatus", -1)); - UNUSED(entry.gate_version = tableData.getStringField("gate_version", "")); - UNUSED(entry.HQ_valid = tableData.getIntField("HQ_valid", -1)); + auto& entry = entries[rowLot]; + entry.id = rowLot; + ReadEntry(tableData, entry); tableData.nextRow(); } tableData.finalize(); - const auto& it2 = entries.find(lot); - if (it2 != entries.end()) { - return it2->second; + const auto& it = entries.find(lot); + return it != entries.end() ? it->second : ObjDefault; +} + +const CDObjects& CDObjectsTable::GetByID(const uint32_t lot) { + auto& entries = GetEntriesMutable(); + const auto& it = entries.find(lot); + if (it != entries.end()) { + return it->second; } - return ObjDefault; + if (!m_FdbTable) return LoadFromSqlite(lot); + + // Only the objects asked for are kept in memory; the last row of an id wins, as in the SQLite path + std::optional found; + m_FdbTable->ForEachRowWithKey(static_cast(lot), [&](const FdbReader::Row& row) { + if (!found) found.emplace(); + found->id = lot; + const CDFdb::RowFields fields(*m_FdbTable, row); + ReadEntry(fields, *found); + }); + if (!found) { + entries.emplace(lot, ObjDefault); + return ObjDefault; + } + + return entries.emplace(lot, std::move(*found)).first->second; } diff --git a/dDatabase/CDClientDatabase/CDClientTables/CDObjectsTable.h b/dDatabase/CDClientDatabase/CDClientTables/CDObjectsTable.h index 13bb90fab..3c647c8c9 100644 --- a/dDatabase/CDClientDatabase/CDClientTables/CDObjectsTable.h +++ b/dDatabase/CDClientDatabase/CDClientTables/CDObjectsTable.h @@ -5,6 +5,8 @@ #include +#include "FdbReader.h" + struct CDObjects { uint32_t id; //!< The LOT of the object std::string name; //!< The internal name of the object @@ -25,7 +27,17 @@ struct CDObjects { class CDObjectsTable : public CDTable> { public: void LoadValuesFromDatabase(); + + // Reads rows from the client's fdb from now on, if it is open; false keeps the table on CDServer.sqlite + bool LoadFromFdb(); + // Gets an entry by ID const CDObjects& GetByID(const uint32_t lot); + +private: + // Caches the entry of one id from CDServer.sqlite (the default entry when there is none) and returns it + const CDObjects& LoadFromSqlite(const uint32_t lot); + + const FdbReader::Table* m_FdbTable = nullptr; }; diff --git a/dMasterServer/MasterServer.cpp b/dMasterServer/MasterServer.cpp index cdf4d2aa4..a6ccd2d22 100644 --- a/dMasterServer/MasterServer.cpp +++ b/dMasterServer/MasterServer.cpp @@ -494,7 +494,7 @@ int main(int argc, char** argv) { //Get CDClient initial information try { - CDClientManager::LoadValuesFromDatabase(); + CDClientManager::LoadValuesFromDatabase(Game::assetManager->GetResPath() / "cdclient.fdb"); } catch (CppSQLite3Exception& e) { LOG("Failed to initialize CDServer SQLite Database"); LOG("May be caused by corrupted file: %s", (Game::assetManager->GetResPath() / "CDServer.sqlite").string().c_str()); diff --git a/dWorldServer/WorldServer.cpp b/dWorldServer/WorldServer.cpp index 592b180b8..7712cf339 100644 --- a/dWorldServer/WorldServer.cpp +++ b/dWorldServer/WorldServer.cpp @@ -275,7 +275,8 @@ int main(int argc, char** argv) { return EXIT_FAILURE; } - CDClientManager::LoadValuesFromDatabase(); + // The client's fdb, read in place and shared between all server processes, when there is one + CDClientManager::LoadValuesFromDatabase(Game::assetManager->GetResPath() / "cdclient.fdb"); Diagnostics::SetProduceMemoryDump(Game::config->GetValue("generate_dump") == "1"); diff --git a/tests/dGameTests/CDFdbTests.cpp b/tests/dGameTests/CDFdbTests.cpp new file mode 100644 index 000000000..cedbe522f --- /dev/null +++ b/tests/dGameTests/CDFdbTests.cpp @@ -0,0 +1,304 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include "CDClientDatabase.h" +#include "CDClientManager.h" +#include "CDComponentsRegistryTable.h" +#include "CDFdb.h" +#include "CDItemComponentTable.h" +#include "CDObjectsTable.h" +#include "Game.h" +#include "Logger.h" +#include "eReplicaComponentType.h" +#include "../FdbTestWriter.h" + +using FdbTestWriter::Value; + +namespace { + void ClearTables() { + CDClientManager::GetEntriesMutable().clear(); + CDClientManager::GetEntriesMutable().clear(); + CDClientManager::GetEntriesMutable().clear(); + } + + int32_t Component(uint32_t id, int32_t type, int32_t defaultValue = -99) { + return CDComponentsRegistryTable::Instance().GetByIDAndType(id, static_cast(type), defaultValue); + } + + class CDFdbTestBase : public ::testing::Test { + protected: + void SetUp() override { + m_Logger = std::make_unique("./testing.log", false, false); + Game::logger = m_Logger.get(); + ClearTables(); + } + + void TearDown() override { + // Let go of the fdb before its file is removed; the tables stop using it on their next load + CDFdb::Close(); + CDComponentsRegistryTable::Instance().LoadFromFdb(); + CDItemComponentTable::Instance().LoadFromFdb(); + CDObjectsTable::Instance().LoadFromFdb(); + ClearTables(); + Game::logger = nullptr; + } + + std::unique_ptr m_Logger; + }; + + // A ComponentsRegistry in both files, where CDServer.sqlite changes, drops and adds rows the way migrations do + class CDFdbOverlayTest : public CDFdbTestBase { + protected: + void SetUp() override { + CDFdbTestBase::SetUp(); + m_Dir = std::filesystem::temp_directory_path() / ("dlu_cdfdb_" + std::string(::testing::UnitTest::GetInstance()->current_test_info()->name())); + std::filesystem::create_directories(m_Dir); + + FdbTestWriter::Table registry; + registry.name = "ComponentsRegistry"; + registry.columns = { { "id", eSqliteDataType::INT32 }, { "component_type", eSqliteDataType::INT32 }, { "component_id", eSqliteDataType::INT32 } }; + registry.bucketCount = 4; + // 1, 5 and 9 share bucket 1 + registry.rows = { + { Value::Int(1), Value::Int(1), Value::Int(10) }, + { Value::Int(1), Value::Int(2), Value::Int(20) }, + { Value::Int(5), Value::Int(1), Value::Int(50) }, + { Value::Int(9), Value::Int(1), Value::Int(90) }, + { Value::Int(9), Value::Int(2), Value::Int(91) }, + { Value::Int(2), Value::Int(1), Value::Int(200) }, + // A repeated (id, type): the last row wins + { Value::Int(2), Value::Int(1), Value::Int(201) }, + }; + FdbTestWriter::WriteFile(m_Dir / "cdclient.fdb", FdbTestWriter::Write({ registry })); + + std::filesystem::remove(m_Dir / "CDServer.sqlite"); + CDClientDatabase::Connect((m_Dir / "CDServer.sqlite").string()); + CDClientDatabase::ExecuteDML("CREATE TABLE ComponentsRegistry ('id' int32, 'component_type' int32, 'component_id' int32);"); + CDClientDatabase::ExecuteDML( + "INSERT INTO ComponentsRegistry VALUES (1, 1, 10), (1, 2, 20), (5, 1, 55), (2, 1, 200), (2, 1, 201), (13, 1, 130);"); + } + + void TearDown() override { + CDFdbTestBase::TearDown(); + std::error_code error; + std::filesystem::remove_all(m_Dir, error); + } + + void ExpectRegistryAnswers() { + EXPECT_EQ(Component(1, 1), 10); + EXPECT_EQ(Component(1, 2), 20); + EXPECT_EQ(Component(1, 3), -99); + // Changed by CDServer.sqlite + EXPECT_EQ(Component(5, 1), 55); + // Dropped by CDServer.sqlite + EXPECT_EQ(Component(9, 1), -99); + EXPECT_EQ(Component(9, 2), -99); + // Added by CDServer.sqlite + EXPECT_EQ(Component(13, 1), 130); + EXPECT_EQ(Component(2, 1), 201); + EXPECT_EQ(Component(3, 1), -99); + } + + std::filesystem::path m_Dir; + }; +} + +TEST_F(CDFdbOverlayTest, FindsTheKeysSqliteChanges) { + ASSERT_TRUE(CDFdb::Open(m_Dir / "cdclient.fdb")); + const auto* table = CDFdb::GetTable("ComponentsRegistry"); + ASSERT_NE(table, nullptr); + const auto changed = CDFdb::FindChangedKeys(*table); + ASSERT_TRUE(changed.has_value()); + EXPECT_EQ(std::set(changed->begin(), changed->end()), (std::set{ 5, 9, 13 })); + + // Tables that aren't in both files, or whose columns differ, aren't read from the fdb + EXPECT_EQ(CDFdb::GetTable("Objects"), nullptr); +} + +TEST_F(CDFdbOverlayTest, ReadsFromTheFdbWithSqliteChangesOnTop) { + ASSERT_TRUE(CDFdb::Open(m_Dir / "cdclient.fdb")); + ASSERT_TRUE(CDComponentsRegistryTable::Instance().LoadFromFdb()); + + // Only the changed ids are cached (an id marker plus its rows), not the table + EXPECT_EQ(CDClientManager::GetEntriesMutable().size(), 5u); + + ExpectRegistryAnswers(); +} + +TEST_F(CDFdbOverlayTest, WithoutAnFdbReadsSqlite) { + EXPECT_FALSE(CDFdb::Open(m_Dir / "missing.fdb")); + EXPECT_EQ(CDFdb::Get(), nullptr); + EXPECT_FALSE(CDComponentsRegistryTable::Instance().LoadFromFdb()); + + // Lazy SQLite lookups, as before + ExpectRegistryAnswers(); + + // And the whole table in memory, as the server loads it without an fdb + ClearTables(); + CDComponentsRegistryTable::Instance().LoadValuesFromDatabase(); + ExpectRegistryAnswers(); +} + +TEST_F(CDFdbOverlayTest, UnmappedFdbReadsTheSame) { + ASSERT_TRUE(CDFdb::Open(m_Dir / "cdclient.fdb", false)); + EXPECT_FALSE(CDFdb::Get()->IsMapped()); + ASSERT_TRUE(CDComponentsRegistryTable::Instance().LoadFromFdb()); + ExpectRegistryAnswers(); +} + +namespace { + std::vector DistinctIds(const std::string& table) { + std::vector ids; + auto query = CDClientDatabase::ExecuteQuery("SELECT DISTINCT id FROM " + table + ";"); + while (!query.eof()) { + ids.push_back(query.getInt64Field(0)); + query.nextRow(); + } + return ids; + } + + bool SameFloat(float a, float b) { + return a == b || (std::isnan(a) && std::isnan(b)); + } + + // Every id of the table in either file, plus some that are in neither + std::vector AllIds(const std::string& name) { + std::set ids{ 0, 0xFFFFFFFF, 99999999 }; + for (const auto id : DistinctIds(name)) ids.insert(static_cast(id)); + if (const auto* table = CDFdb::Get() ? CDFdb::Get()->GetTable(name) : nullptr) { + table->ForEachRow([&](const FdbReader::Row& row) { + if (const auto key = FdbReader::Table::KeyOf(row)) ids.insert(static_cast(*key)); + }); + } + return { ids.begin(), ids.end() }; + } + + struct Snapshot { + std::vector components; + std::vector items; + std::vector objects; + }; + + Snapshot TakeSnapshot(const std::vector& registryIds, const std::vector& types, + const std::vector& itemIds, const std::vector& objectIds) { + Snapshot snapshot; + for (const auto id : registryIds) { + for (const auto type : types) snapshot.components.push_back(Component(id, type, -12345)); + } + for (const auto id : itemIds) snapshot.items.push_back(CDItemComponentTable::Instance().GetItemComponentByID(id)); + for (const auto id : objectIds) snapshot.objects.push_back(CDObjectsTable::Instance().GetByID(id)); + return snapshot; + } +} + +// Every row of the switched tables through the fdb against CDServer.sqlite, when both are around: +// DLU_CLIENT_RES is the client's res folder (with cdclient.fdb), DLU_CDSERVER_SQLITE the converted +// database (default: the source tree's build/resServer/CDServer.sqlite). Opened read-only. +TEST_F(CDFdbTestBase, RealClientParity) { + const char* res = std::getenv("DLU_CLIENT_RES"); + if (!res) GTEST_SKIP() << "Set DLU_CLIENT_RES to a client res folder to compare against its cdclient.fdb"; + const auto fdbPath = std::filesystem::path(res) / "cdclient.fdb"; + if (!std::filesystem::exists(fdbPath)) GTEST_SKIP() << "No " << fdbPath.string(); + + std::string sqlitePath = std::string(PROJECT_SOURCE_DIR) + "/build/resServer/CDServer.sqlite"; + if (const char* env = std::getenv("DLU_CDSERVER_SQLITE")) sqlitePath = env; + if (!std::filesystem::exists(sqlitePath)) GTEST_SKIP() << "No CDServer.sqlite at " << sqlitePath; + + CDClientDatabase::Connect(sqlitePath); + CDClientDatabase::ExecuteDML("PRAGMA query_only = ON;"); // never write to the CDClient + + ASSERT_TRUE(CDFdb::Open(fdbPath)); + EXPECT_TRUE(CDFdb::Get()->IsMapped()); + + const auto registryIds = AllIds("ComponentsRegistry"); + const auto itemIds = AllIds("ItemComponent"); + const auto objectIds = AllIds("Objects"); + std::vector types{ 999 }; + for (const auto type : [] { + std::vector found; + auto query = CDClientDatabase::ExecuteQuery("SELECT DISTINCT component_type FROM ComponentsRegistry;"); + while (!query.eof()) { + found.push_back(query.getInt64Field(0)); + query.nextRow(); + } + return found; + }()) types.push_back(static_cast(type)); + + for (const auto* name : { "ComponentsRegistry", "ItemComponent", "Objects" }) { + const auto* table = CDFdb::GetTable(name); + ASSERT_NE(table, nullptr) << name; + const auto changed = CDFdb::FindChangedKeys(*table); + ASSERT_TRUE(changed.has_value()) << name; + std::cout << name << ": " << changed->size() << " ids differ between the fdb and CDServer.sqlite" << std::endl; + } + + // Through the fdb + ASSERT_TRUE(CDComponentsRegistryTable::Instance().LoadFromFdb()); + ASSERT_TRUE(CDItemComponentTable::Instance().LoadFromFdb()); + ASSERT_TRUE(CDObjectsTable::Instance().LoadFromFdb()); + const auto fromFdb = TakeSnapshot(registryIds, types, itemIds, objectIds); + + // Through CDServer.sqlite, loaded the way the server does without an fdb + CDFdb::Close(); + ClearTables(); + EXPECT_FALSE(CDComponentsRegistryTable::Instance().LoadFromFdb()); + EXPECT_FALSE(CDItemComponentTable::Instance().LoadFromFdb()); + EXPECT_FALSE(CDObjectsTable::Instance().LoadFromFdb()); + CDComponentsRegistryTable::Instance().LoadValuesFromDatabase(); + CDItemComponentTable::Instance().LoadValuesFromDatabase(); + CDObjectsTable::Instance().LoadValuesFromDatabase(); + const auto fromSqlite = TakeSnapshot(registryIds, types, itemIds, objectIds); + + ASSERT_EQ(fromFdb.components.size(), fromSqlite.components.size()); + uint32_t componentMismatches = 0; + for (size_t i = 0; i < fromFdb.components.size(); i++) { + if (fromFdb.components[i] != fromSqlite.components[i]) { + if (componentMismatches++ < 20) { + ADD_FAILURE() << "ComponentsRegistry id " << registryIds[i / types.size()] << " type " << types[i % types.size()] + << ": fdb " << fromFdb.components[i] << " sqlite " << fromSqlite.components[i]; + } + } + } + EXPECT_EQ(componentMismatches, 0u); + + ASSERT_EQ(fromFdb.items.size(), fromSqlite.items.size()); + uint32_t itemMismatches = 0; + for (size_t i = 0; i < fromFdb.items.size(); i++) { + const auto& a = fromFdb.items[i]; + const auto& b = fromSqlite.items[i]; + const bool same = a.id == b.id && a.equipLocation == b.equipLocation && a.baseValue == b.baseValue && + a.isKitPiece == b.isKitPiece && a.rarity == b.rarity && a.itemType == b.itemType && a.itemInfo == b.itemInfo && + a.inLootTable == b.inLootTable && a.inVendor == b.inVendor && a.isUnique == b.isUnique && a.isBOP == b.isBOP && + a.isBOE == b.isBOE && a.reqFlagID == b.reqFlagID && a.reqSpecialtyID == b.reqSpecialtyID && + a.reqSpecRank == b.reqSpecRank && a.reqAchievementID == b.reqAchievementID && a.stackSize == b.stackSize && + a.color1 == b.color1 && a.decal == b.decal && a.offsetGroupID == b.offsetGroupID && a.buildTypes == b.buildTypes && + a.reqPrecondition == b.reqPrecondition && a.animationFlag == b.animationFlag && a.equipEffects == b.equipEffects && + a.readyForQA == b.readyForQA && a.itemRating == b.itemRating && a.isTwoHanded == b.isTwoHanded && + a.minNumRequired == b.minNumRequired && a.delResIndex == b.delResIndex && a.currencyLOT == b.currencyLOT && + a.altCurrencyCost == b.altCurrencyCost && a.subItems == b.subItems && a.noEquipAnimation == b.noEquipAnimation && + a.commendationLOT == b.commendationLOT && a.commendationCost == b.commendationCost && + a.currencyCosts == b.currencyCosts && a.locStatus == b.locStatus && a.forgeType == b.forgeType && + SameFloat(a.SellMultiplier, b.SellMultiplier); + if (!same && itemMismatches++ < 20) ADD_FAILURE() << "ItemComponent id " << itemIds[i] << " differs"; + } + EXPECT_EQ(itemMismatches, 0u); + + ASSERT_EQ(fromFdb.objects.size(), fromSqlite.objects.size()); + uint32_t objectMismatches = 0; + for (size_t i = 0; i < fromFdb.objects.size(); i++) { + const auto& a = fromFdb.objects[i]; + const auto& b = fromSqlite.objects[i]; + const bool same = a.id == b.id && a.name == b.name && a.type == b.type && SameFloat(a.interactionDistance, b.interactionDistance); + if (!same && objectMismatches++ < 20) ADD_FAILURE() << "Objects id " << objectIds[i] << " differs: fdb '" << a.name << "' sqlite '" << b.name << "'"; + } + EXPECT_EQ(objectMismatches, 0u); + + std::cout << "Compared " << registryIds.size() << " ComponentsRegistry ids x " << types.size() << " types, " + << itemIds.size() << " ItemComponent ids, " << objectIds.size() << " Objects ids" << std::endl; +} diff --git a/tests/dGameTests/CMakeLists.txt b/tests/dGameTests/CMakeLists.txt index 4219a6b87..caff760c5 100644 --- a/tests/dGameTests/CMakeLists.txt +++ b/tests/dGameTests/CMakeLists.txt @@ -27,6 +27,7 @@ set(DGAMETEST_SOURCES "PreconditionItemCostTests.cpp" "SpiderQueenTests.cpp" "TacArcTests.cpp" + "CDFdbTests.cpp" ) add_subdirectory(dComponentsTests)