mirror of
https://github.com/DarkflameUniverse/DarkflameServer.git
synced 2026-10-02 10:53:44 +00:00
perf(cdclient): read ComponentsRegistry, ItemComponent and Objects from the fdb
The world and master servers pass the client's res/cdclient.fdb to CDClientManager. When it opens, these three tables, all looked up by their first column, find rows through the fdb's buckets instead of each process caching the whole table: ComponentsRegistry keeps nothing, ItemComponent and Objects keep only the entries asked for (their API returns references). Ids whose rows CDServer.sqlite changes are loaded from SQLite at startup and win. Without an fdb (or with one whose columns don't match) the tables load from CDServer.sqlite as before. ItemComponent and Objects now fill entries from one template for both sources instead of copies of the same field list. Tests cover the SQLite changes on top of the fdb, the no-fdb and unmapped paths, and, when DLU_CLIENT_RES points at a client, every id of the three tables through the fdb against CDServer.sqlite. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,4 +1,6 @@
|
||||
#include "CDClientManager.h"
|
||||
#include "CDFdb.h"
|
||||
#include "Logger.h"
|
||||
#include "CDActivityRewardsTable.h"
|
||||
#include "CDAnimationsTable.h"
|
||||
#include "CDBehaviorParameterTable.h"
|
||||
@@ -111,11 +113,20 @@ DEFINE_TABLE_STORAGE(CDSkillBehaviorTable);
|
||||
DEFINE_TABLE_STORAGE(CDTamingBuildPuzzleTable);
|
||||
DEFINE_TABLE_STORAGE(CDVendorComponentTable);
|
||||
|
||||
void CDClientManager::LoadValuesFromDatabase() {
|
||||
void CDClientManager::LoadValuesFromDatabase(const std::filesystem::path& fdbPath) {
|
||||
if (!CDClientDatabase::isConnected) {
|
||||
throw std::runtime_error{ "CDClientDatabase is not connected!" };
|
||||
}
|
||||
|
||||
if (!fdbPath.empty()) {
|
||||
if (CDFdb::Open(fdbPath)) {
|
||||
LOG("Reading CDClient rows by id from %s (%s)", fdbPath.string().c_str(),
|
||||
CDFdb::Get()->IsMapped() ? "mapped, shared between processes" : "could not map it, read into memory");
|
||||
} else {
|
||||
LOG("Could not open %s, reading CDClient from CDServer.sqlite only", fdbPath.string().c_str());
|
||||
}
|
||||
}
|
||||
|
||||
CDActivityRewardsTable::Instance().LoadValuesFromDatabase();
|
||||
CDActivitiesTable::Instance().LoadValuesFromDatabase();
|
||||
CDCLIENT_DONT_CACHE_TABLE(CDAnimationsTable::Instance().LoadValuesFromDatabase());
|
||||
@@ -123,10 +134,11 @@ void CDClientManager::LoadValuesFromDatabase() {
|
||||
CDBehaviorTemplateTable::Instance().LoadValuesFromDatabase();
|
||||
CDBrickIDTableTable::Instance().LoadValuesFromDatabase();
|
||||
CDCollectibleComponentTable::Instance().LoadValuesFromDatabase();
|
||||
// Always in memory: every entity and every inventory item asks for its components and item data, and looking up an
|
||||
// ComponentsRegistry, ItemComponent and Objects read rows from the fdb when it is open. Without it, the first two are
|
||||
// always in memory: every entity and every inventory item asks for its components and item data, and looking up an
|
||||
// id not seen yet in the unindexed CDClient scans the whole table (a character holding 3000 different items took a
|
||||
// minute to load)
|
||||
CDComponentsRegistryTable::Instance().LoadValuesFromDatabase();
|
||||
if (!CDComponentsRegistryTable::Instance().LoadFromFdb()) CDComponentsRegistryTable::Instance().LoadValuesFromDatabase();
|
||||
CDCurrencyTableTable::Instance().LoadValuesFromDatabase();
|
||||
CDDeletionRestrictionsTable::Instance().LoadValuesFromDatabase();
|
||||
CDDestructibleComponentTable::Instance().LoadValuesFromDatabase();
|
||||
@@ -134,7 +146,7 @@ void CDClientManager::LoadValuesFromDatabase() {
|
||||
CDEventGatingTable::Instance().LoadValuesFromDatabase();
|
||||
CDFeatureGatingTable::Instance().LoadValuesFromDatabase();
|
||||
CDInventoryComponentTable::Instance().LoadValuesFromDatabase();
|
||||
CDItemComponentTable::Instance().LoadValuesFromDatabase();
|
||||
if (!CDItemComponentTable::Instance().LoadFromFdb()) CDItemComponentTable::Instance().LoadValuesFromDatabase();
|
||||
CDItemSetSkillsTable::Instance().LoadValuesFromDatabase();
|
||||
CDItemSetsTable::Instance().LoadValuesFromDatabase();
|
||||
CDLevelProgressionLookupTable::Instance().LoadValuesFromDatabase();
|
||||
@@ -146,7 +158,7 @@ void CDClientManager::LoadValuesFromDatabase() {
|
||||
CDMissionsTable::Instance().LoadValuesFromDatabase();
|
||||
CDMovementAIComponentTable::Instance().LoadValuesFromDatabase();
|
||||
CDObjectSkillsTable::Instance().LoadValuesFromDatabase();
|
||||
CDCLIENT_DONT_CACHE_TABLE(CDObjectsTable::Instance().LoadValuesFromDatabase());
|
||||
if (!CDObjectsTable::Instance().LoadFromFdb()) CDCLIENT_DONT_CACHE_TABLE(CDObjectsTable::Instance().LoadValuesFromDatabase());
|
||||
CDPhysicsComponentTable::Instance().LoadValuesFromDatabase();
|
||||
CDPackageComponentTable::Instance().LoadValuesFromDatabase();
|
||||
CDPetComponentTable::Instance().LoadValuesFromDatabase();
|
||||
|
||||
@@ -3,11 +3,18 @@
|
||||
|
||||
#define UNUSED_TABLE(v)
|
||||
|
||||
#include <filesystem>
|
||||
|
||||
/**
|
||||
* Initialize the CDClient tables so they are all loaded into memory.
|
||||
*/
|
||||
namespace CDClientManager {
|
||||
void LoadValuesFromDatabase();
|
||||
/**
|
||||
* @param fdbPath The client's cdclient.fdb. When it opens, the tables looked up by their first
|
||||
* column read their rows from it (shared between processes) instead of caching them. Empty, or
|
||||
* a file that doesn't open, keeps every table on CDServer.sqlite.
|
||||
*/
|
||||
void LoadValuesFromDatabase(const std::filesystem::path& fdbPath = {});
|
||||
void LoadValuesFromDefaults();
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
#include "CDComponentsRegistryTable.h"
|
||||
#include "CDFdb.h"
|
||||
#include "Logger.h"
|
||||
#include "eReplicaComponentType.h"
|
||||
|
||||
void CDComponentsRegistryTable::LoadValuesFromDatabase() {
|
||||
@@ -20,15 +22,30 @@ void CDComponentsRegistryTable::LoadValuesFromDatabase() {
|
||||
tableData.finalize();
|
||||
}
|
||||
|
||||
int32_t CDComponentsRegistryTable::GetByIDAndType(uint32_t id, eReplicaComponentType componentType, int32_t defaultValue) {
|
||||
auto& entries = GetEntriesMutable();
|
||||
auto exists = entries.find(id);
|
||||
if (exists != entries.end()) {
|
||||
auto iter = entries.find(static_cast<uint64_t>(componentType) << 32 | static_cast<uint64_t>(id));
|
||||
return iter == entries.end() ? defaultValue : iter->second;
|
||||
}
|
||||
bool CDComponentsRegistryTable::LoadFromFdb() {
|
||||
m_FdbTable = nullptr;
|
||||
const auto* table = CDFdb::GetTable("ComponentsRegistry");
|
||||
if (!table) return false;
|
||||
|
||||
// Now get the data. Get all components of this entity so we dont do a query for each component
|
||||
m_TypeColumn = table->GetColumnIndex("component_type");
|
||||
m_ComponentIdColumn = table->GetColumnIndex("component_id");
|
||||
if (m_TypeColumn < 0 || m_ComponentIdColumn < 0) return false;
|
||||
|
||||
const auto changed = CDFdb::FindChangedKeys(*table);
|
||||
if (!changed) return false;
|
||||
|
||||
// Ids whose rows CDServer.sqlite changes are read from it once and kept; everything else comes from the fdb
|
||||
for (const auto id : *changed) LoadFromSqlite(static_cast<uint32_t>(id));
|
||||
LOG("ComponentsRegistry: reading from the fdb, %zu ids differ in CDServer.sqlite and are kept in memory", changed->size());
|
||||
|
||||
m_FdbTable = table;
|
||||
return true;
|
||||
}
|
||||
|
||||
void CDComponentsRegistryTable::LoadFromSqlite(uint32_t id) {
|
||||
auto& entries = GetEntriesMutable();
|
||||
|
||||
// Get all components of this entity so we dont do a query for each component
|
||||
auto query = CDClientDatabase::CreatePreppedStmt("SELECT * FROM ComponentsRegistry WHERE id = ?;");
|
||||
query.bind(1, static_cast<int32_t>(id));
|
||||
|
||||
@@ -46,6 +63,29 @@ int32_t CDComponentsRegistryTable::GetByIDAndType(uint32_t id, eReplicaComponent
|
||||
}
|
||||
|
||||
entries.insert_or_assign(id, 0);
|
||||
}
|
||||
|
||||
int32_t CDComponentsRegistryTable::GetByIDAndType(uint32_t id, eReplicaComponentType componentType, int32_t defaultValue) {
|
||||
auto& entries = GetEntriesMutable();
|
||||
auto exists = entries.find(id);
|
||||
if (exists != entries.end()) {
|
||||
auto iter = entries.find(static_cast<uint64_t>(componentType) << 32 | static_cast<uint64_t>(id));
|
||||
return iter == entries.end() ? defaultValue : iter->second;
|
||||
}
|
||||
|
||||
if (m_FdbTable) {
|
||||
// The last matching row wins, as when the rows are loaded into the map in file order
|
||||
int32_t result = defaultValue;
|
||||
const auto type = static_cast<int32_t>(componentType);
|
||||
m_FdbTable->ForEachRowWithKey(static_cast<int32_t>(id), [&](const FdbReader::Row& row) {
|
||||
if (row.GetInt(static_cast<uint32_t>(m_TypeColumn), 0) == type) {
|
||||
result = row.GetInt(static_cast<uint32_t>(m_ComponentIdColumn), -1);
|
||||
}
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
LoadFromSqlite(id);
|
||||
|
||||
auto iter = entries.find(static_cast<uint64_t>(componentType) << 32 | static_cast<uint64_t>(id));
|
||||
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
|
||||
#include <unordered_map>
|
||||
|
||||
#include "FdbReader.h"
|
||||
|
||||
enum class eReplicaComponentType : uint32_t;
|
||||
struct CDComponentsRegistry {
|
||||
uint32_t id; //!< The LOT is used as the ID
|
||||
@@ -16,5 +18,17 @@ struct CDComponentsRegistry {
|
||||
class CDComponentsRegistryTable : public CDTable<CDComponentsRegistryTable, std::unordered_map<uint64_t, uint32_t>> {
|
||||
public:
|
||||
void LoadValuesFromDatabase();
|
||||
|
||||
// Reads rows from the client's fdb from now on, if it is open; false keeps the table on CDServer.sqlite
|
||||
bool LoadFromFdb();
|
||||
|
||||
int32_t GetByIDAndType(uint32_t id, eReplicaComponentType componentType, int32_t defaultValue = 0);
|
||||
|
||||
private:
|
||||
// Caches every row of one id from CDServer.sqlite
|
||||
void LoadFromSqlite(uint32_t id);
|
||||
|
||||
const FdbReader::Table* m_FdbTable = nullptr;
|
||||
int32_t m_TypeColumn = -1;
|
||||
int32_t m_ComponentIdColumn = -1;
|
||||
};
|
||||
|
||||
@@ -1,68 +1,67 @@
|
||||
#include "CDItemComponentTable.h"
|
||||
#include "GeneralUtils.h"
|
||||
#include "CDFdb.h"
|
||||
#include "Logger.h"
|
||||
|
||||
CDItemComponent CDItemComponentTable::Default = {};
|
||||
|
||||
void CDItemComponentTable::LoadValuesFromDatabase() {
|
||||
// First, get the size of the table
|
||||
uint32_t size = 0;
|
||||
auto tableSize = CDClientDatabase::ExecuteQuery("SELECT COUNT(*) FROM ItemComponent");
|
||||
while (!tableSize.eof()) {
|
||||
size = tableSize.getIntField(0, 0);
|
||||
|
||||
tableSize.nextRow();
|
||||
namespace {
|
||||
// Fills an entry from a CDServer.sqlite row or an fdb row (CDFdb::RowFields), which read alike
|
||||
template<typename Row>
|
||||
CDItemComponent ReadEntry(Row& row) {
|
||||
CDItemComponent entry;
|
||||
entry.id = row.getIntField("id", -1);
|
||||
entry.equipLocation = row.getStringField("equipLocation", "");
|
||||
entry.baseValue = row.getIntField("baseValue", -1);
|
||||
entry.isKitPiece = row.getIntField("isKitPiece", -1) == 1 ? true : false;
|
||||
entry.rarity = row.getIntField("rarity", 0);
|
||||
entry.itemType = row.getIntField("itemType", -1);
|
||||
entry.itemInfo = row.getInt64Field("itemInfo", -1);
|
||||
entry.inLootTable = row.getIntField("inLootTable", -1) == 1 ? true : false;
|
||||
entry.inVendor = row.getIntField("inVendor", -1) == 1 ? true : false;
|
||||
entry.isUnique = row.getIntField("isUnique", -1) == 1 ? true : false;
|
||||
entry.isBOP = row.getIntField("isBOP", -1) == 1 ? true : false;
|
||||
entry.isBOE = row.getIntField("isBOE", -1) == 1 ? true : false;
|
||||
entry.reqFlagID = row.getIntField("reqFlagID", -1);
|
||||
entry.reqSpecialtyID = row.getIntField("reqSpecialtyID", -1);
|
||||
entry.reqSpecRank = row.getIntField("reqSpecRank", -1);
|
||||
entry.reqAchievementID = row.getIntField("reqAchievementID", -1);
|
||||
entry.stackSize = row.getIntField("stackSize", -1);
|
||||
entry.color1 = row.getIntField("color1", -1);
|
||||
entry.decal = row.getIntField("decal", -1);
|
||||
entry.offsetGroupID = row.getIntField("offsetGroupID", -1);
|
||||
entry.buildTypes = row.getIntField("buildTypes", -1);
|
||||
entry.reqPrecondition = row.getStringField("reqPrecondition", "");
|
||||
entry.animationFlag = row.getIntField("animationFlag", 0);
|
||||
entry.equipEffects = row.getIntField("equipEffects", -1);
|
||||
entry.readyForQA = row.getIntField("readyForQA", -1) == 1 ? true : false;
|
||||
entry.itemRating = row.getIntField("itemRating", -1);
|
||||
entry.isTwoHanded = row.getIntField("isTwoHanded", -1) == 1 ? true : false;
|
||||
entry.minNumRequired = row.getIntField("minNumRequired", -1);
|
||||
entry.delResIndex = row.getIntField("delResIndex", -1);
|
||||
entry.currencyLOT = row.getIntField("currencyLOT", -1);
|
||||
entry.altCurrencyCost = row.getIntField("altCurrencyCost", -1);
|
||||
entry.subItems = row.getStringField("subItems", "");
|
||||
UNUSED_COLUMN(entry.audioEventUse = row.getStringField("audioEventUse", ""));
|
||||
entry.noEquipAnimation = row.getIntField("noEquipAnimation", -1) == 1 ? true : false;
|
||||
entry.commendationLOT = row.getIntField("commendationLOT", -1);
|
||||
entry.commendationCost = row.getIntField("commendationCost", -1);
|
||||
UNUSED_COLUMN(entry.audioEquipMetaEventSet = row.getStringField("audioEquipMetaEventSet", ""));
|
||||
entry.currencyCosts = row.getStringField("currencyCosts", "");
|
||||
UNUSED_COLUMN(entry.ingredientInfo = row.getStringField("ingredientInfo", ""));
|
||||
entry.locStatus = row.getIntField("locStatus", -1);
|
||||
entry.forgeType = row.getIntField("forgeType", -1);
|
||||
entry.SellMultiplier = row.getFloatField("SellMultiplier", -1.0f);
|
||||
return entry;
|
||||
}
|
||||
}
|
||||
|
||||
tableSize.finalize();
|
||||
|
||||
void CDItemComponentTable::LoadValuesFromDatabase() {
|
||||
// Now get the data
|
||||
auto tableData = CDClientDatabase::ExecuteQuery("SELECT * FROM ItemComponent");
|
||||
auto& entries = GetEntriesMutable();
|
||||
while (!tableData.eof()) {
|
||||
CDItemComponent entry;
|
||||
entry.id = tableData.getIntField("id", -1);
|
||||
entry.equipLocation = tableData.getStringField("equipLocation", "");
|
||||
entry.baseValue = tableData.getIntField("baseValue", -1);
|
||||
entry.isKitPiece = tableData.getIntField("isKitPiece", -1) == 1 ? true : false;
|
||||
entry.rarity = tableData.getIntField("rarity", 0);
|
||||
entry.itemType = tableData.getIntField("itemType", -1);
|
||||
entry.itemInfo = tableData.getInt64Field("itemInfo", -1);
|
||||
entry.inLootTable = tableData.getIntField("inLootTable", -1) == 1 ? true : false;
|
||||
entry.inVendor = tableData.getIntField("inVendor", -1) == 1 ? true : false;
|
||||
entry.isUnique = tableData.getIntField("isUnique", -1) == 1 ? true : false;
|
||||
entry.isBOP = tableData.getIntField("isBOP", -1) == 1 ? true : false;
|
||||
entry.isBOE = tableData.getIntField("isBOE", -1) == 1 ? true : false;
|
||||
entry.reqFlagID = tableData.getIntField("reqFlagID", -1);
|
||||
entry.reqSpecialtyID = tableData.getIntField("reqSpecialtyID", -1);
|
||||
entry.reqSpecRank = tableData.getIntField("reqSpecRank", -1);
|
||||
entry.reqAchievementID = tableData.getIntField("reqAchievementID", -1);
|
||||
entry.stackSize = tableData.getIntField("stackSize", -1);
|
||||
entry.color1 = tableData.getIntField("color1", -1);
|
||||
entry.decal = tableData.getIntField("decal", -1);
|
||||
entry.offsetGroupID = tableData.getIntField("offsetGroupID", -1);
|
||||
entry.buildTypes = tableData.getIntField("buildTypes", -1);
|
||||
entry.reqPrecondition = tableData.getStringField("reqPrecondition", "");
|
||||
entry.animationFlag = tableData.getIntField("animationFlag", 0);
|
||||
entry.equipEffects = tableData.getIntField("equipEffects", -1);
|
||||
entry.readyForQA = tableData.getIntField("readyForQA", -1) == 1 ? true : false;
|
||||
entry.itemRating = tableData.getIntField("itemRating", -1);
|
||||
entry.isTwoHanded = tableData.getIntField("isTwoHanded", -1) == 1 ? true : false;
|
||||
entry.minNumRequired = tableData.getIntField("minNumRequired", -1);
|
||||
entry.delResIndex = tableData.getIntField("delResIndex", -1);
|
||||
entry.currencyLOT = tableData.getIntField("currencyLOT", -1);
|
||||
entry.altCurrencyCost = tableData.getIntField("altCurrencyCost", -1);
|
||||
entry.subItems = tableData.getStringField("subItems", "");
|
||||
UNUSED_COLUMN(entry.audioEventUse = tableData.getStringField("audioEventUse", ""));
|
||||
entry.noEquipAnimation = tableData.getIntField("noEquipAnimation", -1) == 1 ? true : false;
|
||||
entry.commendationLOT = tableData.getIntField("commendationLOT", -1);
|
||||
entry.commendationCost = tableData.getIntField("commendationCost", -1);
|
||||
UNUSED_COLUMN(entry.audioEquipMetaEventSet = tableData.getStringField("audioEquipMetaEventSet", ""));
|
||||
entry.currencyCosts = tableData.getStringField("currencyCosts", "");
|
||||
UNUSED_COLUMN(entry.ingredientInfo = tableData.getStringField("ingredientInfo", ""));
|
||||
entry.locStatus = tableData.getIntField("locStatus", -1);
|
||||
entry.forgeType = tableData.getIntField("forgeType", -1);
|
||||
entry.SellMultiplier = tableData.getFloatField("SellMultiplier", -1.0f);
|
||||
|
||||
CDItemComponent entry = ReadEntry(tableData);
|
||||
entries.insert(std::make_pair(entry.id, entry));
|
||||
tableData.nextRow();
|
||||
}
|
||||
@@ -70,6 +69,43 @@ void CDItemComponentTable::LoadValuesFromDatabase() {
|
||||
tableData.finalize();
|
||||
}
|
||||
|
||||
bool CDItemComponentTable::LoadFromFdb() {
|
||||
m_FdbTable = nullptr;
|
||||
const auto* table = CDFdb::GetTable("ItemComponent");
|
||||
if (!table) return false;
|
||||
|
||||
const auto changed = CDFdb::FindChangedKeys(*table);
|
||||
if (!changed) return false;
|
||||
|
||||
// Ids whose rows CDServer.sqlite changes are read from it once and kept; everything else comes from the fdb
|
||||
for (const auto id : *changed) LoadFromSqlite(static_cast<uint32_t>(id));
|
||||
LOG("ItemComponent: reading from the fdb, %zu ids differ in CDServer.sqlite and are kept in memory", changed->size());
|
||||
|
||||
m_FdbTable = table;
|
||||
return true;
|
||||
}
|
||||
|
||||
const CDItemComponent& CDItemComponentTable::LoadFromSqlite(uint32_t id) {
|
||||
auto& entries = GetEntriesMutable();
|
||||
auto query = CDClientDatabase::CreatePreppedStmt("SELECT * FROM ItemComponent WHERE id = ?;");
|
||||
query.bind(1, static_cast<int32_t>(id));
|
||||
|
||||
auto tableData = query.execQuery();
|
||||
if (tableData.eof()) {
|
||||
entries.insert(std::make_pair(id, Default));
|
||||
return Default;
|
||||
}
|
||||
|
||||
while (!tableData.eof()) {
|
||||
CDItemComponent entry = ReadEntry(tableData);
|
||||
entries.insert(std::make_pair(entry.id, entry));
|
||||
tableData.nextRow();
|
||||
}
|
||||
|
||||
const auto& it = entries.find(id);
|
||||
return it != entries.end() ? it->second : Default;
|
||||
}
|
||||
|
||||
const CDItemComponent& CDItemComponentTable::GetItemComponentByID(uint32_t skillID) {
|
||||
auto& entries = GetEntriesMutable();
|
||||
const auto& it = entries.find(skillID);
|
||||
@@ -77,70 +113,17 @@ const CDItemComponent& CDItemComponentTable::GetItemComponentByID(uint32_t skill
|
||||
return it->second;
|
||||
}
|
||||
|
||||
auto query = CDClientDatabase::CreatePreppedStmt("SELECT * FROM ItemComponent WHERE id = ?;");
|
||||
query.bind(1, static_cast<int32_t>(skillID));
|
||||
if (!m_FdbTable) return LoadFromSqlite(skillID);
|
||||
|
||||
auto tableData = query.execQuery();
|
||||
if (tableData.eof()) {
|
||||
// Only the items asked for are kept in memory; the first row of an id wins, as in the SQLite path
|
||||
const auto row = m_FdbTable->FindFirst(static_cast<int32_t>(skillID));
|
||||
if (!row) {
|
||||
entries.insert(std::make_pair(skillID, Default));
|
||||
return Default;
|
||||
}
|
||||
|
||||
while (!tableData.eof()) {
|
||||
CDItemComponent entry;
|
||||
entry.id = tableData.getIntField("id", -1);
|
||||
entry.equipLocation = tableData.getStringField("equipLocation", "");
|
||||
entry.baseValue = tableData.getIntField("baseValue", -1);
|
||||
entry.isKitPiece = tableData.getIntField("isKitPiece", -1) == 1 ? true : false;
|
||||
entry.rarity = tableData.getIntField("rarity", 0);
|
||||
entry.itemType = tableData.getIntField("itemType", -1);
|
||||
entry.itemInfo = tableData.getInt64Field("itemInfo", -1);
|
||||
entry.inLootTable = tableData.getIntField("inLootTable", -1) == 1 ? true : false;
|
||||
entry.inVendor = tableData.getIntField("inVendor", -1) == 1 ? true : false;
|
||||
entry.isUnique = tableData.getIntField("isUnique", -1) == 1 ? true : false;
|
||||
entry.isBOP = tableData.getIntField("isBOP", -1) == 1 ? true : false;
|
||||
entry.isBOE = tableData.getIntField("isBOE", -1) == 1 ? true : false;
|
||||
entry.reqFlagID = tableData.getIntField("reqFlagID", -1);
|
||||
entry.reqSpecialtyID = tableData.getIntField("reqSpecialtyID", -1);
|
||||
entry.reqSpecRank = tableData.getIntField("reqSpecRank", -1);
|
||||
entry.reqAchievementID = tableData.getIntField("reqAchievementID", -1);
|
||||
entry.stackSize = tableData.getIntField("stackSize", -1);
|
||||
entry.color1 = tableData.getIntField("color1", -1);
|
||||
entry.decal = tableData.getIntField("decal", -1);
|
||||
entry.offsetGroupID = tableData.getIntField("offsetGroupID", -1);
|
||||
entry.buildTypes = tableData.getIntField("buildTypes", -1);
|
||||
entry.reqPrecondition = tableData.getStringField("reqPrecondition", "");
|
||||
entry.animationFlag = tableData.getIntField("animationFlag", 0);
|
||||
entry.equipEffects = tableData.getIntField("equipEffects", -1);
|
||||
entry.readyForQA = tableData.getIntField("readyForQA", -1) == 1 ? true : false;
|
||||
entry.itemRating = tableData.getIntField("itemRating", -1);
|
||||
entry.isTwoHanded = tableData.getIntField("isTwoHanded", -1) == 1 ? true : false;
|
||||
entry.minNumRequired = tableData.getIntField("minNumRequired", -1);
|
||||
entry.delResIndex = tableData.getIntField("delResIndex", -1);
|
||||
entry.currencyLOT = tableData.getIntField("currencyLOT", -1);
|
||||
entry.altCurrencyCost = tableData.getIntField("altCurrencyCost", -1);
|
||||
entry.subItems = tableData.getStringField("subItems", "");
|
||||
UNUSED(entry.audioEventUse = tableData.getStringField("audioEventUse", ""));
|
||||
entry.noEquipAnimation = tableData.getIntField("noEquipAnimation", -1) == 1 ? true : false;
|
||||
entry.commendationLOT = tableData.getIntField("commendationLOT", -1);
|
||||
entry.commendationCost = tableData.getIntField("commendationCost", -1);
|
||||
UNUSED(entry.audioEquipMetaEventSet = tableData.getStringField("audioEquipMetaEventSet", ""));
|
||||
entry.currencyCosts = tableData.getStringField("currencyCosts", "");
|
||||
UNUSED(entry.ingredientInfo = tableData.getStringField("ingredientInfo", ""));
|
||||
entry.locStatus = tableData.getIntField("locStatus", -1);
|
||||
entry.forgeType = tableData.getIntField("forgeType", -1);
|
||||
entry.SellMultiplier = tableData.getFloatField("SellMultiplier", -1.0f);
|
||||
|
||||
entries.insert(std::make_pair(entry.id, entry));
|
||||
tableData.nextRow();
|
||||
}
|
||||
|
||||
const auto& it2 = entries.find(skillID);
|
||||
if (it2 != entries.end()) {
|
||||
return it2->second;
|
||||
}
|
||||
|
||||
return Default;
|
||||
const CDFdb::RowFields fields(*m_FdbTable, *row);
|
||||
return entries.insert(std::make_pair(skillID, ReadEntry(fields))).first->second;
|
||||
}
|
||||
|
||||
std::map<LOT, uint32_t> CDItemComponentTable::ParseCraftingCurrencies(const CDItemComponent& itemComponent) {
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
// Custom Classes
|
||||
#include "CDTable.h"
|
||||
#include "dCommonVars.h"
|
||||
#include "FdbReader.h"
|
||||
|
||||
struct CDItemComponent {
|
||||
uint32_t id; //!< The Component ID
|
||||
@@ -52,10 +53,20 @@ struct CDItemComponent {
|
||||
class CDItemComponentTable : public CDTable<CDItemComponentTable, std::map<uint32_t, CDItemComponent>> {
|
||||
public:
|
||||
void LoadValuesFromDatabase();
|
||||
|
||||
// Reads rows from the client's fdb from now on, if it is open; false keeps the table on CDServer.sqlite
|
||||
bool LoadFromFdb();
|
||||
|
||||
static std::map<LOT, uint32_t> ParseCraftingCurrencies(const CDItemComponent& itemComponent);
|
||||
|
||||
// Gets an entry by ID
|
||||
const CDItemComponent& GetItemComponentByID(uint32_t skillID);
|
||||
|
||||
static CDItemComponent Default;
|
||||
|
||||
private:
|
||||
// Caches the entry of one id from CDServer.sqlite (Default when there is none) and returns it
|
||||
const CDItemComponent& LoadFromSqlite(uint32_t id);
|
||||
|
||||
const FdbReader::Table* m_FdbTable = nullptr;
|
||||
};
|
||||
|
||||
@@ -1,21 +1,30 @@
|
||||
#include "CDObjectsTable.h"
|
||||
#include "CDFdb.h"
|
||||
#include "Logger.h"
|
||||
|
||||
namespace {
|
||||
CDObjects ObjDefault;
|
||||
|
||||
// Fills an entry from a CDServer.sqlite row or an fdb row (CDFdb::RowFields), which read alike
|
||||
template<typename Row>
|
||||
void ReadEntry(Row& row, CDObjects& entry) {
|
||||
entry.name = row.getStringField("name", "");
|
||||
UNUSED_COLUMN(entry.placeable = row.getIntField("placeable", -1);)
|
||||
entry.type = row.getStringField("type", "");
|
||||
UNUSED_COLUMN(entry.description = row.getStringField("description", "");)
|
||||
UNUSED_COLUMN(entry.localize = row.getIntField("localize", -1);)
|
||||
UNUSED_COLUMN(entry.npcTemplateID = row.getIntField("npcTemplateID", -1);)
|
||||
UNUSED_COLUMN(entry.displayName = row.getStringField("displayName", "");)
|
||||
entry.interactionDistance = row.getFloatField("interactionDistance", -1.0f);
|
||||
UNUSED_COLUMN(entry.nametag = row.getIntField("nametag", -1);)
|
||||
UNUSED_COLUMN(entry._internalNotes = row.getStringField("_internalNotes", "");)
|
||||
UNUSED_COLUMN(entry.locStatus = row.getIntField("locStatus", -1);)
|
||||
UNUSED_COLUMN(entry.gate_version = row.getStringField("gate_version", "");)
|
||||
UNUSED_COLUMN(entry.HQ_valid = row.getIntField("HQ_valid", -1);)
|
||||
}
|
||||
};
|
||||
|
||||
void CDObjectsTable::LoadValuesFromDatabase() {
|
||||
// First, get the size of the table
|
||||
uint32_t size = 0;
|
||||
auto tableSize = CDClientDatabase::ExecuteQuery("SELECT COUNT(*) FROM Objects");
|
||||
while (!tableSize.eof()) {
|
||||
size = tableSize.getIntField(0, 0);
|
||||
|
||||
tableSize.nextRow();
|
||||
}
|
||||
|
||||
tableSize.finalize();
|
||||
|
||||
// Now get the data
|
||||
auto tableData = CDClientDatabase::ExecuteQuery("SELECT * FROM Objects");
|
||||
auto& entries = GetEntriesMutable();
|
||||
@@ -24,19 +33,7 @@ void CDObjectsTable::LoadValuesFromDatabase() {
|
||||
|
||||
auto& entry = entries[lot];
|
||||
entry.id = lot;
|
||||
entry.name = tableData.getStringField("name", "");
|
||||
UNUSED_COLUMN(entry.placeable = tableData.getIntField("placeable", -1);)
|
||||
entry.type = tableData.getStringField("type", "");
|
||||
UNUSED_COLUMN(entry.description = tableData.getStringField("description", "");)
|
||||
UNUSED_COLUMN(entry.localize = tableData.getIntField("localize", -1);)
|
||||
UNUSED_COLUMN(entry.npcTemplateID = tableData.getIntField("npcTemplateID", -1);)
|
||||
UNUSED_COLUMN(entry.displayName = tableData.getStringField("displayName", "");)
|
||||
entry.interactionDistance = tableData.getFloatField("interactionDistance", -1.0f);
|
||||
UNUSED_COLUMN(entry.nametag = tableData.getIntField("nametag", -1);)
|
||||
UNUSED_COLUMN(entry._internalNotes = tableData.getStringField("_internalNotes", "");)
|
||||
UNUSED_COLUMN(entry.locStatus = tableData.getIntField("locStatus", -1);)
|
||||
UNUSED_COLUMN(entry.gate_version = tableData.getStringField("gate_version", "");)
|
||||
UNUSED_COLUMN(entry.HQ_valid = tableData.getIntField("HQ_valid", -1);)
|
||||
ReadEntry(tableData, entry);
|
||||
|
||||
tableData.nextRow();
|
||||
}
|
||||
@@ -44,13 +41,24 @@ void CDObjectsTable::LoadValuesFromDatabase() {
|
||||
ObjDefault.id = 0;
|
||||
}
|
||||
|
||||
const CDObjects& CDObjectsTable::GetByID(const uint32_t lot) {
|
||||
auto& entries = GetEntriesMutable();
|
||||
const auto& it = entries.find(lot);
|
||||
if (it != entries.end()) {
|
||||
return it->second;
|
||||
}
|
||||
bool CDObjectsTable::LoadFromFdb() {
|
||||
m_FdbTable = nullptr;
|
||||
const auto* table = CDFdb::GetTable("Objects");
|
||||
if (!table) return false;
|
||||
|
||||
const auto changed = CDFdb::FindChangedKeys(*table);
|
||||
if (!changed) return false;
|
||||
|
||||
// Ids whose rows CDServer.sqlite changes are read from it once and kept; everything else comes from the fdb
|
||||
for (const auto id : *changed) LoadFromSqlite(static_cast<uint32_t>(id));
|
||||
LOG("Objects: reading from the fdb, %zu ids differ in CDServer.sqlite and are kept in memory", changed->size());
|
||||
|
||||
m_FdbTable = table;
|
||||
return true;
|
||||
}
|
||||
|
||||
const CDObjects& CDObjectsTable::LoadFromSqlite(const uint32_t lot) {
|
||||
auto& entries = GetEntriesMutable();
|
||||
auto query = CDClientDatabase::CreatePreppedStmt("SELECT * FROM Objects WHERE id = ?;");
|
||||
query.bind(1, static_cast<int32_t>(lot));
|
||||
|
||||
@@ -62,33 +70,42 @@ const CDObjects& CDObjectsTable::GetByID(const uint32_t lot) {
|
||||
|
||||
// Now get the data
|
||||
while (!tableData.eof()) {
|
||||
const uint32_t lot = tableData.getIntField("id", 0);
|
||||
const uint32_t rowLot = tableData.getIntField("id", 0);
|
||||
|
||||
auto& entry = entries[lot];
|
||||
entry.id = lot;
|
||||
entry.name = tableData.getStringField("name", "");
|
||||
UNUSED(entry.placeable = tableData.getIntField("placeable", -1));
|
||||
entry.type = tableData.getStringField("type", "");
|
||||
UNUSED(entry.description = tableData.getStringField(4, ""));
|
||||
UNUSED(entry.localize = tableData.getIntField("localize", -1));
|
||||
UNUSED(entry.npcTemplateID = tableData.getIntField("npcTemplateID", -1));
|
||||
UNUSED(entry.displayName = tableData.getStringField("displayName", ""));
|
||||
entry.interactionDistance = tableData.getFloatField("interactionDistance", -1.0f);
|
||||
UNUSED(entry.nametag = tableData.getIntField("nametag", -1));
|
||||
UNUSED(entry._internalNotes = tableData.getStringField("_internalNotes", ""));
|
||||
UNUSED(entry.locStatus = tableData.getIntField("locStatus", -1));
|
||||
UNUSED(entry.gate_version = tableData.getStringField("gate_version", ""));
|
||||
UNUSED(entry.HQ_valid = tableData.getIntField("HQ_valid", -1));
|
||||
auto& entry = entries[rowLot];
|
||||
entry.id = rowLot;
|
||||
ReadEntry(tableData, entry);
|
||||
|
||||
tableData.nextRow();
|
||||
}
|
||||
|
||||
tableData.finalize();
|
||||
|
||||
const auto& it2 = entries.find(lot);
|
||||
if (it2 != entries.end()) {
|
||||
return it2->second;
|
||||
const auto& it = entries.find(lot);
|
||||
return it != entries.end() ? it->second : ObjDefault;
|
||||
}
|
||||
|
||||
const CDObjects& CDObjectsTable::GetByID(const uint32_t lot) {
|
||||
auto& entries = GetEntriesMutable();
|
||||
const auto& it = entries.find(lot);
|
||||
if (it != entries.end()) {
|
||||
return it->second;
|
||||
}
|
||||
|
||||
return ObjDefault;
|
||||
if (!m_FdbTable) return LoadFromSqlite(lot);
|
||||
|
||||
// Only the objects asked for are kept in memory; the last row of an id wins, as in the SQLite path
|
||||
std::optional<CDObjects> found;
|
||||
m_FdbTable->ForEachRowWithKey(static_cast<int32_t>(lot), [&](const FdbReader::Row& row) {
|
||||
if (!found) found.emplace();
|
||||
found->id = lot;
|
||||
const CDFdb::RowFields fields(*m_FdbTable, row);
|
||||
ReadEntry(fields, *found);
|
||||
});
|
||||
if (!found) {
|
||||
entries.emplace(lot, ObjDefault);
|
||||
return ObjDefault;
|
||||
}
|
||||
|
||||
return entries.emplace(lot, std::move(*found)).first->second;
|
||||
}
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
#include "FdbReader.h"
|
||||
|
||||
struct CDObjects {
|
||||
uint32_t id; //!< The LOT of the object
|
||||
std::string name; //!< The internal name of the object
|
||||
@@ -25,7 +27,17 @@ struct CDObjects {
|
||||
class CDObjectsTable : public CDTable<CDObjectsTable, std::map<uint32_t, CDObjects>> {
|
||||
public:
|
||||
void LoadValuesFromDatabase();
|
||||
|
||||
// Reads rows from the client's fdb from now on, if it is open; false keeps the table on CDServer.sqlite
|
||||
bool LoadFromFdb();
|
||||
|
||||
// Gets an entry by ID
|
||||
const CDObjects& GetByID(const uint32_t lot);
|
||||
|
||||
private:
|
||||
// Caches the entry of one id from CDServer.sqlite (the default entry when there is none) and returns it
|
||||
const CDObjects& LoadFromSqlite(const uint32_t lot);
|
||||
|
||||
const FdbReader::Table* m_FdbTable = nullptr;
|
||||
};
|
||||
|
||||
|
||||
@@ -494,7 +494,7 @@ int main(int argc, char** argv) {
|
||||
|
||||
//Get CDClient initial information
|
||||
try {
|
||||
CDClientManager::LoadValuesFromDatabase();
|
||||
CDClientManager::LoadValuesFromDatabase(Game::assetManager->GetResPath() / "cdclient.fdb");
|
||||
} catch (CppSQLite3Exception& e) {
|
||||
LOG("Failed to initialize CDServer SQLite Database");
|
||||
LOG("May be caused by corrupted file: %s", (Game::assetManager->GetResPath() / "CDServer.sqlite").string().c_str());
|
||||
|
||||
@@ -275,7 +275,8 @@ int main(int argc, char** argv) {
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
CDClientManager::LoadValuesFromDatabase();
|
||||
// The client's fdb, read in place and shared between all server processes, when there is one
|
||||
CDClientManager::LoadValuesFromDatabase(Game::assetManager->GetResPath() / "cdclient.fdb");
|
||||
|
||||
Diagnostics::SetProduceMemoryDump(Game::config->GetValue("generate_dump") == "1");
|
||||
|
||||
|
||||
304
tests/dGameTests/CDFdbTests.cpp
Normal file
304
tests/dGameTests/CDFdbTests.cpp
Normal file
@@ -0,0 +1,304 @@
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "CDClientDatabase.h"
|
||||
#include "CDClientManager.h"
|
||||
#include "CDComponentsRegistryTable.h"
|
||||
#include "CDFdb.h"
|
||||
#include "CDItemComponentTable.h"
|
||||
#include "CDObjectsTable.h"
|
||||
#include "Game.h"
|
||||
#include "Logger.h"
|
||||
#include "eReplicaComponentType.h"
|
||||
#include "../FdbTestWriter.h"
|
||||
|
||||
using FdbTestWriter::Value;
|
||||
|
||||
namespace {
|
||||
void ClearTables() {
|
||||
CDClientManager::GetEntriesMutable<CDComponentsRegistryTable>().clear();
|
||||
CDClientManager::GetEntriesMutable<CDItemComponentTable>().clear();
|
||||
CDClientManager::GetEntriesMutable<CDObjectsTable>().clear();
|
||||
}
|
||||
|
||||
int32_t Component(uint32_t id, int32_t type, int32_t defaultValue = -99) {
|
||||
return CDComponentsRegistryTable::Instance().GetByIDAndType(id, static_cast<eReplicaComponentType>(type), defaultValue);
|
||||
}
|
||||
|
||||
class CDFdbTestBase : public ::testing::Test {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
m_Logger = std::make_unique<Logger>("./testing.log", false, false);
|
||||
Game::logger = m_Logger.get();
|
||||
ClearTables();
|
||||
}
|
||||
|
||||
void TearDown() override {
|
||||
// Let go of the fdb before its file is removed; the tables stop using it on their next load
|
||||
CDFdb::Close();
|
||||
CDComponentsRegistryTable::Instance().LoadFromFdb();
|
||||
CDItemComponentTable::Instance().LoadFromFdb();
|
||||
CDObjectsTable::Instance().LoadFromFdb();
|
||||
ClearTables();
|
||||
Game::logger = nullptr;
|
||||
}
|
||||
|
||||
std::unique_ptr<Logger> m_Logger;
|
||||
};
|
||||
|
||||
// A ComponentsRegistry in both files, where CDServer.sqlite changes, drops and adds rows the way migrations do
|
||||
class CDFdbOverlayTest : public CDFdbTestBase {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
CDFdbTestBase::SetUp();
|
||||
m_Dir = std::filesystem::temp_directory_path() / ("dlu_cdfdb_" + std::string(::testing::UnitTest::GetInstance()->current_test_info()->name()));
|
||||
std::filesystem::create_directories(m_Dir);
|
||||
|
||||
FdbTestWriter::Table registry;
|
||||
registry.name = "ComponentsRegistry";
|
||||
registry.columns = { { "id", eSqliteDataType::INT32 }, { "component_type", eSqliteDataType::INT32 }, { "component_id", eSqliteDataType::INT32 } };
|
||||
registry.bucketCount = 4;
|
||||
// 1, 5 and 9 share bucket 1
|
||||
registry.rows = {
|
||||
{ Value::Int(1), Value::Int(1), Value::Int(10) },
|
||||
{ Value::Int(1), Value::Int(2), Value::Int(20) },
|
||||
{ Value::Int(5), Value::Int(1), Value::Int(50) },
|
||||
{ Value::Int(9), Value::Int(1), Value::Int(90) },
|
||||
{ Value::Int(9), Value::Int(2), Value::Int(91) },
|
||||
{ Value::Int(2), Value::Int(1), Value::Int(200) },
|
||||
// A repeated (id, type): the last row wins
|
||||
{ Value::Int(2), Value::Int(1), Value::Int(201) },
|
||||
};
|
||||
FdbTestWriter::WriteFile(m_Dir / "cdclient.fdb", FdbTestWriter::Write({ registry }));
|
||||
|
||||
std::filesystem::remove(m_Dir / "CDServer.sqlite");
|
||||
CDClientDatabase::Connect((m_Dir / "CDServer.sqlite").string());
|
||||
CDClientDatabase::ExecuteDML("CREATE TABLE ComponentsRegistry ('id' int32, 'component_type' int32, 'component_id' int32);");
|
||||
CDClientDatabase::ExecuteDML(
|
||||
"INSERT INTO ComponentsRegistry VALUES (1, 1, 10), (1, 2, 20), (5, 1, 55), (2, 1, 200), (2, 1, 201), (13, 1, 130);");
|
||||
}
|
||||
|
||||
void TearDown() override {
|
||||
CDFdbTestBase::TearDown();
|
||||
std::error_code error;
|
||||
std::filesystem::remove_all(m_Dir, error);
|
||||
}
|
||||
|
||||
void ExpectRegistryAnswers() {
|
||||
EXPECT_EQ(Component(1, 1), 10);
|
||||
EXPECT_EQ(Component(1, 2), 20);
|
||||
EXPECT_EQ(Component(1, 3), -99);
|
||||
// Changed by CDServer.sqlite
|
||||
EXPECT_EQ(Component(5, 1), 55);
|
||||
// Dropped by CDServer.sqlite
|
||||
EXPECT_EQ(Component(9, 1), -99);
|
||||
EXPECT_EQ(Component(9, 2), -99);
|
||||
// Added by CDServer.sqlite
|
||||
EXPECT_EQ(Component(13, 1), 130);
|
||||
EXPECT_EQ(Component(2, 1), 201);
|
||||
EXPECT_EQ(Component(3, 1), -99);
|
||||
}
|
||||
|
||||
std::filesystem::path m_Dir;
|
||||
};
|
||||
}
|
||||
|
||||
TEST_F(CDFdbOverlayTest, FindsTheKeysSqliteChanges) {
|
||||
ASSERT_TRUE(CDFdb::Open(m_Dir / "cdclient.fdb"));
|
||||
const auto* table = CDFdb::GetTable("ComponentsRegistry");
|
||||
ASSERT_NE(table, nullptr);
|
||||
const auto changed = CDFdb::FindChangedKeys(*table);
|
||||
ASSERT_TRUE(changed.has_value());
|
||||
EXPECT_EQ(std::set<int64_t>(changed->begin(), changed->end()), (std::set<int64_t>{ 5, 9, 13 }));
|
||||
|
||||
// Tables that aren't in both files, or whose columns differ, aren't read from the fdb
|
||||
EXPECT_EQ(CDFdb::GetTable("Objects"), nullptr);
|
||||
}
|
||||
|
||||
TEST_F(CDFdbOverlayTest, ReadsFromTheFdbWithSqliteChangesOnTop) {
|
||||
ASSERT_TRUE(CDFdb::Open(m_Dir / "cdclient.fdb"));
|
||||
ASSERT_TRUE(CDComponentsRegistryTable::Instance().LoadFromFdb());
|
||||
|
||||
// Only the changed ids are cached (an id marker plus its rows), not the table
|
||||
EXPECT_EQ(CDClientManager::GetEntriesMutable<CDComponentsRegistryTable>().size(), 5u);
|
||||
|
||||
ExpectRegistryAnswers();
|
||||
}
|
||||
|
||||
TEST_F(CDFdbOverlayTest, WithoutAnFdbReadsSqlite) {
|
||||
EXPECT_FALSE(CDFdb::Open(m_Dir / "missing.fdb"));
|
||||
EXPECT_EQ(CDFdb::Get(), nullptr);
|
||||
EXPECT_FALSE(CDComponentsRegistryTable::Instance().LoadFromFdb());
|
||||
|
||||
// Lazy SQLite lookups, as before
|
||||
ExpectRegistryAnswers();
|
||||
|
||||
// And the whole table in memory, as the server loads it without an fdb
|
||||
ClearTables();
|
||||
CDComponentsRegistryTable::Instance().LoadValuesFromDatabase();
|
||||
ExpectRegistryAnswers();
|
||||
}
|
||||
|
||||
TEST_F(CDFdbOverlayTest, UnmappedFdbReadsTheSame) {
|
||||
ASSERT_TRUE(CDFdb::Open(m_Dir / "cdclient.fdb", false));
|
||||
EXPECT_FALSE(CDFdb::Get()->IsMapped());
|
||||
ASSERT_TRUE(CDComponentsRegistryTable::Instance().LoadFromFdb());
|
||||
ExpectRegistryAnswers();
|
||||
}
|
||||
|
||||
namespace {
|
||||
std::vector<int64_t> DistinctIds(const std::string& table) {
|
||||
std::vector<int64_t> ids;
|
||||
auto query = CDClientDatabase::ExecuteQuery("SELECT DISTINCT id FROM " + table + ";");
|
||||
while (!query.eof()) {
|
||||
ids.push_back(query.getInt64Field(0));
|
||||
query.nextRow();
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
bool SameFloat(float a, float b) {
|
||||
return a == b || (std::isnan(a) && std::isnan(b));
|
||||
}
|
||||
|
||||
// Every id of the table in either file, plus some that are in neither
|
||||
std::vector<uint32_t> AllIds(const std::string& name) {
|
||||
std::set<uint32_t> ids{ 0, 0xFFFFFFFF, 99999999 };
|
||||
for (const auto id : DistinctIds(name)) ids.insert(static_cast<uint32_t>(id));
|
||||
if (const auto* table = CDFdb::Get() ? CDFdb::Get()->GetTable(name) : nullptr) {
|
||||
table->ForEachRow([&](const FdbReader::Row& row) {
|
||||
if (const auto key = FdbReader::Table::KeyOf(row)) ids.insert(static_cast<uint32_t>(*key));
|
||||
});
|
||||
}
|
||||
return { ids.begin(), ids.end() };
|
||||
}
|
||||
|
||||
struct Snapshot {
|
||||
std::vector<int32_t> components;
|
||||
std::vector<CDItemComponent> items;
|
||||
std::vector<CDObjects> objects;
|
||||
};
|
||||
|
||||
Snapshot TakeSnapshot(const std::vector<uint32_t>& registryIds, const std::vector<int32_t>& types,
|
||||
const std::vector<uint32_t>& itemIds, const std::vector<uint32_t>& objectIds) {
|
||||
Snapshot snapshot;
|
||||
for (const auto id : registryIds) {
|
||||
for (const auto type : types) snapshot.components.push_back(Component(id, type, -12345));
|
||||
}
|
||||
for (const auto id : itemIds) snapshot.items.push_back(CDItemComponentTable::Instance().GetItemComponentByID(id));
|
||||
for (const auto id : objectIds) snapshot.objects.push_back(CDObjectsTable::Instance().GetByID(id));
|
||||
return snapshot;
|
||||
}
|
||||
}
|
||||
|
||||
// Every row of the switched tables through the fdb against CDServer.sqlite, when both are around:
|
||||
// DLU_CLIENT_RES is the client's res folder (with cdclient.fdb), DLU_CDSERVER_SQLITE the converted
|
||||
// database (default: the source tree's build/resServer/CDServer.sqlite). Opened read-only.
|
||||
TEST_F(CDFdbTestBase, RealClientParity) {
|
||||
const char* res = std::getenv("DLU_CLIENT_RES");
|
||||
if (!res) GTEST_SKIP() << "Set DLU_CLIENT_RES to a client res folder to compare against its cdclient.fdb";
|
||||
const auto fdbPath = std::filesystem::path(res) / "cdclient.fdb";
|
||||
if (!std::filesystem::exists(fdbPath)) GTEST_SKIP() << "No " << fdbPath.string();
|
||||
|
||||
std::string sqlitePath = std::string(PROJECT_SOURCE_DIR) + "/build/resServer/CDServer.sqlite";
|
||||
if (const char* env = std::getenv("DLU_CDSERVER_SQLITE")) sqlitePath = env;
|
||||
if (!std::filesystem::exists(sqlitePath)) GTEST_SKIP() << "No CDServer.sqlite at " << sqlitePath;
|
||||
|
||||
CDClientDatabase::Connect(sqlitePath);
|
||||
CDClientDatabase::ExecuteDML("PRAGMA query_only = ON;"); // never write to the CDClient
|
||||
|
||||
ASSERT_TRUE(CDFdb::Open(fdbPath));
|
||||
EXPECT_TRUE(CDFdb::Get()->IsMapped());
|
||||
|
||||
const auto registryIds = AllIds("ComponentsRegistry");
|
||||
const auto itemIds = AllIds("ItemComponent");
|
||||
const auto objectIds = AllIds("Objects");
|
||||
std::vector<int32_t> types{ 999 };
|
||||
for (const auto type : [] {
|
||||
std::vector<int64_t> found;
|
||||
auto query = CDClientDatabase::ExecuteQuery("SELECT DISTINCT component_type FROM ComponentsRegistry;");
|
||||
while (!query.eof()) {
|
||||
found.push_back(query.getInt64Field(0));
|
||||
query.nextRow();
|
||||
}
|
||||
return found;
|
||||
}()) types.push_back(static_cast<int32_t>(type));
|
||||
|
||||
for (const auto* name : { "ComponentsRegistry", "ItemComponent", "Objects" }) {
|
||||
const auto* table = CDFdb::GetTable(name);
|
||||
ASSERT_NE(table, nullptr) << name;
|
||||
const auto changed = CDFdb::FindChangedKeys(*table);
|
||||
ASSERT_TRUE(changed.has_value()) << name;
|
||||
std::cout << name << ": " << changed->size() << " ids differ between the fdb and CDServer.sqlite" << std::endl;
|
||||
}
|
||||
|
||||
// Through the fdb
|
||||
ASSERT_TRUE(CDComponentsRegistryTable::Instance().LoadFromFdb());
|
||||
ASSERT_TRUE(CDItemComponentTable::Instance().LoadFromFdb());
|
||||
ASSERT_TRUE(CDObjectsTable::Instance().LoadFromFdb());
|
||||
const auto fromFdb = TakeSnapshot(registryIds, types, itemIds, objectIds);
|
||||
|
||||
// Through CDServer.sqlite, loaded the way the server does without an fdb
|
||||
CDFdb::Close();
|
||||
ClearTables();
|
||||
EXPECT_FALSE(CDComponentsRegistryTable::Instance().LoadFromFdb());
|
||||
EXPECT_FALSE(CDItemComponentTable::Instance().LoadFromFdb());
|
||||
EXPECT_FALSE(CDObjectsTable::Instance().LoadFromFdb());
|
||||
CDComponentsRegistryTable::Instance().LoadValuesFromDatabase();
|
||||
CDItemComponentTable::Instance().LoadValuesFromDatabase();
|
||||
CDObjectsTable::Instance().LoadValuesFromDatabase();
|
||||
const auto fromSqlite = TakeSnapshot(registryIds, types, itemIds, objectIds);
|
||||
|
||||
ASSERT_EQ(fromFdb.components.size(), fromSqlite.components.size());
|
||||
uint32_t componentMismatches = 0;
|
||||
for (size_t i = 0; i < fromFdb.components.size(); i++) {
|
||||
if (fromFdb.components[i] != fromSqlite.components[i]) {
|
||||
if (componentMismatches++ < 20) {
|
||||
ADD_FAILURE() << "ComponentsRegistry id " << registryIds[i / types.size()] << " type " << types[i % types.size()]
|
||||
<< ": fdb " << fromFdb.components[i] << " sqlite " << fromSqlite.components[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
EXPECT_EQ(componentMismatches, 0u);
|
||||
|
||||
ASSERT_EQ(fromFdb.items.size(), fromSqlite.items.size());
|
||||
uint32_t itemMismatches = 0;
|
||||
for (size_t i = 0; i < fromFdb.items.size(); i++) {
|
||||
const auto& a = fromFdb.items[i];
|
||||
const auto& b = fromSqlite.items[i];
|
||||
const bool same = a.id == b.id && a.equipLocation == b.equipLocation && a.baseValue == b.baseValue &&
|
||||
a.isKitPiece == b.isKitPiece && a.rarity == b.rarity && a.itemType == b.itemType && a.itemInfo == b.itemInfo &&
|
||||
a.inLootTable == b.inLootTable && a.inVendor == b.inVendor && a.isUnique == b.isUnique && a.isBOP == b.isBOP &&
|
||||
a.isBOE == b.isBOE && a.reqFlagID == b.reqFlagID && a.reqSpecialtyID == b.reqSpecialtyID &&
|
||||
a.reqSpecRank == b.reqSpecRank && a.reqAchievementID == b.reqAchievementID && a.stackSize == b.stackSize &&
|
||||
a.color1 == b.color1 && a.decal == b.decal && a.offsetGroupID == b.offsetGroupID && a.buildTypes == b.buildTypes &&
|
||||
a.reqPrecondition == b.reqPrecondition && a.animationFlag == b.animationFlag && a.equipEffects == b.equipEffects &&
|
||||
a.readyForQA == b.readyForQA && a.itemRating == b.itemRating && a.isTwoHanded == b.isTwoHanded &&
|
||||
a.minNumRequired == b.minNumRequired && a.delResIndex == b.delResIndex && a.currencyLOT == b.currencyLOT &&
|
||||
a.altCurrencyCost == b.altCurrencyCost && a.subItems == b.subItems && a.noEquipAnimation == b.noEquipAnimation &&
|
||||
a.commendationLOT == b.commendationLOT && a.commendationCost == b.commendationCost &&
|
||||
a.currencyCosts == b.currencyCosts && a.locStatus == b.locStatus && a.forgeType == b.forgeType &&
|
||||
SameFloat(a.SellMultiplier, b.SellMultiplier);
|
||||
if (!same && itemMismatches++ < 20) ADD_FAILURE() << "ItemComponent id " << itemIds[i] << " differs";
|
||||
}
|
||||
EXPECT_EQ(itemMismatches, 0u);
|
||||
|
||||
ASSERT_EQ(fromFdb.objects.size(), fromSqlite.objects.size());
|
||||
uint32_t objectMismatches = 0;
|
||||
for (size_t i = 0; i < fromFdb.objects.size(); i++) {
|
||||
const auto& a = fromFdb.objects[i];
|
||||
const auto& b = fromSqlite.objects[i];
|
||||
const bool same = a.id == b.id && a.name == b.name && a.type == b.type && SameFloat(a.interactionDistance, b.interactionDistance);
|
||||
if (!same && objectMismatches++ < 20) ADD_FAILURE() << "Objects id " << objectIds[i] << " differs: fdb '" << a.name << "' sqlite '" << b.name << "'";
|
||||
}
|
||||
EXPECT_EQ(objectMismatches, 0u);
|
||||
|
||||
std::cout << "Compared " << registryIds.size() << " ComponentsRegistry ids x " << types.size() << " types, "
|
||||
<< itemIds.size() << " ItemComponent ids, " << objectIds.size() << " Objects ids" << std::endl;
|
||||
}
|
||||
@@ -27,6 +27,7 @@ set(DGAMETEST_SOURCES
|
||||
"PreconditionItemCostTests.cpp"
|
||||
"SpiderQueenTests.cpp"
|
||||
"TacArcTests.cpp"
|
||||
"CDFdbTests.cpp"
|
||||
)
|
||||
|
||||
add_subdirectory(dComponentsTests)
|
||||
|
||||
Reference in New Issue
Block a user