perf(cdclient): read ComponentsRegistry, ItemComponent and Objects from the fdb

The world and master servers pass the client's res/cdclient.fdb to
CDClientManager. When it opens, these three tables, all looked up by
their first column, find rows through the fdb's buckets instead of each
process caching the whole table: ComponentsRegistry keeps nothing,
ItemComponent and Objects keep only the entries asked for (their API
returns references). Ids whose rows CDServer.sqlite changes are loaded
from SQLite at startup and win. Without an fdb (or with one whose
columns don't match) the tables load from CDServer.sqlite as before.

ItemComponent and Objects now fill entries from one template for both
sources instead of copies of the same field list.

Tests cover the SQLite changes on top of the fdb, the no-fdb and
unmapped paths, and, when DLU_CLIENT_RES points at a client, every id of
the three tables through the fdb against CDServer.sqlite.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-09-29 21:56:32 -05:00
parent 17eb26fb1d
commit a4fa7eb30a
12 changed files with 581 additions and 179 deletions

View File

@@ -1,4 +1,6 @@
#include "CDClientManager.h"
#include "CDFdb.h"
#include "Logger.h"
#include "CDActivityRewardsTable.h"
#include "CDAnimationsTable.h"
#include "CDBehaviorParameterTable.h"
@@ -111,11 +113,20 @@ DEFINE_TABLE_STORAGE(CDSkillBehaviorTable);
DEFINE_TABLE_STORAGE(CDTamingBuildPuzzleTable);
DEFINE_TABLE_STORAGE(CDVendorComponentTable);
void CDClientManager::LoadValuesFromDatabase() {
void CDClientManager::LoadValuesFromDatabase(const std::filesystem::path& fdbPath) {
if (!CDClientDatabase::isConnected) {
throw std::runtime_error{ "CDClientDatabase is not connected!" };
}
if (!fdbPath.empty()) {
if (CDFdb::Open(fdbPath)) {
LOG("Reading CDClient rows by id from %s (%s)", fdbPath.string().c_str(),
CDFdb::Get()->IsMapped() ? "mapped, shared between processes" : "could not map it, read into memory");
} else {
LOG("Could not open %s, reading CDClient from CDServer.sqlite only", fdbPath.string().c_str());
}
}
CDActivityRewardsTable::Instance().LoadValuesFromDatabase();
CDActivitiesTable::Instance().LoadValuesFromDatabase();
CDCLIENT_DONT_CACHE_TABLE(CDAnimationsTable::Instance().LoadValuesFromDatabase());
@@ -123,10 +134,11 @@ void CDClientManager::LoadValuesFromDatabase() {
CDBehaviorTemplateTable::Instance().LoadValuesFromDatabase();
CDBrickIDTableTable::Instance().LoadValuesFromDatabase();
CDCollectibleComponentTable::Instance().LoadValuesFromDatabase();
// Always in memory: every entity and every inventory item asks for its components and item data, and looking up an
// ComponentsRegistry, ItemComponent and Objects read rows from the fdb when it is open. Without it, the first two are
// always in memory: every entity and every inventory item asks for its components and item data, and looking up an
// id not seen yet in the unindexed CDClient scans the whole table (a character holding 3000 different items took a
// minute to load)
CDComponentsRegistryTable::Instance().LoadValuesFromDatabase();
if (!CDComponentsRegistryTable::Instance().LoadFromFdb()) CDComponentsRegistryTable::Instance().LoadValuesFromDatabase();
CDCurrencyTableTable::Instance().LoadValuesFromDatabase();
CDDeletionRestrictionsTable::Instance().LoadValuesFromDatabase();
CDDestructibleComponentTable::Instance().LoadValuesFromDatabase();
@@ -134,7 +146,7 @@ void CDClientManager::LoadValuesFromDatabase() {
CDEventGatingTable::Instance().LoadValuesFromDatabase();
CDFeatureGatingTable::Instance().LoadValuesFromDatabase();
CDInventoryComponentTable::Instance().LoadValuesFromDatabase();
CDItemComponentTable::Instance().LoadValuesFromDatabase();
if (!CDItemComponentTable::Instance().LoadFromFdb()) CDItemComponentTable::Instance().LoadValuesFromDatabase();
CDItemSetSkillsTable::Instance().LoadValuesFromDatabase();
CDItemSetsTable::Instance().LoadValuesFromDatabase();
CDLevelProgressionLookupTable::Instance().LoadValuesFromDatabase();
@@ -146,7 +158,7 @@ void CDClientManager::LoadValuesFromDatabase() {
CDMissionsTable::Instance().LoadValuesFromDatabase();
CDMovementAIComponentTable::Instance().LoadValuesFromDatabase();
CDObjectSkillsTable::Instance().LoadValuesFromDatabase();
CDCLIENT_DONT_CACHE_TABLE(CDObjectsTable::Instance().LoadValuesFromDatabase());
if (!CDObjectsTable::Instance().LoadFromFdb()) CDCLIENT_DONT_CACHE_TABLE(CDObjectsTable::Instance().LoadValuesFromDatabase());
CDPhysicsComponentTable::Instance().LoadValuesFromDatabase();
CDPackageComponentTable::Instance().LoadValuesFromDatabase();
CDPetComponentTable::Instance().LoadValuesFromDatabase();

View File

@@ -3,11 +3,18 @@
#define UNUSED_TABLE(v)
#include <filesystem>
/**
* Initialize the CDClient tables so they are all loaded into memory.
*/
namespace CDClientManager {
void LoadValuesFromDatabase();
/**
* @param fdbPath The client's cdclient.fdb. When it opens, the tables looked up by their first
* column read their rows from it (shared between processes) instead of caching them. Empty, or
* a file that doesn't open, keeps every table on CDServer.sqlite.
*/
void LoadValuesFromDatabase(const std::filesystem::path& fdbPath = {});
void LoadValuesFromDefaults();
/**

View File

@@ -1,4 +1,6 @@
#include "CDComponentsRegistryTable.h"
#include "CDFdb.h"
#include "Logger.h"
#include "eReplicaComponentType.h"
void CDComponentsRegistryTable::LoadValuesFromDatabase() {
@@ -20,15 +22,30 @@ void CDComponentsRegistryTable::LoadValuesFromDatabase() {
tableData.finalize();
}
int32_t CDComponentsRegistryTable::GetByIDAndType(uint32_t id, eReplicaComponentType componentType, int32_t defaultValue) {
auto& entries = GetEntriesMutable();
auto exists = entries.find(id);
if (exists != entries.end()) {
auto iter = entries.find(static_cast<uint64_t>(componentType) << 32 | static_cast<uint64_t>(id));
return iter == entries.end() ? defaultValue : iter->second;
}
bool CDComponentsRegistryTable::LoadFromFdb() {
m_FdbTable = nullptr;
const auto* table = CDFdb::GetTable("ComponentsRegistry");
if (!table) return false;
// Now get the data. Get all components of this entity so we dont do a query for each component
m_TypeColumn = table->GetColumnIndex("component_type");
m_ComponentIdColumn = table->GetColumnIndex("component_id");
if (m_TypeColumn < 0 || m_ComponentIdColumn < 0) return false;
const auto changed = CDFdb::FindChangedKeys(*table);
if (!changed) return false;
// Ids whose rows CDServer.sqlite changes are read from it once and kept; everything else comes from the fdb
for (const auto id : *changed) LoadFromSqlite(static_cast<uint32_t>(id));
LOG("ComponentsRegistry: reading from the fdb, %zu ids differ in CDServer.sqlite and are kept in memory", changed->size());
m_FdbTable = table;
return true;
}
void CDComponentsRegistryTable::LoadFromSqlite(uint32_t id) {
auto& entries = GetEntriesMutable();
// Get all components of this entity so we dont do a query for each component
auto query = CDClientDatabase::CreatePreppedStmt("SELECT * FROM ComponentsRegistry WHERE id = ?;");
query.bind(1, static_cast<int32_t>(id));
@@ -46,6 +63,29 @@ int32_t CDComponentsRegistryTable::GetByIDAndType(uint32_t id, eReplicaComponent
}
entries.insert_or_assign(id, 0);
}
int32_t CDComponentsRegistryTable::GetByIDAndType(uint32_t id, eReplicaComponentType componentType, int32_t defaultValue) {
auto& entries = GetEntriesMutable();
auto exists = entries.find(id);
if (exists != entries.end()) {
auto iter = entries.find(static_cast<uint64_t>(componentType) << 32 | static_cast<uint64_t>(id));
return iter == entries.end() ? defaultValue : iter->second;
}
if (m_FdbTable) {
// The last matching row wins, as when the rows are loaded into the map in file order
int32_t result = defaultValue;
const auto type = static_cast<int32_t>(componentType);
m_FdbTable->ForEachRowWithKey(static_cast<int32_t>(id), [&](const FdbReader::Row& row) {
if (row.GetInt(static_cast<uint32_t>(m_TypeColumn), 0) == type) {
result = row.GetInt(static_cast<uint32_t>(m_ComponentIdColumn), -1);
}
});
return result;
}
LoadFromSqlite(id);
auto iter = entries.find(static_cast<uint64_t>(componentType) << 32 | static_cast<uint64_t>(id));

View File

@@ -5,6 +5,8 @@
#include <unordered_map>
#include "FdbReader.h"
enum class eReplicaComponentType : uint32_t;
struct CDComponentsRegistry {
uint32_t id; //!< The LOT is used as the ID
@@ -16,5 +18,17 @@ struct CDComponentsRegistry {
class CDComponentsRegistryTable : public CDTable<CDComponentsRegistryTable, std::unordered_map<uint64_t, uint32_t>> {
public:
void LoadValuesFromDatabase();
// Reads rows from the client's fdb from now on, if it is open; false keeps the table on CDServer.sqlite
bool LoadFromFdb();
int32_t GetByIDAndType(uint32_t id, eReplicaComponentType componentType, int32_t defaultValue = 0);
private:
// Caches every row of one id from CDServer.sqlite
void LoadFromSqlite(uint32_t id);
const FdbReader::Table* m_FdbTable = nullptr;
int32_t m_TypeColumn = -1;
int32_t m_ComponentIdColumn = -1;
};

View File

@@ -1,68 +1,67 @@
#include "CDItemComponentTable.h"
#include "GeneralUtils.h"
#include "CDFdb.h"
#include "Logger.h"
CDItemComponent CDItemComponentTable::Default = {};
void CDItemComponentTable::LoadValuesFromDatabase() {
// First, get the size of the table
uint32_t size = 0;
auto tableSize = CDClientDatabase::ExecuteQuery("SELECT COUNT(*) FROM ItemComponent");
while (!tableSize.eof()) {
size = tableSize.getIntField(0, 0);
tableSize.nextRow();
namespace {
// Fills an entry from a CDServer.sqlite row or an fdb row (CDFdb::RowFields), which read alike
template<typename Row>
CDItemComponent ReadEntry(Row& row) {
CDItemComponent entry;
entry.id = row.getIntField("id", -1);
entry.equipLocation = row.getStringField("equipLocation", "");
entry.baseValue = row.getIntField("baseValue", -1);
entry.isKitPiece = row.getIntField("isKitPiece", -1) == 1 ? true : false;
entry.rarity = row.getIntField("rarity", 0);
entry.itemType = row.getIntField("itemType", -1);
entry.itemInfo = row.getInt64Field("itemInfo", -1);
entry.inLootTable = row.getIntField("inLootTable", -1) == 1 ? true : false;
entry.inVendor = row.getIntField("inVendor", -1) == 1 ? true : false;
entry.isUnique = row.getIntField("isUnique", -1) == 1 ? true : false;
entry.isBOP = row.getIntField("isBOP", -1) == 1 ? true : false;
entry.isBOE = row.getIntField("isBOE", -1) == 1 ? true : false;
entry.reqFlagID = row.getIntField("reqFlagID", -1);
entry.reqSpecialtyID = row.getIntField("reqSpecialtyID", -1);
entry.reqSpecRank = row.getIntField("reqSpecRank", -1);
entry.reqAchievementID = row.getIntField("reqAchievementID", -1);
entry.stackSize = row.getIntField("stackSize", -1);
entry.color1 = row.getIntField("color1", -1);
entry.decal = row.getIntField("decal", -1);
entry.offsetGroupID = row.getIntField("offsetGroupID", -1);
entry.buildTypes = row.getIntField("buildTypes", -1);
entry.reqPrecondition = row.getStringField("reqPrecondition", "");
entry.animationFlag = row.getIntField("animationFlag", 0);
entry.equipEffects = row.getIntField("equipEffects", -1);
entry.readyForQA = row.getIntField("readyForQA", -1) == 1 ? true : false;
entry.itemRating = row.getIntField("itemRating", -1);
entry.isTwoHanded = row.getIntField("isTwoHanded", -1) == 1 ? true : false;
entry.minNumRequired = row.getIntField("minNumRequired", -1);
entry.delResIndex = row.getIntField("delResIndex", -1);
entry.currencyLOT = row.getIntField("currencyLOT", -1);
entry.altCurrencyCost = row.getIntField("altCurrencyCost", -1);
entry.subItems = row.getStringField("subItems", "");
UNUSED_COLUMN(entry.audioEventUse = row.getStringField("audioEventUse", ""));
entry.noEquipAnimation = row.getIntField("noEquipAnimation", -1) == 1 ? true : false;
entry.commendationLOT = row.getIntField("commendationLOT", -1);
entry.commendationCost = row.getIntField("commendationCost", -1);
UNUSED_COLUMN(entry.audioEquipMetaEventSet = row.getStringField("audioEquipMetaEventSet", ""));
entry.currencyCosts = row.getStringField("currencyCosts", "");
UNUSED_COLUMN(entry.ingredientInfo = row.getStringField("ingredientInfo", ""));
entry.locStatus = row.getIntField("locStatus", -1);
entry.forgeType = row.getIntField("forgeType", -1);
entry.SellMultiplier = row.getFloatField("SellMultiplier", -1.0f);
return entry;
}
}
tableSize.finalize();
void CDItemComponentTable::LoadValuesFromDatabase() {
// Now get the data
auto tableData = CDClientDatabase::ExecuteQuery("SELECT * FROM ItemComponent");
auto& entries = GetEntriesMutable();
while (!tableData.eof()) {
CDItemComponent entry;
entry.id = tableData.getIntField("id", -1);
entry.equipLocation = tableData.getStringField("equipLocation", "");
entry.baseValue = tableData.getIntField("baseValue", -1);
entry.isKitPiece = tableData.getIntField("isKitPiece", -1) == 1 ? true : false;
entry.rarity = tableData.getIntField("rarity", 0);
entry.itemType = tableData.getIntField("itemType", -1);
entry.itemInfo = tableData.getInt64Field("itemInfo", -1);
entry.inLootTable = tableData.getIntField("inLootTable", -1) == 1 ? true : false;
entry.inVendor = tableData.getIntField("inVendor", -1) == 1 ? true : false;
entry.isUnique = tableData.getIntField("isUnique", -1) == 1 ? true : false;
entry.isBOP = tableData.getIntField("isBOP", -1) == 1 ? true : false;
entry.isBOE = tableData.getIntField("isBOE", -1) == 1 ? true : false;
entry.reqFlagID = tableData.getIntField("reqFlagID", -1);
entry.reqSpecialtyID = tableData.getIntField("reqSpecialtyID", -1);
entry.reqSpecRank = tableData.getIntField("reqSpecRank", -1);
entry.reqAchievementID = tableData.getIntField("reqAchievementID", -1);
entry.stackSize = tableData.getIntField("stackSize", -1);
entry.color1 = tableData.getIntField("color1", -1);
entry.decal = tableData.getIntField("decal", -1);
entry.offsetGroupID = tableData.getIntField("offsetGroupID", -1);
entry.buildTypes = tableData.getIntField("buildTypes", -1);
entry.reqPrecondition = tableData.getStringField("reqPrecondition", "");
entry.animationFlag = tableData.getIntField("animationFlag", 0);
entry.equipEffects = tableData.getIntField("equipEffects", -1);
entry.readyForQA = tableData.getIntField("readyForQA", -1) == 1 ? true : false;
entry.itemRating = tableData.getIntField("itemRating", -1);
entry.isTwoHanded = tableData.getIntField("isTwoHanded", -1) == 1 ? true : false;
entry.minNumRequired = tableData.getIntField("minNumRequired", -1);
entry.delResIndex = tableData.getIntField("delResIndex", -1);
entry.currencyLOT = tableData.getIntField("currencyLOT", -1);
entry.altCurrencyCost = tableData.getIntField("altCurrencyCost", -1);
entry.subItems = tableData.getStringField("subItems", "");
UNUSED_COLUMN(entry.audioEventUse = tableData.getStringField("audioEventUse", ""));
entry.noEquipAnimation = tableData.getIntField("noEquipAnimation", -1) == 1 ? true : false;
entry.commendationLOT = tableData.getIntField("commendationLOT", -1);
entry.commendationCost = tableData.getIntField("commendationCost", -1);
UNUSED_COLUMN(entry.audioEquipMetaEventSet = tableData.getStringField("audioEquipMetaEventSet", ""));
entry.currencyCosts = tableData.getStringField("currencyCosts", "");
UNUSED_COLUMN(entry.ingredientInfo = tableData.getStringField("ingredientInfo", ""));
entry.locStatus = tableData.getIntField("locStatus", -1);
entry.forgeType = tableData.getIntField("forgeType", -1);
entry.SellMultiplier = tableData.getFloatField("SellMultiplier", -1.0f);
CDItemComponent entry = ReadEntry(tableData);
entries.insert(std::make_pair(entry.id, entry));
tableData.nextRow();
}
@@ -70,6 +69,43 @@ void CDItemComponentTable::LoadValuesFromDatabase() {
tableData.finalize();
}
bool CDItemComponentTable::LoadFromFdb() {
m_FdbTable = nullptr;
const auto* table = CDFdb::GetTable("ItemComponent");
if (!table) return false;
const auto changed = CDFdb::FindChangedKeys(*table);
if (!changed) return false;
// Ids whose rows CDServer.sqlite changes are read from it once and kept; everything else comes from the fdb
for (const auto id : *changed) LoadFromSqlite(static_cast<uint32_t>(id));
LOG("ItemComponent: reading from the fdb, %zu ids differ in CDServer.sqlite and are kept in memory", changed->size());
m_FdbTable = table;
return true;
}
const CDItemComponent& CDItemComponentTable::LoadFromSqlite(uint32_t id) {
auto& entries = GetEntriesMutable();
auto query = CDClientDatabase::CreatePreppedStmt("SELECT * FROM ItemComponent WHERE id = ?;");
query.bind(1, static_cast<int32_t>(id));
auto tableData = query.execQuery();
if (tableData.eof()) {
entries.insert(std::make_pair(id, Default));
return Default;
}
while (!tableData.eof()) {
CDItemComponent entry = ReadEntry(tableData);
entries.insert(std::make_pair(entry.id, entry));
tableData.nextRow();
}
const auto& it = entries.find(id);
return it != entries.end() ? it->second : Default;
}
const CDItemComponent& CDItemComponentTable::GetItemComponentByID(uint32_t skillID) {
auto& entries = GetEntriesMutable();
const auto& it = entries.find(skillID);
@@ -77,70 +113,17 @@ const CDItemComponent& CDItemComponentTable::GetItemComponentByID(uint32_t skill
return it->second;
}
auto query = CDClientDatabase::CreatePreppedStmt("SELECT * FROM ItemComponent WHERE id = ?;");
query.bind(1, static_cast<int32_t>(skillID));
if (!m_FdbTable) return LoadFromSqlite(skillID);
auto tableData = query.execQuery();
if (tableData.eof()) {
// Only the items asked for are kept in memory; the first row of an id wins, as in the SQLite path
const auto row = m_FdbTable->FindFirst(static_cast<int32_t>(skillID));
if (!row) {
entries.insert(std::make_pair(skillID, Default));
return Default;
}
while (!tableData.eof()) {
CDItemComponent entry;
entry.id = tableData.getIntField("id", -1);
entry.equipLocation = tableData.getStringField("equipLocation", "");
entry.baseValue = tableData.getIntField("baseValue", -1);
entry.isKitPiece = tableData.getIntField("isKitPiece", -1) == 1 ? true : false;
entry.rarity = tableData.getIntField("rarity", 0);
entry.itemType = tableData.getIntField("itemType", -1);
entry.itemInfo = tableData.getInt64Field("itemInfo", -1);
entry.inLootTable = tableData.getIntField("inLootTable", -1) == 1 ? true : false;
entry.inVendor = tableData.getIntField("inVendor", -1) == 1 ? true : false;
entry.isUnique = tableData.getIntField("isUnique", -1) == 1 ? true : false;
entry.isBOP = tableData.getIntField("isBOP", -1) == 1 ? true : false;
entry.isBOE = tableData.getIntField("isBOE", -1) == 1 ? true : false;
entry.reqFlagID = tableData.getIntField("reqFlagID", -1);
entry.reqSpecialtyID = tableData.getIntField("reqSpecialtyID", -1);
entry.reqSpecRank = tableData.getIntField("reqSpecRank", -1);
entry.reqAchievementID = tableData.getIntField("reqAchievementID", -1);
entry.stackSize = tableData.getIntField("stackSize", -1);
entry.color1 = tableData.getIntField("color1", -1);
entry.decal = tableData.getIntField("decal", -1);
entry.offsetGroupID = tableData.getIntField("offsetGroupID", -1);
entry.buildTypes = tableData.getIntField("buildTypes", -1);
entry.reqPrecondition = tableData.getStringField("reqPrecondition", "");
entry.animationFlag = tableData.getIntField("animationFlag", 0);
entry.equipEffects = tableData.getIntField("equipEffects", -1);
entry.readyForQA = tableData.getIntField("readyForQA", -1) == 1 ? true : false;
entry.itemRating = tableData.getIntField("itemRating", -1);
entry.isTwoHanded = tableData.getIntField("isTwoHanded", -1) == 1 ? true : false;
entry.minNumRequired = tableData.getIntField("minNumRequired", -1);
entry.delResIndex = tableData.getIntField("delResIndex", -1);
entry.currencyLOT = tableData.getIntField("currencyLOT", -1);
entry.altCurrencyCost = tableData.getIntField("altCurrencyCost", -1);
entry.subItems = tableData.getStringField("subItems", "");
UNUSED(entry.audioEventUse = tableData.getStringField("audioEventUse", ""));
entry.noEquipAnimation = tableData.getIntField("noEquipAnimation", -1) == 1 ? true : false;
entry.commendationLOT = tableData.getIntField("commendationLOT", -1);
entry.commendationCost = tableData.getIntField("commendationCost", -1);
UNUSED(entry.audioEquipMetaEventSet = tableData.getStringField("audioEquipMetaEventSet", ""));
entry.currencyCosts = tableData.getStringField("currencyCosts", "");
UNUSED(entry.ingredientInfo = tableData.getStringField("ingredientInfo", ""));
entry.locStatus = tableData.getIntField("locStatus", -1);
entry.forgeType = tableData.getIntField("forgeType", -1);
entry.SellMultiplier = tableData.getFloatField("SellMultiplier", -1.0f);
entries.insert(std::make_pair(entry.id, entry));
tableData.nextRow();
}
const auto& it2 = entries.find(skillID);
if (it2 != entries.end()) {
return it2->second;
}
return Default;
const CDFdb::RowFields fields(*m_FdbTable, *row);
return entries.insert(std::make_pair(skillID, ReadEntry(fields))).first->second;
}
std::map<LOT, uint32_t> CDItemComponentTable::ParseCraftingCurrencies(const CDItemComponent& itemComponent) {

View File

@@ -3,6 +3,7 @@
// Custom Classes
#include "CDTable.h"
#include "dCommonVars.h"
#include "FdbReader.h"
struct CDItemComponent {
uint32_t id; //!< The Component ID
@@ -52,10 +53,20 @@ struct CDItemComponent {
class CDItemComponentTable : public CDTable<CDItemComponentTable, std::map<uint32_t, CDItemComponent>> {
public:
void LoadValuesFromDatabase();
// Reads rows from the client's fdb from now on, if it is open; false keeps the table on CDServer.sqlite
bool LoadFromFdb();
static std::map<LOT, uint32_t> ParseCraftingCurrencies(const CDItemComponent& itemComponent);
// Gets an entry by ID
const CDItemComponent& GetItemComponentByID(uint32_t skillID);
static CDItemComponent Default;
private:
// Caches the entry of one id from CDServer.sqlite (Default when there is none) and returns it
const CDItemComponent& LoadFromSqlite(uint32_t id);
const FdbReader::Table* m_FdbTable = nullptr;
};

View File

@@ -1,21 +1,30 @@
#include "CDObjectsTable.h"
#include "CDFdb.h"
#include "Logger.h"
namespace {
CDObjects ObjDefault;
// Fills an entry from a CDServer.sqlite row or an fdb row (CDFdb::RowFields), which read alike
template<typename Row>
void ReadEntry(Row& row, CDObjects& entry) {
entry.name = row.getStringField("name", "");
UNUSED_COLUMN(entry.placeable = row.getIntField("placeable", -1);)
entry.type = row.getStringField("type", "");
UNUSED_COLUMN(entry.description = row.getStringField("description", "");)
UNUSED_COLUMN(entry.localize = row.getIntField("localize", -1);)
UNUSED_COLUMN(entry.npcTemplateID = row.getIntField("npcTemplateID", -1);)
UNUSED_COLUMN(entry.displayName = row.getStringField("displayName", "");)
entry.interactionDistance = row.getFloatField("interactionDistance", -1.0f);
UNUSED_COLUMN(entry.nametag = row.getIntField("nametag", -1);)
UNUSED_COLUMN(entry._internalNotes = row.getStringField("_internalNotes", "");)
UNUSED_COLUMN(entry.locStatus = row.getIntField("locStatus", -1);)
UNUSED_COLUMN(entry.gate_version = row.getStringField("gate_version", "");)
UNUSED_COLUMN(entry.HQ_valid = row.getIntField("HQ_valid", -1);)
}
};
void CDObjectsTable::LoadValuesFromDatabase() {
// First, get the size of the table
uint32_t size = 0;
auto tableSize = CDClientDatabase::ExecuteQuery("SELECT COUNT(*) FROM Objects");
while (!tableSize.eof()) {
size = tableSize.getIntField(0, 0);
tableSize.nextRow();
}
tableSize.finalize();
// Now get the data
auto tableData = CDClientDatabase::ExecuteQuery("SELECT * FROM Objects");
auto& entries = GetEntriesMutable();
@@ -24,19 +33,7 @@ void CDObjectsTable::LoadValuesFromDatabase() {
auto& entry = entries[lot];
entry.id = lot;
entry.name = tableData.getStringField("name", "");
UNUSED_COLUMN(entry.placeable = tableData.getIntField("placeable", -1);)
entry.type = tableData.getStringField("type", "");
UNUSED_COLUMN(entry.description = tableData.getStringField("description", "");)
UNUSED_COLUMN(entry.localize = tableData.getIntField("localize", -1);)
UNUSED_COLUMN(entry.npcTemplateID = tableData.getIntField("npcTemplateID", -1);)
UNUSED_COLUMN(entry.displayName = tableData.getStringField("displayName", "");)
entry.interactionDistance = tableData.getFloatField("interactionDistance", -1.0f);
UNUSED_COLUMN(entry.nametag = tableData.getIntField("nametag", -1);)
UNUSED_COLUMN(entry._internalNotes = tableData.getStringField("_internalNotes", "");)
UNUSED_COLUMN(entry.locStatus = tableData.getIntField("locStatus", -1);)
UNUSED_COLUMN(entry.gate_version = tableData.getStringField("gate_version", "");)
UNUSED_COLUMN(entry.HQ_valid = tableData.getIntField("HQ_valid", -1);)
ReadEntry(tableData, entry);
tableData.nextRow();
}
@@ -44,13 +41,24 @@ void CDObjectsTable::LoadValuesFromDatabase() {
ObjDefault.id = 0;
}
const CDObjects& CDObjectsTable::GetByID(const uint32_t lot) {
auto& entries = GetEntriesMutable();
const auto& it = entries.find(lot);
if (it != entries.end()) {
return it->second;
}
bool CDObjectsTable::LoadFromFdb() {
m_FdbTable = nullptr;
const auto* table = CDFdb::GetTable("Objects");
if (!table) return false;
const auto changed = CDFdb::FindChangedKeys(*table);
if (!changed) return false;
// Ids whose rows CDServer.sqlite changes are read from it once and kept; everything else comes from the fdb
for (const auto id : *changed) LoadFromSqlite(static_cast<uint32_t>(id));
LOG("Objects: reading from the fdb, %zu ids differ in CDServer.sqlite and are kept in memory", changed->size());
m_FdbTable = table;
return true;
}
const CDObjects& CDObjectsTable::LoadFromSqlite(const uint32_t lot) {
auto& entries = GetEntriesMutable();
auto query = CDClientDatabase::CreatePreppedStmt("SELECT * FROM Objects WHERE id = ?;");
query.bind(1, static_cast<int32_t>(lot));
@@ -62,33 +70,42 @@ const CDObjects& CDObjectsTable::GetByID(const uint32_t lot) {
// Now get the data
while (!tableData.eof()) {
const uint32_t lot = tableData.getIntField("id", 0);
const uint32_t rowLot = tableData.getIntField("id", 0);
auto& entry = entries[lot];
entry.id = lot;
entry.name = tableData.getStringField("name", "");
UNUSED(entry.placeable = tableData.getIntField("placeable", -1));
entry.type = tableData.getStringField("type", "");
UNUSED(entry.description = tableData.getStringField(4, ""));
UNUSED(entry.localize = tableData.getIntField("localize", -1));
UNUSED(entry.npcTemplateID = tableData.getIntField("npcTemplateID", -1));
UNUSED(entry.displayName = tableData.getStringField("displayName", ""));
entry.interactionDistance = tableData.getFloatField("interactionDistance", -1.0f);
UNUSED(entry.nametag = tableData.getIntField("nametag", -1));
UNUSED(entry._internalNotes = tableData.getStringField("_internalNotes", ""));
UNUSED(entry.locStatus = tableData.getIntField("locStatus", -1));
UNUSED(entry.gate_version = tableData.getStringField("gate_version", ""));
UNUSED(entry.HQ_valid = tableData.getIntField("HQ_valid", -1));
auto& entry = entries[rowLot];
entry.id = rowLot;
ReadEntry(tableData, entry);
tableData.nextRow();
}
tableData.finalize();
const auto& it2 = entries.find(lot);
if (it2 != entries.end()) {
return it2->second;
const auto& it = entries.find(lot);
return it != entries.end() ? it->second : ObjDefault;
}
const CDObjects& CDObjectsTable::GetByID(const uint32_t lot) {
auto& entries = GetEntriesMutable();
const auto& it = entries.find(lot);
if (it != entries.end()) {
return it->second;
}
return ObjDefault;
if (!m_FdbTable) return LoadFromSqlite(lot);
// Only the objects asked for are kept in memory; the last row of an id wins, as in the SQLite path
std::optional<CDObjects> found;
m_FdbTable->ForEachRowWithKey(static_cast<int32_t>(lot), [&](const FdbReader::Row& row) {
if (!found) found.emplace();
found->id = lot;
const CDFdb::RowFields fields(*m_FdbTable, row);
ReadEntry(fields, *found);
});
if (!found) {
entries.emplace(lot, ObjDefault);
return ObjDefault;
}
return entries.emplace(lot, std::move(*found)).first->second;
}

View File

@@ -5,6 +5,8 @@
#include <cstdint>
#include "FdbReader.h"
struct CDObjects {
uint32_t id; //!< The LOT of the object
std::string name; //!< The internal name of the object
@@ -25,7 +27,17 @@ struct CDObjects {
class CDObjectsTable : public CDTable<CDObjectsTable, std::map<uint32_t, CDObjects>> {
public:
void LoadValuesFromDatabase();
// Reads rows from the client's fdb from now on, if it is open; false keeps the table on CDServer.sqlite
bool LoadFromFdb();
// Gets an entry by ID
const CDObjects& GetByID(const uint32_t lot);
private:
// Caches the entry of one id from CDServer.sqlite (the default entry when there is none) and returns it
const CDObjects& LoadFromSqlite(const uint32_t lot);
const FdbReader::Table* m_FdbTable = nullptr;
};

View File

@@ -494,7 +494,7 @@ int main(int argc, char** argv) {
//Get CDClient initial information
try {
CDClientManager::LoadValuesFromDatabase();
CDClientManager::LoadValuesFromDatabase(Game::assetManager->GetResPath() / "cdclient.fdb");
} catch (CppSQLite3Exception& e) {
LOG("Failed to initialize CDServer SQLite Database");
LOG("May be caused by corrupted file: %s", (Game::assetManager->GetResPath() / "CDServer.sqlite").string().c_str());

View File

@@ -275,7 +275,8 @@ int main(int argc, char** argv) {
return EXIT_FAILURE;
}
CDClientManager::LoadValuesFromDatabase();
// The client's fdb, read in place and shared between all server processes, when there is one
CDClientManager::LoadValuesFromDatabase(Game::assetManager->GetResPath() / "cdclient.fdb");
Diagnostics::SetProduceMemoryDump(Game::config->GetValue("generate_dump") == "1");

View File

@@ -0,0 +1,304 @@
#include <gtest/gtest.h>
#include <cmath>
#include <cstdlib>
#include <filesystem>
#include <set>
#include <string>
#include <vector>
#include "CDClientDatabase.h"
#include "CDClientManager.h"
#include "CDComponentsRegistryTable.h"
#include "CDFdb.h"
#include "CDItemComponentTable.h"
#include "CDObjectsTable.h"
#include "Game.h"
#include "Logger.h"
#include "eReplicaComponentType.h"
#include "../FdbTestWriter.h"
using FdbTestWriter::Value;
namespace {
void ClearTables() {
CDClientManager::GetEntriesMutable<CDComponentsRegistryTable>().clear();
CDClientManager::GetEntriesMutable<CDItemComponentTable>().clear();
CDClientManager::GetEntriesMutable<CDObjectsTable>().clear();
}
int32_t Component(uint32_t id, int32_t type, int32_t defaultValue = -99) {
return CDComponentsRegistryTable::Instance().GetByIDAndType(id, static_cast<eReplicaComponentType>(type), defaultValue);
}
class CDFdbTestBase : public ::testing::Test {
protected:
void SetUp() override {
m_Logger = std::make_unique<Logger>("./testing.log", false, false);
Game::logger = m_Logger.get();
ClearTables();
}
void TearDown() override {
// Let go of the fdb before its file is removed; the tables stop using it on their next load
CDFdb::Close();
CDComponentsRegistryTable::Instance().LoadFromFdb();
CDItemComponentTable::Instance().LoadFromFdb();
CDObjectsTable::Instance().LoadFromFdb();
ClearTables();
Game::logger = nullptr;
}
std::unique_ptr<Logger> m_Logger;
};
// A ComponentsRegistry in both files, where CDServer.sqlite changes, drops and adds rows the way migrations do
class CDFdbOverlayTest : public CDFdbTestBase {
protected:
void SetUp() override {
CDFdbTestBase::SetUp();
m_Dir = std::filesystem::temp_directory_path() / ("dlu_cdfdb_" + std::string(::testing::UnitTest::GetInstance()->current_test_info()->name()));
std::filesystem::create_directories(m_Dir);
FdbTestWriter::Table registry;
registry.name = "ComponentsRegistry";
registry.columns = { { "id", eSqliteDataType::INT32 }, { "component_type", eSqliteDataType::INT32 }, { "component_id", eSqliteDataType::INT32 } };
registry.bucketCount = 4;
// 1, 5 and 9 share bucket 1
registry.rows = {
{ Value::Int(1), Value::Int(1), Value::Int(10) },
{ Value::Int(1), Value::Int(2), Value::Int(20) },
{ Value::Int(5), Value::Int(1), Value::Int(50) },
{ Value::Int(9), Value::Int(1), Value::Int(90) },
{ Value::Int(9), Value::Int(2), Value::Int(91) },
{ Value::Int(2), Value::Int(1), Value::Int(200) },
// A repeated (id, type): the last row wins
{ Value::Int(2), Value::Int(1), Value::Int(201) },
};
FdbTestWriter::WriteFile(m_Dir / "cdclient.fdb", FdbTestWriter::Write({ registry }));
std::filesystem::remove(m_Dir / "CDServer.sqlite");
CDClientDatabase::Connect((m_Dir / "CDServer.sqlite").string());
CDClientDatabase::ExecuteDML("CREATE TABLE ComponentsRegistry ('id' int32, 'component_type' int32, 'component_id' int32);");
CDClientDatabase::ExecuteDML(
"INSERT INTO ComponentsRegistry VALUES (1, 1, 10), (1, 2, 20), (5, 1, 55), (2, 1, 200), (2, 1, 201), (13, 1, 130);");
}
void TearDown() override {
CDFdbTestBase::TearDown();
std::error_code error;
std::filesystem::remove_all(m_Dir, error);
}
void ExpectRegistryAnswers() {
EXPECT_EQ(Component(1, 1), 10);
EXPECT_EQ(Component(1, 2), 20);
EXPECT_EQ(Component(1, 3), -99);
// Changed by CDServer.sqlite
EXPECT_EQ(Component(5, 1), 55);
// Dropped by CDServer.sqlite
EXPECT_EQ(Component(9, 1), -99);
EXPECT_EQ(Component(9, 2), -99);
// Added by CDServer.sqlite
EXPECT_EQ(Component(13, 1), 130);
EXPECT_EQ(Component(2, 1), 201);
EXPECT_EQ(Component(3, 1), -99);
}
std::filesystem::path m_Dir;
};
}
TEST_F(CDFdbOverlayTest, FindsTheKeysSqliteChanges) {
ASSERT_TRUE(CDFdb::Open(m_Dir / "cdclient.fdb"));
const auto* table = CDFdb::GetTable("ComponentsRegistry");
ASSERT_NE(table, nullptr);
const auto changed = CDFdb::FindChangedKeys(*table);
ASSERT_TRUE(changed.has_value());
EXPECT_EQ(std::set<int64_t>(changed->begin(), changed->end()), (std::set<int64_t>{ 5, 9, 13 }));
// Tables that aren't in both files, or whose columns differ, aren't read from the fdb
EXPECT_EQ(CDFdb::GetTable("Objects"), nullptr);
}
TEST_F(CDFdbOverlayTest, ReadsFromTheFdbWithSqliteChangesOnTop) {
ASSERT_TRUE(CDFdb::Open(m_Dir / "cdclient.fdb"));
ASSERT_TRUE(CDComponentsRegistryTable::Instance().LoadFromFdb());
// Only the changed ids are cached (an id marker plus its rows), not the table
EXPECT_EQ(CDClientManager::GetEntriesMutable<CDComponentsRegistryTable>().size(), 5u);
ExpectRegistryAnswers();
}
TEST_F(CDFdbOverlayTest, WithoutAnFdbReadsSqlite) {
EXPECT_FALSE(CDFdb::Open(m_Dir / "missing.fdb"));
EXPECT_EQ(CDFdb::Get(), nullptr);
EXPECT_FALSE(CDComponentsRegistryTable::Instance().LoadFromFdb());
// Lazy SQLite lookups, as before
ExpectRegistryAnswers();
// And the whole table in memory, as the server loads it without an fdb
ClearTables();
CDComponentsRegistryTable::Instance().LoadValuesFromDatabase();
ExpectRegistryAnswers();
}
TEST_F(CDFdbOverlayTest, UnmappedFdbReadsTheSame) {
ASSERT_TRUE(CDFdb::Open(m_Dir / "cdclient.fdb", false));
EXPECT_FALSE(CDFdb::Get()->IsMapped());
ASSERT_TRUE(CDComponentsRegistryTable::Instance().LoadFromFdb());
ExpectRegistryAnswers();
}
namespace {
std::vector<int64_t> DistinctIds(const std::string& table) {
std::vector<int64_t> ids;
auto query = CDClientDatabase::ExecuteQuery("SELECT DISTINCT id FROM " + table + ";");
while (!query.eof()) {
ids.push_back(query.getInt64Field(0));
query.nextRow();
}
return ids;
}
bool SameFloat(float a, float b) {
return a == b || (std::isnan(a) && std::isnan(b));
}
// Every id of the table in either file, plus some that are in neither
std::vector<uint32_t> AllIds(const std::string& name) {
std::set<uint32_t> ids{ 0, 0xFFFFFFFF, 99999999 };
for (const auto id : DistinctIds(name)) ids.insert(static_cast<uint32_t>(id));
if (const auto* table = CDFdb::Get() ? CDFdb::Get()->GetTable(name) : nullptr) {
table->ForEachRow([&](const FdbReader::Row& row) {
if (const auto key = FdbReader::Table::KeyOf(row)) ids.insert(static_cast<uint32_t>(*key));
});
}
return { ids.begin(), ids.end() };
}
struct Snapshot {
std::vector<int32_t> components;
std::vector<CDItemComponent> items;
std::vector<CDObjects> objects;
};
Snapshot TakeSnapshot(const std::vector<uint32_t>& registryIds, const std::vector<int32_t>& types,
const std::vector<uint32_t>& itemIds, const std::vector<uint32_t>& objectIds) {
Snapshot snapshot;
for (const auto id : registryIds) {
for (const auto type : types) snapshot.components.push_back(Component(id, type, -12345));
}
for (const auto id : itemIds) snapshot.items.push_back(CDItemComponentTable::Instance().GetItemComponentByID(id));
for (const auto id : objectIds) snapshot.objects.push_back(CDObjectsTable::Instance().GetByID(id));
return snapshot;
}
}
// Every row of the switched tables through the fdb against CDServer.sqlite, when both are around:
// DLU_CLIENT_RES is the client's res folder (with cdclient.fdb), DLU_CDSERVER_SQLITE the converted
// database (default: the source tree's build/resServer/CDServer.sqlite). Opened read-only.
TEST_F(CDFdbTestBase, RealClientParity) {
const char* res = std::getenv("DLU_CLIENT_RES");
if (!res) GTEST_SKIP() << "Set DLU_CLIENT_RES to a client res folder to compare against its cdclient.fdb";
const auto fdbPath = std::filesystem::path(res) / "cdclient.fdb";
if (!std::filesystem::exists(fdbPath)) GTEST_SKIP() << "No " << fdbPath.string();
std::string sqlitePath = std::string(PROJECT_SOURCE_DIR) + "/build/resServer/CDServer.sqlite";
if (const char* env = std::getenv("DLU_CDSERVER_SQLITE")) sqlitePath = env;
if (!std::filesystem::exists(sqlitePath)) GTEST_SKIP() << "No CDServer.sqlite at " << sqlitePath;
CDClientDatabase::Connect(sqlitePath);
CDClientDatabase::ExecuteDML("PRAGMA query_only = ON;"); // never write to the CDClient
ASSERT_TRUE(CDFdb::Open(fdbPath));
EXPECT_TRUE(CDFdb::Get()->IsMapped());
const auto registryIds = AllIds("ComponentsRegistry");
const auto itemIds = AllIds("ItemComponent");
const auto objectIds = AllIds("Objects");
std::vector<int32_t> types{ 999 };
for (const auto type : [] {
std::vector<int64_t> found;
auto query = CDClientDatabase::ExecuteQuery("SELECT DISTINCT component_type FROM ComponentsRegistry;");
while (!query.eof()) {
found.push_back(query.getInt64Field(0));
query.nextRow();
}
return found;
}()) types.push_back(static_cast<int32_t>(type));
for (const auto* name : { "ComponentsRegistry", "ItemComponent", "Objects" }) {
const auto* table = CDFdb::GetTable(name);
ASSERT_NE(table, nullptr) << name;
const auto changed = CDFdb::FindChangedKeys(*table);
ASSERT_TRUE(changed.has_value()) << name;
std::cout << name << ": " << changed->size() << " ids differ between the fdb and CDServer.sqlite" << std::endl;
}
// Through the fdb
ASSERT_TRUE(CDComponentsRegistryTable::Instance().LoadFromFdb());
ASSERT_TRUE(CDItemComponentTable::Instance().LoadFromFdb());
ASSERT_TRUE(CDObjectsTable::Instance().LoadFromFdb());
const auto fromFdb = TakeSnapshot(registryIds, types, itemIds, objectIds);
// Through CDServer.sqlite, loaded the way the server does without an fdb
CDFdb::Close();
ClearTables();
EXPECT_FALSE(CDComponentsRegistryTable::Instance().LoadFromFdb());
EXPECT_FALSE(CDItemComponentTable::Instance().LoadFromFdb());
EXPECT_FALSE(CDObjectsTable::Instance().LoadFromFdb());
CDComponentsRegistryTable::Instance().LoadValuesFromDatabase();
CDItemComponentTable::Instance().LoadValuesFromDatabase();
CDObjectsTable::Instance().LoadValuesFromDatabase();
const auto fromSqlite = TakeSnapshot(registryIds, types, itemIds, objectIds);
ASSERT_EQ(fromFdb.components.size(), fromSqlite.components.size());
uint32_t componentMismatches = 0;
for (size_t i = 0; i < fromFdb.components.size(); i++) {
if (fromFdb.components[i] != fromSqlite.components[i]) {
if (componentMismatches++ < 20) {
ADD_FAILURE() << "ComponentsRegistry id " << registryIds[i / types.size()] << " type " << types[i % types.size()]
<< ": fdb " << fromFdb.components[i] << " sqlite " << fromSqlite.components[i];
}
}
}
EXPECT_EQ(componentMismatches, 0u);
ASSERT_EQ(fromFdb.items.size(), fromSqlite.items.size());
uint32_t itemMismatches = 0;
for (size_t i = 0; i < fromFdb.items.size(); i++) {
const auto& a = fromFdb.items[i];
const auto& b = fromSqlite.items[i];
const bool same = a.id == b.id && a.equipLocation == b.equipLocation && a.baseValue == b.baseValue &&
a.isKitPiece == b.isKitPiece && a.rarity == b.rarity && a.itemType == b.itemType && a.itemInfo == b.itemInfo &&
a.inLootTable == b.inLootTable && a.inVendor == b.inVendor && a.isUnique == b.isUnique && a.isBOP == b.isBOP &&
a.isBOE == b.isBOE && a.reqFlagID == b.reqFlagID && a.reqSpecialtyID == b.reqSpecialtyID &&
a.reqSpecRank == b.reqSpecRank && a.reqAchievementID == b.reqAchievementID && a.stackSize == b.stackSize &&
a.color1 == b.color1 && a.decal == b.decal && a.offsetGroupID == b.offsetGroupID && a.buildTypes == b.buildTypes &&
a.reqPrecondition == b.reqPrecondition && a.animationFlag == b.animationFlag && a.equipEffects == b.equipEffects &&
a.readyForQA == b.readyForQA && a.itemRating == b.itemRating && a.isTwoHanded == b.isTwoHanded &&
a.minNumRequired == b.minNumRequired && a.delResIndex == b.delResIndex && a.currencyLOT == b.currencyLOT &&
a.altCurrencyCost == b.altCurrencyCost && a.subItems == b.subItems && a.noEquipAnimation == b.noEquipAnimation &&
a.commendationLOT == b.commendationLOT && a.commendationCost == b.commendationCost &&
a.currencyCosts == b.currencyCosts && a.locStatus == b.locStatus && a.forgeType == b.forgeType &&
SameFloat(a.SellMultiplier, b.SellMultiplier);
if (!same && itemMismatches++ < 20) ADD_FAILURE() << "ItemComponent id " << itemIds[i] << " differs";
}
EXPECT_EQ(itemMismatches, 0u);
ASSERT_EQ(fromFdb.objects.size(), fromSqlite.objects.size());
uint32_t objectMismatches = 0;
for (size_t i = 0; i < fromFdb.objects.size(); i++) {
const auto& a = fromFdb.objects[i];
const auto& b = fromSqlite.objects[i];
const bool same = a.id == b.id && a.name == b.name && a.type == b.type && SameFloat(a.interactionDistance, b.interactionDistance);
if (!same && objectMismatches++ < 20) ADD_FAILURE() << "Objects id " << objectIds[i] << " differs: fdb '" << a.name << "' sqlite '" << b.name << "'";
}
EXPECT_EQ(objectMismatches, 0u);
std::cout << "Compared " << registryIds.size() << " ComponentsRegistry ids x " << types.size() << " types, "
<< itemIds.size() << " ItemComponent ids, " << objectIds.size() << " Objects ids" << std::endl;
}

View File

@@ -27,6 +27,7 @@ set(DGAMETEST_SOURCES
"PreconditionItemCostTests.cpp"
"SpiderQueenTests.cpp"
"TacArcTests.cpp"
"CDFdbTests.cpp"
)
add_subdirectory(dComponentsTests)