feat(cdclient): open the client's fdb next to CDServer.sqlite

CDFdb opens cdclient.fdb once per process and hands out a table when its
columns match the CDServer.sqlite table of the same name. CDServer.sqlite
stays the source of truth: the CDServer migrations change a few rows, so
FindChangedKeys compares both files row by row (a hash per row, summed
per key) and returns the keys that differ, for the tables to read from
SQLite. RowFields reads an fdb row with CppSQLite3Query's accessors and
defaults, so a table fills its entries from either file with one piece
of code.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dDatabase/CDClientDatabase/CDFdb.cpp

@@ -0,0 +1,230 @@
+#include "CDFdb.h"
+
+#include <cstring>
+#include <memory>
+
+#include "CDClientDatabase.h"
+
+namespace {
+	std::unique_ptr<FdbReader> g_Fdb;
+
+	// Column name -> index for each table handed out by GetTable, so RowFields looks up names in O(1)
+	std::unordered_map<const FdbReader::Table*, std::unordered_map<std::string, int32_t>> g_ColumnIndices;
+
+	constexpr uint64_t FNV_OFFSET = 14695981039346656037ULL;
+	constexpr uint64_t FNV_PRIME = 1099511628211ULL;
+
+	// A value hash that is the same for a value read from either file
+	class ValueHasher {
+	public:
+		void Null() { Byte(0); }
+
+		void Integer(int64_t value) {
+			Byte(1);
+			const auto bits = static_cast<uint64_t>(value);
+			for (uint32_t i = 0; i < 8; i++) Byte(static_cast<uint8_t>(bits >> (i * 8)));
+		}
+
+		void Real(float value) {
+			Byte(2);
+			uint32_t bits = 0;
+			std::memcpy(&bits, &value, sizeof(bits));
+			for (uint32_t i = 0; i < 4; i++) Byte(static_cast<uint8_t>(bits >> (i * 8)));
+		}
+
+		void Text(std::string_view value) {
+			Byte(3);
+			for (const char c : value) Byte(static_cast<uint8_t>(c));
+			// Length ends the value so ("ab","c") and ("a","bc") differ
+			Integer(static_cast<int64_t>(value.size()));
+		}
+
+		// splitmix64 finish so row hashes summed per key don't cancel out by accident
+		[[nodiscard]] uint64_t Finish() const {
+			uint64_t z = m_Hash + 0x9E3779B97F4A7C15ULL;
+			z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL;
+			z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL;
+			return z ^ (z >> 31);
+		}
+
+	private:
+		void Byte(uint8_t byte) {
+			m_Hash ^= byte;
+			m_Hash *= FNV_PRIME;
+		}
+
+		uint64_t m_Hash = FNV_OFFSET;
+	};
+
+	uint64_t HashFdbRow(const FdbReader::Row& row, uint32_t columnCount) {
+		ValueHasher hasher;
+		for (uint32_t c = 0; c < columnCount; c++) {
+			switch (row.GetType(c)) {
+			case eSqliteDataType::INT32:
+			case eSqliteDataType::INT_BOOL:
+			case eSqliteDataType::INT64:
+				hasher.Integer(row.GetInt64(c));
+				break;
+			case eSqliteDataType::REAL:
+				hasher.Real(row.GetFloat(c));
+				break;
+			case eSqliteDataType::TEXT_4:
+			case eSqliteDataType::TEXT_8:
+				hasher.Text(row.GetString(c));
+				break;
+			default:
+				hasher.Null();
+				break;
+			}
+		}
+		return hasher.Finish();
+	}
+
+	uint64_t HashSqliteRow(CppSQLite3Query& query, int32_t columnCount) {
+		ValueHasher hasher;
+		for (int32_t c = 0; c < columnCount; c++) {
+			switch (query.fieldDataType(c)) {
+			case SQLITE_INTEGER:
+				hasher.Integer(query.getInt64Field(c));
+				break;
+			case SQLITE_FLOAT:
+				// The fdb holds 32-bit floats; the conversion printed them exactly, so this rounds back to the same float
+				hasher.Real(static_cast<float>(query.getFloatField(c)));
+				break;
+			case SQLITE_TEXT:
+				hasher.Text(query.getStringField(c));
+				break;
+			case SQLITE_NULL:
+				hasher.Null();
+				break;
+			default: {
+				// Blobs never come out of the conversion; hash them apart from text
+				int length = 0;
+				const auto* blob = query.getBlobField(c, length);
+				hasher.Integer(-1);
+				hasher.Text(std::string_view(reinterpret_cast<const char*>(blob), blob ? static_cast<uint32_t>(length) : 0));
+				break;
+			}
+			}
+		}
+		return hasher.Finish();
+	}
+}
+
+bool CDFdb::Open(const std::filesystem::path& path, bool allowMapping) {
+	Close();
+	std::error_code error;
+	if (path.empty() || !std::filesystem::is_regular_file(path, error)) return false;
+
+	auto reader = std::make_unique<FdbReader>();
+	if (!reader->Open(path, allowMapping)) return false;
+	g_Fdb = std::move(reader);
+	return true;
+}
+
+void CDFdb::Close() {
+	g_ColumnIndices.clear();
+	g_Fdb.reset();
+}
+
+const FdbReader* CDFdb::Get() {
+	return g_Fdb.get();
+}
+
+const FdbReader::Table* CDFdb::GetTable(const std::string& name) {
+	if (!g_Fdb || !CDClientDatabase::isConnected) return nullptr;
+	const auto* table = g_Fdb->GetTable(name);
+	if (!table || table->GetColumns().empty()) return nullptr;
+
+	const auto firstType = table->GetColumns()[0].type;
+	if (firstType != eSqliteDataType::INT32 && firstType != eSqliteDataType::INT64) return nullptr;
+
+	// Same columns in the same order as the SQLite table
+	try {
+		auto query = CDClientDatabase::ExecuteQuery("SELECT * FROM \"" + name + "\" LIMIT 0;");
+		const auto& columns = table->GetColumns();
+		if (query.numFields() != static_cast<int>(columns.size())) return nullptr;
+		for (int32_t i = 0; i < query.numFields(); i++) {
+			if (columns[i].name != query.fieldName(i)) return nullptr;
+		}
+	} catch (const CppSQLite3Exception&) {
+		return nullptr;
+	}
+
+	auto& indices = g_ColumnIndices[table];
+	if (indices.empty()) {
+		for (uint32_t i = 0; i < table->GetColumns().size(); i++) indices.emplace(table->GetColumns()[i].name, static_cast<int32_t>(i));
+	}
+	return table;
+}
+
+std::optional<std::vector<int64_t>> CDFdb::FindChangedKeys(const FdbReader::Table& table) {
+	if (!CDClientDatabase::isConnected) return std::nullopt;
+	const auto columnCount = static_cast<uint32_t>(table.GetColumns().size());
+
+	// Per key: the sum of the fdb row hashes minus the sum of the SQLite row hashes
+	std::unordered_map<int64_t, uint64_t> balance;
+	balance.reserve(table.GetBucketCount());
+
+	table.ForEachRow([&](const FdbReader::Row& row) {
+		const auto key = FdbReader::Table::KeyOf(row);
+		// A row without an integer key can't be looked up by key; count it so the table isn't used
+		balance[key.value_or(INT64_MIN)] += HashFdbRow(row, columnCount);
+	});
+
+	try {
+		auto query = CDClientDatabase::ExecuteQuery("SELECT * FROM \"" + table.GetName() + "\";");
+		if (query.numFields() != static_cast<int>(columnCount)) return std::nullopt;
+		while (!query.eof()) {
+			const int64_t key = query.fieldDataType(0) == SQLITE_INTEGER ? query.getInt64Field(0) : INT64_MIN;
+			balance[key] -= HashSqliteRow(query, static_cast<int32_t>(columnCount));
+			query.nextRow();
+		}
+		query.finalize();
+	} catch (const CppSQLite3Exception&) {
+		return std::nullopt;
+	}
+
+	std::vector<int64_t> changed;
+	for (const auto& [key, sum] : balance) {
+		if (sum == 0) continue;
+		if (key == INT64_MIN) return std::nullopt;
+		changed.push_back(key);
+	}
+	return changed;
+}
+
+CDFdb::RowFields::RowFields(const FdbReader::Table& table, const FdbReader::Row& row) : m_Table(table), m_Row(row) {}
+
+int32_t CDFdb::RowFields::Column(const char* field) const {
+	const auto table = g_ColumnIndices.find(&m_Table);
+	if (table == g_ColumnIndices.end()) return m_Table.GetColumnIndex(field);
+	const auto it = table->second.find(field);
+	return it == table->second.end() ? -1 : it->second;
+}
+
+int CDFdb::RowFields::getIntField(const char* field, int nullValue) const {
+	const auto column = Column(field);
+	return column < 0 ? nullValue : m_Row.GetInt(static_cast<uint32_t>(column), nullValue);
+}
+
+int64_t CDFdb::RowFields::getInt64Field(const char* field, int64_t nullValue) const {
+	const auto column = Column(field);
+	return column < 0 ? nullValue : m_Row.GetInt64(static_cast<uint32_t>(column), nullValue);
+}
+
+double CDFdb::RowFields::getFloatField(const char* field, double nullValue) const {
+	const auto column = Column(field);
+	if (column < 0 || m_Row.IsNull(static_cast<uint32_t>(column))) return nullValue;
+	return m_Row.GetFloat(static_cast<uint32_t>(column));
+}
+
+std::string CDFdb::RowFields::getStringField(const char* field, const char* nullValue) const {
+	const auto column = Column(field);
+	return column < 0 ? std::string(nullValue) : m_Row.GetString(static_cast<uint32_t>(column), nullValue);
+}
+
+bool CDFdb::RowFields::fieldIsNull(const char* field) const {
+	const auto column = Column(field);
+	return column < 0 || m_Row.IsNull(static_cast<uint32_t>(column));
+}

dDatabase/CDClientDatabase/CDFdb.h

@@ -0,0 +1,72 @@
+#ifndef CDFDB_H
+#define CDFDB_H
+
+#include <cstdint>
+#include <filesystem>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <vector>
+
+#include "FdbReader.h"
+
+/**
+ * Reading CDClient tables straight from the client's cdclient.fdb.
+ *
+ * The fdb is mapped read-only, so every server process on the machine shares one copy of it
+ * through the OS page cache, and a row is found through the fdb's own hash buckets instead of
+ * a per-process cache of the whole table.
+ *
+ * CDServer.sqlite stays the source of truth: the server's CDServer migrations change a few rows
+ * of it, so at load each table compares its rows in both files and reads the keys that differ
+ * from SQLite (see FindChangedKeys).
+ */
+namespace CDFdb {
+	/**
+	 * Opens the fdb. Does nothing and returns false if the file is missing or unreadable, and
+	 * then the tables keep reading CDServer.sqlite as before.
+	 */
+	bool Open(const std::filesystem::path& path, bool allowMapping = true);
+	void Close();
+
+	// The open fdb, or nullptr
+	const FdbReader* Get();
+
+	/**
+	 * The fdb table that can stand in for the SQLite table of the same name: it exists in both,
+	 * has the same columns in the same order, and its first column is an integer. nullptr otherwise.
+	 */
+	const FdbReader::Table* GetTable(const std::string& name);
+
+	/**
+	 * The first-column keys whose rows differ between the fdb table and the SQLite table of the
+	 * same name (values compared by type and content, rows as a set per key). Keys only in one of
+	 * the two are included. nullopt if the tables can't be compared.
+	 */
+	std::optional<std::vector<int64_t>> FindChangedKeys(const FdbReader::Table& table);
+
+	/**
+	 * Reads one fdb row with the same accessors, defaults and conversions as CppSQLite3Query, so
+	 * a table can fill its entries from either source with the same code.
+	 */
+	class RowFields {
+	public:
+		RowFields(const FdbReader::Table& table, const FdbReader::Row& row);
+
+		int getIntField(const char* field, int nullValue = 0) const;
+		int64_t getInt64Field(const char* field, int64_t nullValue = 0) const;
+		double getFloatField(const char* field, double nullValue = 0.0) const;
+		std::string getStringField(const char* field, const char* nullValue = "") const;
+		bool fieldIsNull(const char* field) const;
+
+	private:
+		// -1 when the table has no such column, which reads as null
+		int32_t Column(const char* field) const;
+
+		const FdbReader::Table& m_Table;
+		const FdbReader::Row& m_Row;
+	};
+};
+
+#endif // CDFDB_H

dDatabase/CDClientDatabase/CMakeLists.txt

@@ -1,6 +1,7 @@
 set(DDATABASE_CDCLIENTDATABASE_SOURCES
 	"CDClientDatabase.cpp"
 	"CDClientManager.cpp"
+	"CDFdb.cpp"
 )
 
 add_subdirectory(CDClientTables)
@@ -15,7 +16,8 @@ target_include_directories(dDatabaseCDClient PUBLIC "."
 	"${PROJECT_SOURCE_DIR}/dCommon"
 	"${PROJECT_SOURCE_DIR}/dCommon/dEnums"
 )
-target_link_libraries(dDatabaseCDClient PRIVATE sqlite3 glm::glm)
+# FdbReader lives in dCommon
+target_link_libraries(dDatabaseCDClient PRIVATE sqlite3 glm::glm dCommon)
 
 if (${CDCLIENT_CACHE_ALL})
 	add_compile_definitions(dDatabaseCDClient PRIVATE CDCLIENT_CACHE_ALL=${CDCLIENT_CACHE_ALL})

tests/dDatabaseTests/CMakeLists.txt

@@ -27,6 +27,6 @@ target_include_directories(dDatabaseSqliteTests PRIVATE
 )
 
 # dCommon uses raknet without linking it: link them as a group so the order of the static libraries does not matter
-target_link_libraries(dDatabaseSqliteTests glm::glm "dNet" "magic_enum" "$<LINK_GROUP:RESCAN,dDatabase,dDatabaseGame,dCommon,raknet>" MariaDB::ConnCpp sqlite3 GTest::gtest_main ${CMAKE_DL_LIBS})
+target_link_libraries(dDatabaseSqliteTests glm::glm "dNet" "magic_enum" "$<LINK_GROUP:RESCAN,dDatabase,dDatabaseGame,dDatabaseCDClient,dCommon,raknet>" MariaDB::ConnCpp sqlite3 GTest::gtest_main ${CMAKE_DL_LIBS})
 
 gtest_discover_tests(dDatabaseSqliteTests)