diff --git a/dDatabase/CDClientDatabase/CDFdb.cpp b/dDatabase/CDClientDatabase/CDFdb.cpp new file mode 100644 index 000000000..cabcf9335 --- /dev/null +++ b/dDatabase/CDClientDatabase/CDFdb.cpp @@ -0,0 +1,230 @@ +#include "CDFdb.h" + +#include +#include + +#include "CDClientDatabase.h" + +namespace { + std::unique_ptr g_Fdb; + + // Column name -> index for each table handed out by GetTable, so RowFields looks up names in O(1) + std::unordered_map> g_ColumnIndices; + + constexpr uint64_t FNV_OFFSET = 14695981039346656037ULL; + constexpr uint64_t FNV_PRIME = 1099511628211ULL; + + // A value hash that is the same for a value read from either file + class ValueHasher { + public: + void Null() { Byte(0); } + + void Integer(int64_t value) { + Byte(1); + const auto bits = static_cast(value); + for (uint32_t i = 0; i < 8; i++) Byte(static_cast(bits >> (i * 8))); + } + + void Real(float value) { + Byte(2); + uint32_t bits = 0; + std::memcpy(&bits, &value, sizeof(bits)); + for (uint32_t i = 0; i < 4; i++) Byte(static_cast(bits >> (i * 8))); + } + + void Text(std::string_view value) { + Byte(3); + for (const char c : value) Byte(static_cast(c)); + // Length ends the value so ("ab","c") and ("a","bc") differ + Integer(static_cast(value.size())); + } + + // splitmix64 finish so row hashes summed per key don't cancel out by accident + [[nodiscard]] uint64_t Finish() const { + uint64_t z = m_Hash + 0x9E3779B97F4A7C15ULL; + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL; + return z ^ (z >> 31); + } + + private: + void Byte(uint8_t byte) { + m_Hash ^= byte; + m_Hash *= FNV_PRIME; + } + + uint64_t m_Hash = FNV_OFFSET; + }; + + uint64_t HashFdbRow(const FdbReader::Row& row, uint32_t columnCount) { + ValueHasher hasher; + for (uint32_t c = 0; c < columnCount; c++) { + switch (row.GetType(c)) { + case eSqliteDataType::INT32: + case eSqliteDataType::INT_BOOL: + case eSqliteDataType::INT64: + hasher.Integer(row.GetInt64(c)); + break; + case eSqliteDataType::REAL: + hasher.Real(row.GetFloat(c)); + break; + case eSqliteDataType::TEXT_4: + case eSqliteDataType::TEXT_8: + hasher.Text(row.GetString(c)); + break; + default: + hasher.Null(); + break; + } + } + return hasher.Finish(); + } + + uint64_t HashSqliteRow(CppSQLite3Query& query, int32_t columnCount) { + ValueHasher hasher; + for (int32_t c = 0; c < columnCount; c++) { + switch (query.fieldDataType(c)) { + case SQLITE_INTEGER: + hasher.Integer(query.getInt64Field(c)); + break; + case SQLITE_FLOAT: + // The fdb holds 32-bit floats; the conversion printed them exactly, so this rounds back to the same float + hasher.Real(static_cast(query.getFloatField(c))); + break; + case SQLITE_TEXT: + hasher.Text(query.getStringField(c)); + break; + case SQLITE_NULL: + hasher.Null(); + break; + default: { + // Blobs never come out of the conversion; hash them apart from text + int length = 0; + const auto* blob = query.getBlobField(c, length); + hasher.Integer(-1); + hasher.Text(std::string_view(reinterpret_cast(blob), blob ? static_cast(length) : 0)); + break; + } + } + } + return hasher.Finish(); + } +} + +bool CDFdb::Open(const std::filesystem::path& path, bool allowMapping) { + Close(); + std::error_code error; + if (path.empty() || !std::filesystem::is_regular_file(path, error)) return false; + + auto reader = std::make_unique(); + if (!reader->Open(path, allowMapping)) return false; + g_Fdb = std::move(reader); + return true; +} + +void CDFdb::Close() { + g_ColumnIndices.clear(); + g_Fdb.reset(); +} + +const FdbReader* CDFdb::Get() { + return g_Fdb.get(); +} + +const FdbReader::Table* CDFdb::GetTable(const std::string& name) { + if (!g_Fdb || !CDClientDatabase::isConnected) return nullptr; + const auto* table = g_Fdb->GetTable(name); + if (!table || table->GetColumns().empty()) return nullptr; + + const auto firstType = table->GetColumns()[0].type; + if (firstType != eSqliteDataType::INT32 && firstType != eSqliteDataType::INT64) return nullptr; + + // Same columns in the same order as the SQLite table + try { + auto query = CDClientDatabase::ExecuteQuery("SELECT * FROM \"" + name + "\" LIMIT 0;"); + const auto& columns = table->GetColumns(); + if (query.numFields() != static_cast(columns.size())) return nullptr; + for (int32_t i = 0; i < query.numFields(); i++) { + if (columns[i].name != query.fieldName(i)) return nullptr; + } + } catch (const CppSQLite3Exception&) { + return nullptr; + } + + auto& indices = g_ColumnIndices[table]; + if (indices.empty()) { + for (uint32_t i = 0; i < table->GetColumns().size(); i++) indices.emplace(table->GetColumns()[i].name, static_cast(i)); + } + return table; +} + +std::optional> CDFdb::FindChangedKeys(const FdbReader::Table& table) { + if (!CDClientDatabase::isConnected) return std::nullopt; + const auto columnCount = static_cast(table.GetColumns().size()); + + // Per key: the sum of the fdb row hashes minus the sum of the SQLite row hashes + std::unordered_map balance; + balance.reserve(table.GetBucketCount()); + + table.ForEachRow([&](const FdbReader::Row& row) { + const auto key = FdbReader::Table::KeyOf(row); + // A row without an integer key can't be looked up by key; count it so the table isn't used + balance[key.value_or(INT64_MIN)] += HashFdbRow(row, columnCount); + }); + + try { + auto query = CDClientDatabase::ExecuteQuery("SELECT * FROM \"" + table.GetName() + "\";"); + if (query.numFields() != static_cast(columnCount)) return std::nullopt; + while (!query.eof()) { + const int64_t key = query.fieldDataType(0) == SQLITE_INTEGER ? query.getInt64Field(0) : INT64_MIN; + balance[key] -= HashSqliteRow(query, static_cast(columnCount)); + query.nextRow(); + } + query.finalize(); + } catch (const CppSQLite3Exception&) { + return std::nullopt; + } + + std::vector changed; + for (const auto& [key, sum] : balance) { + if (sum == 0) continue; + if (key == INT64_MIN) return std::nullopt; + changed.push_back(key); + } + return changed; +} + +CDFdb::RowFields::RowFields(const FdbReader::Table& table, const FdbReader::Row& row) : m_Table(table), m_Row(row) {} + +int32_t CDFdb::RowFields::Column(const char* field) const { + const auto table = g_ColumnIndices.find(&m_Table); + if (table == g_ColumnIndices.end()) return m_Table.GetColumnIndex(field); + const auto it = table->second.find(field); + return it == table->second.end() ? -1 : it->second; +} + +int CDFdb::RowFields::getIntField(const char* field, int nullValue) const { + const auto column = Column(field); + return column < 0 ? nullValue : m_Row.GetInt(static_cast(column), nullValue); +} + +int64_t CDFdb::RowFields::getInt64Field(const char* field, int64_t nullValue) const { + const auto column = Column(field); + return column < 0 ? nullValue : m_Row.GetInt64(static_cast(column), nullValue); +} + +double CDFdb::RowFields::getFloatField(const char* field, double nullValue) const { + const auto column = Column(field); + if (column < 0 || m_Row.IsNull(static_cast(column))) return nullValue; + return m_Row.GetFloat(static_cast(column)); +} + +std::string CDFdb::RowFields::getStringField(const char* field, const char* nullValue) const { + const auto column = Column(field); + return column < 0 ? std::string(nullValue) : m_Row.GetString(static_cast(column), nullValue); +} + +bool CDFdb::RowFields::fieldIsNull(const char* field) const { + const auto column = Column(field); + return column < 0 || m_Row.IsNull(static_cast(column)); +} diff --git a/dDatabase/CDClientDatabase/CDFdb.h b/dDatabase/CDClientDatabase/CDFdb.h new file mode 100644 index 000000000..dcc19e800 --- /dev/null +++ b/dDatabase/CDClientDatabase/CDFdb.h @@ -0,0 +1,72 @@ +#ifndef CDFDB_H +#define CDFDB_H + +#include +#include +#include +#include +#include +#include +#include + +#include "FdbReader.h" + +/** + * Reading CDClient tables straight from the client's cdclient.fdb. + * + * The fdb is mapped read-only, so every server process on the machine shares one copy of it + * through the OS page cache, and a row is found through the fdb's own hash buckets instead of + * a per-process cache of the whole table. + * + * CDServer.sqlite stays the source of truth: the server's CDServer migrations change a few rows + * of it, so at load each table compares its rows in both files and reads the keys that differ + * from SQLite (see FindChangedKeys). + */ +namespace CDFdb { + /** + * Opens the fdb. Does nothing and returns false if the file is missing or unreadable, and + * then the tables keep reading CDServer.sqlite as before. + */ + bool Open(const std::filesystem::path& path, bool allowMapping = true); + void Close(); + + // The open fdb, or nullptr + const FdbReader* Get(); + + /** + * The fdb table that can stand in for the SQLite table of the same name: it exists in both, + * has the same columns in the same order, and its first column is an integer. nullptr otherwise. + */ + const FdbReader::Table* GetTable(const std::string& name); + + /** + * The first-column keys whose rows differ between the fdb table and the SQLite table of the + * same name (values compared by type and content, rows as a set per key). Keys only in one of + * the two are included. nullopt if the tables can't be compared. + */ + std::optional> FindChangedKeys(const FdbReader::Table& table); + + /** + * Reads one fdb row with the same accessors, defaults and conversions as CppSQLite3Query, so + * a table can fill its entries from either source with the same code. + */ + class RowFields { + public: + RowFields(const FdbReader::Table& table, const FdbReader::Row& row); + + int getIntField(const char* field, int nullValue = 0) const; + int64_t getInt64Field(const char* field, int64_t nullValue = 0) const; + double getFloatField(const char* field, double nullValue = 0.0) const; + std::string getStringField(const char* field, const char* nullValue = "") const; + bool fieldIsNull(const char* field) const; + + private: + // -1 when the table has no such column, which reads as null + int32_t Column(const char* field) const; + + const FdbReader::Table& m_Table; + const FdbReader::Row& m_Row; + }; +}; + +#endif // CDFDB_H diff --git a/dDatabase/CDClientDatabase/CMakeLists.txt b/dDatabase/CDClientDatabase/CMakeLists.txt index 9aca3bda7..a9e3b004f 100644 --- a/dDatabase/CDClientDatabase/CMakeLists.txt +++ b/dDatabase/CDClientDatabase/CMakeLists.txt @@ -1,6 +1,7 @@ set(DDATABASE_CDCLIENTDATABASE_SOURCES "CDClientDatabase.cpp" "CDClientManager.cpp" + "CDFdb.cpp" ) add_subdirectory(CDClientTables) @@ -15,7 +16,8 @@ target_include_directories(dDatabaseCDClient PUBLIC "." "${PROJECT_SOURCE_DIR}/dCommon" "${PROJECT_SOURCE_DIR}/dCommon/dEnums" ) -target_link_libraries(dDatabaseCDClient PRIVATE sqlite3 glm::glm) +# FdbReader lives in dCommon +target_link_libraries(dDatabaseCDClient PRIVATE sqlite3 glm::glm dCommon) if (${CDCLIENT_CACHE_ALL}) add_compile_definitions(dDatabaseCDClient PRIVATE CDCLIENT_CACHE_ALL=${CDCLIENT_CACHE_ALL}) diff --git a/tests/dDatabaseTests/CMakeLists.txt b/tests/dDatabaseTests/CMakeLists.txt index 3dd4e65bb..39d5543a9 100644 --- a/tests/dDatabaseTests/CMakeLists.txt +++ b/tests/dDatabaseTests/CMakeLists.txt @@ -27,6 +27,6 @@ target_include_directories(dDatabaseSqliteTests PRIVATE ) # dCommon uses raknet without linking it: link them as a group so the order of the static libraries does not matter -target_link_libraries(dDatabaseSqliteTests glm::glm "dNet" "magic_enum" "$" MariaDB::ConnCpp sqlite3 GTest::gtest_main ${CMAKE_DL_LIBS}) +target_link_libraries(dDatabaseSqliteTests glm::glm "dNet" "magic_enum" "$" MariaDB::ConnCpp sqlite3 GTest::gtest_main ${CMAKE_DL_LIBS}) gtest_discover_tests(dDatabaseSqliteTests)