Mirrored_Repos/DarkflameServer
1
0
Fork 0
mirror of https://github.com/DarkflameUniverse/DarkflameServer.git synced 2026-10-02 02:43:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(dCommon): read cdclient.fdb in place through its hash buckets

Browse Source 
FdbMappedFile maps a file read-only (CreateFileMapping/MapViewOfFile on
Windows, mmap elsewhere) and falls back to reading it into memory when
mapping fails. FdbReader reads the table and column headers from it and
looks rows up by their first column through the fdb's own buckets,
decoding every integer as little-endian with bounds checks, so the rows
never get copied out of the file.

Tests write small fdb files (collisions, text, int64, nulls) and read
them both mapped and from memory.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-09-29 21:56:18 -05:00
parent 94b411a32e
commit d670484b68
8 changed files with 1020 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
tests/FdbTestWriter.h Normal file
View File

@@ -0,0 +1,157 @@
#ifndef FDBTESTWRITER_H
#define FDBTESTWRITER_H
// Writes small fdb files for tests, in the client's layout (see FdbReader.h)
#include <cstdint>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <string>
#include <vector>
#include "eSqliteDataType.h"
namespace FdbTestWriter {
struct Value {
eSqliteDataType type = eSqliteDataType::NONE;
int64_t integer = 0;
float real = 0.0f;
std::string text; // latin-1 bytes
static Value Null() { return {}; }
static Value Int(int32_t v) { Value r; r.type = eSqliteDataType::INT32; r.integer = v; return r; }
static Value Bool(bool v) { Value r; r.type = eSqliteDataType::INT_BOOL; r.integer = v ? 1 : 0; return r; }
static Value Int64(int64_t v) { Value r; r.type = eSqliteDataType::INT64; r.integer = v; return r; }
static Value Real(float v) { Value r; r.type = eSqliteDataType::REAL; r.real = v; return r; }
static Value Text(std::string v, bool wide = false) {
Value r; r.type = wide ? eSqliteDataType::TEXT_8 : eSqliteDataType::TEXT_4; r.text = std::move(v); return r;
}
};
struct Column {
std::string name;
eSqliteDataType type;
};
struct Table {
std::string name;
std::vector<Column> columns;
uint32_t bucketCount = 0;
std::vector<std::vector<Value>> rows;
};
class Buffer {
public:
uint32_t Alloc(uint32_t size) {
const auto offset = static_cast<uint32_t>(m_Bytes.size());
m_Bytes.resize(m_Bytes.size() + size);
return offset;
}
void PutU32(uint32_t offset, uint32_t value) {
for (uint32_t i = 0; i < 4; i++) m_Bytes[offset + i] = static_cast<uint8_t>(value >> (i * 8));
}
uint32_t String(const std::string& text) {
const auto offset = Alloc(static_cast<uint32_t>(text.size()) + 1);
if (!text.empty()) std::memcpy(m_Bytes.data() + offset, text.data(), text.size());
return offset;
}
uint32_t I64(int64_t value) {
const auto offset = Alloc(8);
PutU32(offset, static_cast<uint32_t>(static_cast<uint64_t>(value)));
PutU32(offset + 4, static_cast<uint32_t>(static_cast<uint64_t>(value) >> 32));
return offset;
}
std::vector<uint8_t>& Bytes() { return m_Bytes; }
private:
std::vector<uint8_t> m_Bytes;
};
inline std::vector<uint8_t> Write(const std::vector<Table>& tables) {
constexpr uint32_t NONE = 0xFFFFFFFF;
Buffer out;
out.Alloc(8);
const auto headers = out.Alloc(static_cast<uint32_t>(tables.size()) * 8);
out.PutU32(0, static_cast<uint32_t>(tables.size()));
out.PutU32(4, headers);
for (uint32_t t = 0; t < tables.size(); t++) {
const auto& table = tables[t];
const auto columnHeader = out.Alloc(12);
const auto columns = out.Alloc(static_cast<uint32_t>(table.columns.size()) * 8);
out.PutU32(columnHeader, static_cast<uint32_t>(table.columns.size()));
out.PutU32(columnHeader + 4, out.String(table.name));
out.PutU32(columnHeader + 8, columns);
for (uint32_t c = 0; c < table.columns.size(); c++) {
out.PutU32(columns + c * 8, static_cast<uint32_t>(table.columns[c].type));
out.PutU32(columns + c * 8 + 4, out.String(table.columns[c].name));
}
const auto rowTop = out.Alloc(8);
const auto buckets = out.Alloc(table.bucketCount * 4);
out.PutU32(rowTop, table.bucketCount);
out.PutU32(rowTop + 4, buckets);
std::vector<uint32_t> lastInBucket(table.bucketCount, NONE);
for (uint32_t b = 0; b < table.bucketCount; b++) out.PutU32(buckets + b * 4, NONE);
for (const auto& row : table.rows) {
const auto fields = out.Alloc(static_cast<uint32_t>(row.size()) * 8);
for (uint32_t c = 0; c < row.size(); c++) {
const auto& value = row[c];
uint32_t raw = 0;
switch (value.type) {
case eSqliteDataType::INT32:
case eSqliteDataType::INT_BOOL:
raw = static_cast<uint32_t>(static_cast<int32_t>(value.integer));
break;
case eSqliteDataType::REAL:
std::memcpy(&raw, &value.real, sizeof(raw));
break;
case eSqliteDataType::INT64:
raw = out.I64(value.integer);
break;
case eSqliteDataType::TEXT_4:
case eSqliteDataType::TEXT_8:
raw = out.String(value.text);
break;
default:
break;
}
out.PutU32(fields + c * 8, static_cast<uint32_t>(value.type));
out.PutU32(fields + c * 8 + 4, raw);
}
const auto rowData = out.Alloc(8);
out.PutU32(rowData, static_cast<uint32_t>(row.size()));
out.PutU32(rowData + 4, fields);
const auto rowInfo = out.Alloc(8);
out.PutU32(rowInfo, rowData);
out.PutU32(rowInfo + 4, NONE);
// Bucketed by the first column, appended to the end of the bucket's chain
const auto bucket = static_cast<uint32_t>(static_cast<uint64_t>(row[0].integer) % table.bucketCount);
if (lastInBucket[bucket] == NONE) out.PutU32(buckets + bucket * 4, rowInfo);
else out.PutU32(lastInBucket[bucket] + 4, rowInfo);
lastInBucket[bucket] = rowInfo;
}
out.PutU32(headers + t * 8, columnHeader);
out.PutU32(headers + t * 8 + 4, rowTop);
}
return out.Bytes();
}
inline void WriteFile(const std::filesystem::path& path, const std::vector<uint8_t>& bytes) {
std::ofstream file(path, std::ios::binary | std::ios::trunc);
file.write(reinterpret_cast<const char*>(bytes.data()), static_cast<std::streamsize>(bytes.size()));
}
};
#endif // FDBTESTWRITER_H

1
tests/dCommonTests/CMakeLists.txt
View File

@@ -34,6 +34,7 @@ set(DCOMMONTEST_SOURCES
"BindAddressTests.cpp"
"TrafficStatsTests.cpp"
"Sd0Tests.cpp"
"FdbReaderTests.cpp"
)
add_subdirectory(dEnumsTests)

216
tests/dCommonTests/FdbReaderTests.cpp Normal file
View File

@@ -0,0 +1,216 @@
#include <gtest/gtest.h>
#include <filesystem>
#include <string>
#include <vector>
#include "FdbReader.h"
#include "../FdbTestWriter.h"
using FdbTestWriter::Value;
namespace {
std::vector<FdbTestWriter::Table> SampleTables() {
FdbTestWriter::Table items;
items.name = "Items";
items.columns = {
{ "id", eSqliteDataType::INT32 },
{ "name", eSqliteDataType::TEXT_4 },
{ "big", eSqliteDataType::INT64 },
{ "scale", eSqliteDataType::REAL },
{ "flag", eSqliteDataType::INT_BOOL },
{ "notes", eSqliteDataType::TEXT_8 },
};
items.bucketCount = 4;
// 1, 5 and -3 all land in bucket 1
items.rows = {
{ Value::Int(1), Value::Text("one"), Value::Int64(0x123456789ALL), Value::Real(1.5f), Value::Bool(true), Value::Text("a", true) },
{ Value::Int(2), Value::Text("two"), Value::Int64(-2), Value::Real(-0.25f), Value::Bool(false), Value::Null() },
{ Value::Int(5), Value::Text("caf\xE9"), Value::Null(), Value::Null(), Value::Null(), Value::Text("") },
{ Value::Int(1), Value::Text("one again"), Value::Int64(1), Value::Real(2.0f), Value::Bool(false), Value::Null() },
{ Value::Int(-3), Value::Text("negative"), Value::Int64(INT64_MIN), Value::Real(3.0f), Value::Bool(true), Value::Text("42abc") },
};
FdbTestWriter::Table empty;
empty.name = "Empty";
empty.columns = { { "id", eSqliteDataType::INT32 } };
empty.bucketCount = 0;
return { empty, items };
}
class FdbReaderTest : public ::testing::TestWithParam<bool> {
protected:
void SetUp() override {
// Parameterized test names contain '/'
std::string name = ::testing::UnitTest::GetInstance()->current_test_info()->name();
for (auto& c : name) if (c == '/') c = '_';
m_Path = std::filesystem::temp_directory_path() / ("dlu_fdb_reader_" + name + ".fdb");
FdbTestWriter::WriteFile(m_Path, FdbTestWriter::Write(SampleTables()));
}
void TearDown() override {
m_Reader.Close();
std::error_code error;
std::filesystem::remove(m_Path, error);
}
std::vector<std::string> NamesWithKey(const FdbReader::Table& table, int64_t key) {
std::vector<std::string> names;
table.ForEachRowWithKey(key, [&](const FdbReader::Row& row) { names.push_back(row.GetString(1)); });
return names;
}
std::filesystem::path m_Path;
FdbReader m_Reader;
};
}
// true maps the file, false reads it into memory (the fallback)
INSTANTIATE_TEST_SUITE_P(MapOrRead, FdbReaderTest, ::testing::Values(true, false));
TEST_P(FdbReaderTest, OpensAndFindsTables) {
ASSERT_TRUE(m_Reader.Open(m_Path, GetParam()));
EXPECT_EQ(m_Reader.IsMapped(), GetParam());
EXPECT_EQ(m_Reader.GetTables().size(), 2u);
const auto* items = m_Reader.GetTable("Items");
ASSERT_NE(items, nullptr);
EXPECT_EQ(items->GetBucketCount(), 4u);
ASSERT_EQ(items->GetColumns().size(), 6u);
EXPECT_EQ(items->GetColumns()[2].name, "big");
EXPECT_EQ(items->GetColumns()[2].type, eSqliteDataType::INT64);
EXPECT_EQ(items->GetColumnIndex("notes"), 5);
EXPECT_EQ(items->GetColumnIndex("missing"), -1);
EXPECT_EQ(m_Reader.GetTable("items"), nullptr);
EXPECT_EQ(m_Reader.GetTable("Nope"), nullptr);
const auto* empty = m_Reader.GetTable("Empty");
ASSERT_NE(empty, nullptr);
EXPECT_FALSE(empty->FindFirst(0).has_value());
uint32_t rows = 0;
empty->ForEachRow([&](const FdbReader::Row&) { rows++; });
EXPECT_EQ(rows, 0u);
}
TEST_P(FdbReaderTest, LooksUpByKeyThroughCollisions) {
ASSERT_TRUE(m_Reader.Open(m_Path, GetParam()));
const auto& items = *m_Reader.GetTable("Items");
// Same bucket, filtered by key, file order kept
EXPECT_EQ(NamesWithKey(items, 1), (std::vector<std::string>{ "one", "one again" }));
EXPECT_EQ(NamesWithKey(items, 5), (std::vector<std::string>{ "caf\xC3\xA9" }));
EXPECT_EQ(NamesWithKey(items, -3), (std::vector<std::string>{ "negative" }));
EXPECT_EQ(NamesWithKey(items, 2), (std::vector<std::string>{ "two" }));
EXPECT_TRUE(NamesWithKey(items, 9).empty());
EXPECT_TRUE(NamesWithKey(items, 3).empty());
const auto first = items.FindFirst(1);
ASSERT_TRUE(first.has_value());
EXPECT_EQ(first->GetString(1), "one");
uint32_t rows = 0;
items.ForEachRow([&](const FdbReader::Row&) { rows++; });
EXPECT_EQ(rows, 5u);
}
TEST_P(FdbReaderTest, ReadsTypedValues) {
ASSERT_TRUE(m_Reader.Open(m_Path, GetParam()));
const auto& items = *m_Reader.GetTable("Items");
const auto one = items.FindFirst(1);
ASSERT_TRUE(one.has_value());
EXPECT_EQ(one->GetFieldCount(), 6u);
EXPECT_EQ(one->GetInt(0), 1);
EXPECT_EQ(one->GetInt64(2), 0x123456789ALL);
// Low 32 bits, as sqlite3_column_int
EXPECT_EQ(one->GetInt(2), 0x3456789A);
EXPECT_FLOAT_EQ(one->GetFloat(3), 1.5f);
EXPECT_TRUE(one->GetBool(4));
EXPECT_EQ(one->GetType(5), eSqliteDataType::TEXT_8);
EXPECT_EQ(one->GetString(5), "a");
const auto two = items.FindFirst(2);
ASSERT_TRUE(two.has_value());
EXPECT_EQ(two->GetInt64(2), -2);
EXPECT_FLOAT_EQ(two->GetFloat(3), -0.25f);
EXPECT_FALSE(two->GetBool(4, true));
EXPECT_TRUE(two->IsNull(5));
EXPECT_EQ(two->GetString(5, "fallback"), "fallback");
const auto negative = items.FindFirst(-3);
ASSERT_TRUE(negative.has_value());
EXPECT_EQ(negative->GetInt64(2), INT64_MIN);
// Text read as a number, as SQLite does
EXPECT_EQ(negative->GetInt(5), 42);
// A number read as text
EXPECT_EQ(negative->GetString(0), "-3");
// Nulls give the caller's default
const auto five = items.FindFirst(5);
ASSERT_TRUE(five.has_value());
EXPECT_TRUE(five->IsNull(2));
EXPECT_EQ(five->GetInt(2, -1), -1);
EXPECT_EQ(five->GetInt64(2, -7), -7);
EXPECT_FLOAT_EQ(five->GetFloat(3, -1.0f), -1.0f);
EXPECT_TRUE(five->GetBool(4, true));
EXPECT_EQ(five->GetRawString(1), "caf\xE9");
EXPECT_EQ(five->GetString(5, "x"), "");
// Past the last column reads as null
EXPECT_TRUE(five->IsNull(99));
EXPECT_EQ(five->GetInt(99, 11), 11);
}
TEST(FdbReaderFailureTest, MissingEmptyAndTruncatedFiles) {
const auto dir = std::filesystem::temp_directory_path();
FdbReader reader;
EXPECT_FALSE(reader.Open(dir / "dlu_fdb_reader_does_not_exist.fdb"));
EXPECT_FALSE(reader.IsOpen());
const auto emptyPath = dir / "dlu_fdb_reader_empty.fdb";
FdbTestWriter::WriteFile(emptyPath, {});
EXPECT_FALSE(reader.Open(emptyPath));
EXPECT_FALSE(reader.Open(emptyPath, false));
// Cut before the row data: the headers are still sound, the rows read as missing instead of past the end
auto bytes = FdbTestWriter::Write(SampleTables());
const auto truncatedPath = dir / "dlu_fdb_reader_truncated.fdb";
FdbTestWriter::WriteFile(truncatedPath, std::vector<uint8_t>(bytes.begin(), bytes.begin() + 20));
EXPECT_FALSE(reader.Open(truncatedPath));
// A table count far past the end of the file
bytes[0] = 0xFF;
bytes[1] = 0xFF;
const auto badCountPath = dir / "dlu_fdb_reader_bad_count.fdb";
FdbTestWriter::WriteFile(badCountPath, bytes);
EXPECT_FALSE(reader.Open(badCountPath));
std::error_code error;
std::filesystem::remove(emptyPath, error);
std::filesystem::remove(truncatedPath, error);
std::filesystem::remove(badCountPath, error);
}
TEST(FdbReaderFailureTest, MappedFileMovesAndCloses) {
const auto path = std::filesystem::temp_directory_path() / "dlu_fdb_mapped_move.fdb";
FdbTestWriter::WriteFile(path, { 1, 2, 3, 4 });
for (const bool map : { true, false }) {
FdbMappedFile file;
ASSERT_TRUE(file.Open(path, map));
EXPECT_EQ(file.IsMapped(), map);
EXPECT_EQ(file.GetSize(), 4u);
FdbMappedFile moved(std::move(file));
EXPECT_FALSE(file.IsOpen());
ASSERT_TRUE(moved.IsOpen());
EXPECT_EQ(moved.GetData()[3], 4);
moved.Close();
EXPECT_FALSE(moved.IsOpen());
EXPECT_EQ(moved.GetSize(), 0u);
}
std::error_code error;
std::filesystem::remove(path, error);
}