mirror of
https://github.com/DarkflameUniverse/DarkflameServer.git
synced 2026-10-02 02:43:44 +00:00
feat(dCommon): read cdclient.fdb in place through its hash buckets
FdbMappedFile maps a file read-only (CreateFileMapping/MapViewOfFile on Windows, mmap elsewhere) and falls back to reading it into memory when mapping fails. FdbReader reads the table and column headers from it and looks rows up by their first column through the fdb's own buckets, decoding every integer as little-endian with bounds checks, so the rows never get copied out of the file. Tests write small fdb files (collisions, text, int64, nulls) and read them both mapped and from memory. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
@@ -19,6 +19,8 @@ set(DCOMMON_SOURCES
|
||||
"BrickByBrickFix.cpp"
|
||||
"BinaryPathFinder.cpp"
|
||||
"FdbToSqlite.cpp"
|
||||
"FdbMappedFile.cpp"
|
||||
"FdbReader.cpp"
|
||||
"JSONUtils.cpp"
|
||||
"TinyXmlUtils.cpp"
|
||||
"Sd0.cpp"
|
||||
|
||||
145
dCommon/FdbMappedFile.cpp
Normal file
145
dCommon/FdbMappedFile.cpp
Normal file
@@ -0,0 +1,145 @@
|
||||
#include "FdbMappedFile.h"
|
||||
|
||||
#include <fstream>
|
||||
#include <limits>
|
||||
#include <utility>
|
||||
|
||||
#ifdef _WIN32
|
||||
#ifndef WIN32_LEAN_AND_MEAN
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#endif
|
||||
#ifndef NOMINMAX
|
||||
#define NOMINMAX
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <fcntl.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
FdbMappedFile::~FdbMappedFile() {
|
||||
Close();
|
||||
}
|
||||
|
||||
FdbMappedFile::FdbMappedFile(FdbMappedFile&& other) noexcept {
|
||||
*this = std::move(other);
|
||||
}
|
||||
|
||||
FdbMappedFile& FdbMappedFile::operator=(FdbMappedFile&& other) noexcept {
|
||||
if (this == &other) return *this;
|
||||
Close();
|
||||
m_Mapped = std::exchange(other.m_Mapped, false);
|
||||
m_Size = std::exchange(other.m_Size, 0);
|
||||
m_Buffer = std::move(other.m_Buffer);
|
||||
other.m_Buffer.clear();
|
||||
const uint8_t* data = std::exchange(other.m_Data, nullptr);
|
||||
// A moved vector keeps its heap block, so the pointer stays valid; set it again anyway for clarity
|
||||
m_Data = m_Mapped ? data : (m_Buffer.empty() ? nullptr : m_Buffer.data());
|
||||
#ifdef _WIN32
|
||||
m_FileHandle = std::exchange(other.m_FileHandle, nullptr);
|
||||
m_MappingHandle = std::exchange(other.m_MappingHandle, nullptr);
|
||||
#endif
|
||||
return *this;
|
||||
}
|
||||
|
||||
bool FdbMappedFile::Open(const std::filesystem::path& path, bool allowMapping) {
|
||||
Close();
|
||||
if (allowMapping && Map(path)) return true;
|
||||
return Read(path);
|
||||
}
|
||||
|
||||
void FdbMappedFile::Close() {
|
||||
if (m_Mapped && m_Data) {
|
||||
#ifdef _WIN32
|
||||
UnmapViewOfFile(m_Data);
|
||||
#else
|
||||
munmap(const_cast<uint8_t*>(m_Data), static_cast<std::size_t>(m_Size));
|
||||
#endif
|
||||
}
|
||||
#ifdef _WIN32
|
||||
if (m_MappingHandle) CloseHandle(static_cast<HANDLE>(m_MappingHandle));
|
||||
if (m_FileHandle) CloseHandle(static_cast<HANDLE>(m_FileHandle));
|
||||
m_MappingHandle = nullptr;
|
||||
m_FileHandle = nullptr;
|
||||
#endif
|
||||
m_Data = nullptr;
|
||||
m_Size = 0;
|
||||
m_Mapped = false;
|
||||
m_Buffer.clear();
|
||||
m_Buffer.shrink_to_fit();
|
||||
}
|
||||
|
||||
bool FdbMappedFile::Map(const std::filesystem::path& path) {
|
||||
#ifdef _WIN32
|
||||
HANDLE file = CreateFileW(path.wstring().c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
|
||||
if (file == INVALID_HANDLE_VALUE) return false;
|
||||
|
||||
LARGE_INTEGER size{};
|
||||
if (!GetFileSizeEx(file, &size) || size.QuadPart <= 0) {
|
||||
CloseHandle(file);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Size 0/0 maps the whole file
|
||||
HANDLE mapping = CreateFileMappingW(file, nullptr, PAGE_READONLY, 0, 0, nullptr);
|
||||
if (!mapping) {
|
||||
CloseHandle(file);
|
||||
return false;
|
||||
}
|
||||
|
||||
const void* view = MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0);
|
||||
if (!view) {
|
||||
CloseHandle(mapping);
|
||||
CloseHandle(file);
|
||||
return false;
|
||||
}
|
||||
|
||||
m_FileHandle = file;
|
||||
m_MappingHandle = mapping;
|
||||
m_Data = static_cast<const uint8_t*>(view);
|
||||
m_Size = static_cast<uint64_t>(size.QuadPart);
|
||||
m_Mapped = true;
|
||||
return true;
|
||||
#else
|
||||
const int fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
|
||||
if (fd < 0) return false;
|
||||
|
||||
struct stat info {};
|
||||
if (fstat(fd, &info) != 0 || info.st_size <= 0 ||
|
||||
static_cast<uint64_t>(info.st_size) > static_cast<uint64_t>(std::numeric_limits<std::size_t>::max())) {
|
||||
close(fd);
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto length = static_cast<std::size_t>(info.st_size);
|
||||
void* view = mmap(nullptr, length, PROT_READ, MAP_SHARED, fd, 0);
|
||||
// The mapping keeps its own reference to the file
|
||||
close(fd);
|
||||
if (view == MAP_FAILED) return false;
|
||||
|
||||
m_Data = static_cast<const uint8_t*>(view);
|
||||
m_Size = static_cast<uint64_t>(info.st_size);
|
||||
m_Mapped = true;
|
||||
return true;
|
||||
#endif
|
||||
}
|
||||
|
||||
bool FdbMappedFile::Read(const std::filesystem::path& path) {
|
||||
std::ifstream file(path, std::ios::binary | std::ios::ate);
|
||||
if (!file) return false;
|
||||
|
||||
const std::streamoff size = file.tellg();
|
||||
if (size <= 0) return false;
|
||||
|
||||
std::vector<uint8_t> buffer(static_cast<std::size_t>(size));
|
||||
file.seekg(0);
|
||||
if (!file.read(reinterpret_cast<char*>(buffer.data()), size)) return false;
|
||||
|
||||
m_Buffer = std::move(buffer);
|
||||
m_Data = m_Buffer.data();
|
||||
m_Size = static_cast<uint64_t>(size);
|
||||
m_Mapped = false;
|
||||
return true;
|
||||
}
|
||||
58
dCommon/FdbMappedFile.h
Normal file
58
dCommon/FdbMappedFile.h
Normal file
@@ -0,0 +1,58 @@
|
||||
#ifndef FDBMAPPEDFILE_H
|
||||
#define FDBMAPPEDFILE_H
|
||||
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <vector>
|
||||
|
||||
/**
|
||||
* A read-only view of a whole file. The file is memory-mapped when the platform allows it
|
||||
* (CreateFileMapping/MapViewOfFile on Windows, mmap elsewhere), so every process that maps the
|
||||
* same file shares one copy through the OS page cache. If mapping fails the file is read into
|
||||
* memory instead, and the view works the same way.
|
||||
*/
|
||||
class FdbMappedFile {
|
||||
public:
|
||||
FdbMappedFile() = default;
|
||||
~FdbMappedFile();
|
||||
|
||||
FdbMappedFile(const FdbMappedFile&) = delete;
|
||||
FdbMappedFile& operator=(const FdbMappedFile&) = delete;
|
||||
FdbMappedFile(FdbMappedFile&& other) noexcept;
|
||||
FdbMappedFile& operator=(FdbMappedFile&& other) noexcept;
|
||||
|
||||
/**
|
||||
* Opens a file. Closes whatever was open before.
|
||||
*
|
||||
* @param path The file to open
|
||||
* @param allowMapping false skips the mapping and reads the file into memory
|
||||
* @return true if the file is open and not empty
|
||||
*/
|
||||
bool Open(const std::filesystem::path& path, bool allowMapping = true);
|
||||
|
||||
void Close();
|
||||
|
||||
[[nodiscard]] bool IsOpen() const { return m_Data != nullptr; }
|
||||
|
||||
// true if the view is a mapping, false if the file was read into memory
|
||||
[[nodiscard]] bool IsMapped() const { return m_Mapped; }
|
||||
|
||||
[[nodiscard]] const uint8_t* GetData() const { return m_Data; }
|
||||
|
||||
[[nodiscard]] uint64_t GetSize() const { return m_Size; }
|
||||
|
||||
private:
|
||||
bool Map(const std::filesystem::path& path);
|
||||
bool Read(const std::filesystem::path& path);
|
||||
|
||||
const uint8_t* m_Data = nullptr;
|
||||
uint64_t m_Size = 0;
|
||||
bool m_Mapped = false;
|
||||
std::vector<uint8_t> m_Buffer;
|
||||
#ifdef _WIN32
|
||||
void* m_FileHandle = nullptr;
|
||||
void* m_MappingHandle = nullptr;
|
||||
#endif
|
||||
};
|
||||
|
||||
#endif // FDBMAPPEDFILE_H
|
||||
260
dCommon/FdbReader.cpp
Normal file
260
dCommon/FdbReader.cpp
Normal file
@@ -0,0 +1,260 @@
|
||||
#include "FdbReader.h"
|
||||
|
||||
#include <charconv>
|
||||
#include <cstdlib>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
|
||||
#include "GeneralUtils.h"
|
||||
|
||||
namespace {
|
||||
// Little-endian decode from bytes, independent of host byte order and alignment
|
||||
uint32_t DecodeU32(const uint8_t* bytes) {
|
||||
return static_cast<uint32_t>(bytes[0]) |
|
||||
(static_cast<uint32_t>(bytes[1]) << 8) |
|
||||
(static_cast<uint32_t>(bytes[2]) << 16) |
|
||||
(static_cast<uint32_t>(bytes[3]) << 24);
|
||||
}
|
||||
|
||||
uint64_t DecodeU64(const uint8_t* bytes) {
|
||||
return static_cast<uint64_t>(DecodeU32(bytes)) | (static_cast<uint64_t>(DecodeU32(bytes + 4)) << 32);
|
||||
}
|
||||
|
||||
float BitsToFloat(uint32_t bits) {
|
||||
float value;
|
||||
static_assert(sizeof(value) == sizeof(bits));
|
||||
std::memcpy(&value, &bits, sizeof(value));
|
||||
return value;
|
||||
}
|
||||
|
||||
// sqlite3_column_int on text: the leading integer, 0 if there is none
|
||||
int64_t LeadingInteger(std::string_view text) {
|
||||
size_t start = 0;
|
||||
while (start < text.size() && (text[start] == ' ' || text[start] == '\t')) start++;
|
||||
if (start < text.size() && text[start] == '+') start++;
|
||||
int64_t value = 0;
|
||||
std::from_chars(text.data() + start, text.data() + text.size(), value);
|
||||
return value;
|
||||
}
|
||||
|
||||
bool IsText(eSqliteDataType type) {
|
||||
return type == eSqliteDataType::TEXT_4 || type == eSqliteDataType::TEXT_8;
|
||||
}
|
||||
}
|
||||
|
||||
bool FdbReader::Open(const std::filesystem::path& path, bool allowMapping) {
|
||||
Close();
|
||||
if (!m_File.Open(path, allowMapping)) return false;
|
||||
if (!ReadHeaders()) {
|
||||
Close();
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void FdbReader::Close() {
|
||||
m_TableIndex.clear();
|
||||
m_Tables.clear();
|
||||
m_File.Close();
|
||||
}
|
||||
|
||||
bool FdbReader::InBounds(uint64_t offset, uint64_t length) const {
|
||||
const uint64_t size = m_File.GetSize();
|
||||
return offset <= size && length <= size - offset;
|
||||
}
|
||||
|
||||
bool FdbReader::ReadU32(uint64_t offset, uint32_t& out) const {
|
||||
if (!InBounds(offset, 4)) return false;
|
||||
out = DecodeU32(m_File.GetData() + offset);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool FdbReader::ReadI64(uint64_t offset, int64_t& out) const {
|
||||
if (!InBounds(offset, 8)) return false;
|
||||
out = static_cast<int64_t>(DecodeU64(m_File.GetData() + offset));
|
||||
return true;
|
||||
}
|
||||
|
||||
std::optional<std::string_view> FdbReader::ReadCString(uint64_t offset) const {
|
||||
if (!InBounds(offset, 0) || offset == m_File.GetSize()) return std::nullopt;
|
||||
const auto* start = m_File.GetData() + offset;
|
||||
const auto* end = static_cast<const uint8_t*>(std::memchr(start, 0, static_cast<size_t>(m_File.GetSize() - offset)));
|
||||
if (!end) return std::nullopt;
|
||||
return std::string_view(reinterpret_cast<const char*>(start), static_cast<size_t>(end - start));
|
||||
}
|
||||
|
||||
bool FdbReader::ReadHeaders() {
|
||||
uint32_t tableCount = 0;
|
||||
uint32_t tableHeaders = 0;
|
||||
if (!ReadU32(0, tableCount) || !ReadU32(4, tableHeaders)) return false;
|
||||
if (!InBounds(tableHeaders, static_cast<uint64_t>(tableCount) * 8)) return false;
|
||||
|
||||
m_Tables.reserve(tableCount);
|
||||
for (uint32_t i = 0; i < tableCount; i++) {
|
||||
const uint64_t entry = static_cast<uint64_t>(tableHeaders) + static_cast<uint64_t>(i) * 8;
|
||||
uint32_t columnHeader = 0;
|
||||
uint32_t rowTop = 0;
|
||||
if (!ReadU32(entry, columnHeader) || !ReadU32(entry + 4, rowTop)) return false;
|
||||
|
||||
Table table;
|
||||
table.m_Reader = this;
|
||||
|
||||
uint32_t columnCount = 0;
|
||||
uint32_t namePointer = 0;
|
||||
uint32_t columns = 0;
|
||||
if (!ReadU32(columnHeader, columnCount) || !ReadU32(static_cast<uint64_t>(columnHeader) + 4, namePointer) ||
|
||||
!ReadU32(static_cast<uint64_t>(columnHeader) + 8, columns)) return false;
|
||||
const auto name = ReadCString(namePointer);
|
||||
if (!name || !InBounds(columns, static_cast<uint64_t>(columnCount) * 8)) return false;
|
||||
table.m_Name = std::string(*name);
|
||||
|
||||
table.m_Columns.reserve(columnCount);
|
||||
for (uint32_t c = 0; c < columnCount; c++) {
|
||||
const uint64_t column = static_cast<uint64_t>(columns) + static_cast<uint64_t>(c) * 8;
|
||||
uint32_t type = 0;
|
||||
uint32_t columnName = 0;
|
||||
if (!ReadU32(column, type) || !ReadU32(column + 4, columnName)) return false;
|
||||
const auto columnNameString = ReadCString(columnName);
|
||||
if (!columnNameString) return false;
|
||||
table.m_Columns.push_back({ GeneralUtils::Latin1ToUTF8(std::u8string_view(reinterpret_cast<const char8_t*>(columnNameString->data()), columnNameString->size())), static_cast<eSqliteDataType>(type) });
|
||||
}
|
||||
|
||||
if (!ReadU32(rowTop, table.m_BucketCount) || !ReadU32(static_cast<uint64_t>(rowTop) + 4, table.m_BucketArrayOffset)) return false;
|
||||
if (!InBounds(table.m_BucketArrayOffset, static_cast<uint64_t>(table.m_BucketCount) * 4)) return false;
|
||||
|
||||
m_Tables.push_back(std::move(table));
|
||||
}
|
||||
|
||||
// Built after the vector stops growing, since the keys point into the table names
|
||||
for (uint32_t i = 0; i < m_Tables.size(); i++) m_TableIndex.emplace(m_Tables[i].m_Name, i);
|
||||
return true;
|
||||
}
|
||||
|
||||
const FdbReader::Table* FdbReader::GetTable(std::string_view name) const {
|
||||
const auto it = m_TableIndex.find(name);
|
||||
return it == m_TableIndex.end() ? nullptr : &m_Tables[it->second];
|
||||
}
|
||||
|
||||
int32_t FdbReader::Table::GetColumnIndex(std::string_view name) const {
|
||||
for (uint32_t i = 0; i < m_Columns.size(); i++) {
|
||||
if (m_Columns[i].name == name) return static_cast<int32_t>(i);
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
std::optional<FdbReader::Row> FdbReader::Table::FindFirst(int64_t key) const {
|
||||
std::optional<Row> found;
|
||||
ForEachRowWithKey(key, [&found](const Row& row) {
|
||||
if (!found) found = row;
|
||||
});
|
||||
return found;
|
||||
}
|
||||
|
||||
std::optional<int64_t> FdbReader::Table::KeyOf(const Row& row) {
|
||||
switch (row.GetType(0)) {
|
||||
case eSqliteDataType::INT32:
|
||||
case eSqliteDataType::INT_BOOL:
|
||||
return row.GetInt(0);
|
||||
case eSqliteDataType::INT64:
|
||||
return row.GetInt64(0);
|
||||
default:
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
bool FdbReader::Row::ReadField(uint32_t column, eSqliteDataType& type, uint32_t& value) const {
|
||||
if (column >= m_FieldCount) return false;
|
||||
const uint64_t field = static_cast<uint64_t>(m_FieldsOffset) + static_cast<uint64_t>(column) * 8;
|
||||
uint32_t rawType = 0;
|
||||
if (!m_Reader->ReadU32(field, rawType) || !m_Reader->ReadU32(field + 4, value)) return false;
|
||||
type = static_cast<eSqliteDataType>(rawType);
|
||||
return true;
|
||||
}
|
||||
|
||||
eSqliteDataType FdbReader::Row::GetType(uint32_t column) const {
|
||||
eSqliteDataType type{};
|
||||
uint32_t value = 0;
|
||||
return ReadField(column, type, value) ? type : eSqliteDataType::NONE;
|
||||
}
|
||||
|
||||
int64_t FdbReader::Row::GetInt64(uint32_t column, int64_t nullValue) const {
|
||||
eSqliteDataType type{};
|
||||
uint32_t value = 0;
|
||||
if (!ReadField(column, type, value)) return nullValue;
|
||||
switch (type) {
|
||||
case eSqliteDataType::INT32:
|
||||
return static_cast<int32_t>(value);
|
||||
case eSqliteDataType::INT_BOOL:
|
||||
// The conversion stores bools as 0 or 1
|
||||
return value != 0 ? 1 : 0;
|
||||
case eSqliteDataType::INT64: {
|
||||
int64_t wide = 0;
|
||||
return m_Reader->ReadI64(value, wide) ? wide : nullValue;
|
||||
}
|
||||
case eSqliteDataType::REAL:
|
||||
return static_cast<int64_t>(BitsToFloat(value));
|
||||
case eSqliteDataType::TEXT_4:
|
||||
case eSqliteDataType::TEXT_8:
|
||||
return LeadingInteger(GetRawString(column));
|
||||
default:
|
||||
return nullValue;
|
||||
}
|
||||
}
|
||||
|
||||
int32_t FdbReader::Row::GetInt(uint32_t column, int32_t nullValue) const {
|
||||
// sqlite3_column_int keeps the low 32 bits of a wider integer
|
||||
if (IsNull(column)) return nullValue;
|
||||
return static_cast<int32_t>(static_cast<uint32_t>(static_cast<uint64_t>(GetInt64(column, nullValue))));
|
||||
}
|
||||
|
||||
float FdbReader::Row::GetFloat(uint32_t column, float nullValue) const {
|
||||
eSqliteDataType type{};
|
||||
uint32_t value = 0;
|
||||
if (!ReadField(column, type, value)) return nullValue;
|
||||
switch (type) {
|
||||
case eSqliteDataType::REAL:
|
||||
return BitsToFloat(value);
|
||||
case eSqliteDataType::INT32:
|
||||
case eSqliteDataType::INT_BOOL:
|
||||
case eSqliteDataType::INT64:
|
||||
return static_cast<float>(GetInt64(column, 0));
|
||||
case eSqliteDataType::TEXT_4:
|
||||
case eSqliteDataType::TEXT_8: {
|
||||
const std::string text(GetRawString(column));
|
||||
return static_cast<float>(std::strtod(text.c_str(), nullptr));
|
||||
}
|
||||
default:
|
||||
return nullValue;
|
||||
}
|
||||
}
|
||||
|
||||
std::string_view FdbReader::Row::GetRawString(uint32_t column) const {
|
||||
eSqliteDataType type{};
|
||||
uint32_t value = 0;
|
||||
if (!ReadField(column, type, value) || !IsText(type)) return {};
|
||||
return m_Reader->ReadCString(value).value_or(std::string_view{});
|
||||
}
|
||||
|
||||
std::string FdbReader::Row::GetString(uint32_t column, std::string_view nullValue) const {
|
||||
eSqliteDataType type{};
|
||||
uint32_t value = 0;
|
||||
if (!ReadField(column, type, value)) return std::string(nullValue);
|
||||
switch (type) {
|
||||
case eSqliteDataType::TEXT_4:
|
||||
case eSqliteDataType::TEXT_8: {
|
||||
const auto raw = GetRawString(column);
|
||||
return GeneralUtils::Latin1ToUTF8(std::u8string_view(reinterpret_cast<const char8_t*>(raw.data()), raw.size()));
|
||||
}
|
||||
case eSqliteDataType::INT32:
|
||||
case eSqliteDataType::INT_BOOL:
|
||||
case eSqliteDataType::INT64:
|
||||
return std::to_string(GetInt64(column, 0));
|
||||
case eSqliteDataType::REAL: {
|
||||
char buffer[32];
|
||||
std::snprintf(buffer, sizeof(buffer), "%.15g", static_cast<double>(BitsToFloat(value)));
|
||||
return buffer;
|
||||
}
|
||||
default:
|
||||
return std::string(nullValue);
|
||||
}
|
||||
}
|
||||
181
dCommon/FdbReader.h
Normal file
181
dCommon/FdbReader.h
Normal file
@@ -0,0 +1,181 @@
|
||||
#ifndef FDBREADER_H
|
||||
#define FDBREADER_H
|
||||
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "FdbMappedFile.h"
|
||||
#include "eSqliteDataType.h"
|
||||
|
||||
/**
|
||||
* Reads the client's cdclient.fdb in place, without loading it.
|
||||
*
|
||||
* Layout (all integers little-endian, every pointer an absolute u32 file offset, -1 meaning none):
|
||||
* file header: u32 table count, ptr to table header array
|
||||
* table header: ptr to column header, ptr to row top header (one pair per table, sorted by name)
|
||||
* column header: u32 column count, ptr to table name, ptr to column array
|
||||
* column: u32 data type, ptr to column name
|
||||
* row top: u32 bucket count (a power of two, may be 0), ptr to bucket array
|
||||
* bucket array: one ptr per bucket to the first row info of the bucket
|
||||
* row info: ptr to row data, ptr to the next row info in the same bucket
|
||||
* row data: u32 field count, ptr to field array
|
||||
* field: u32 data type, u32 value
|
||||
*
|
||||
* A field value holds the number itself for int32, bool and float, and a pointer for int64
|
||||
* (to 8 bytes) and text (to a null-terminated latin-1 string). A row sits in bucket
|
||||
* (u32)(first field) % bucket count, for int32 first columns. Tables are found by name and
|
||||
* rows by their first column, so a lookup only touches a few pages of the file.
|
||||
*/
|
||||
class FdbReader {
|
||||
public:
|
||||
class Row;
|
||||
class Table;
|
||||
|
||||
struct Column {
|
||||
std::string name;
|
||||
eSqliteDataType type{};
|
||||
};
|
||||
|
||||
class Row {
|
||||
public:
|
||||
[[nodiscard]] uint32_t GetFieldCount() const { return m_FieldCount; }
|
||||
|
||||
// NONE for a null field or a column past the end of the row
|
||||
[[nodiscard]] eSqliteDataType GetType(uint32_t column) const;
|
||||
|
||||
[[nodiscard]] bool IsNull(uint32_t column) const { return GetType(column) == eSqliteDataType::NONE; }
|
||||
|
||||
// These convert between types the way SQLite does for the converted CDServer.sqlite,
|
||||
// and return nullValue for a null field.
|
||||
[[nodiscard]] int32_t GetInt(uint32_t column, int32_t nullValue = 0) const;
|
||||
[[nodiscard]] int64_t GetInt64(uint32_t column, int64_t nullValue = 0) const;
|
||||
[[nodiscard]] float GetFloat(uint32_t column, float nullValue = 0.0f) const;
|
||||
[[nodiscard]] bool GetBool(uint32_t column, bool nullValue = false) const { return GetInt(column, nullValue ? 1 : 0) != 0; }
|
||||
|
||||
// UTF-8
|
||||
[[nodiscard]] std::string GetString(uint32_t column, std::string_view nullValue = "") const;
|
||||
|
||||
// The latin-1 bytes as stored, for text fields; empty otherwise
|
||||
[[nodiscard]] std::string_view GetRawString(uint32_t column) const;
|
||||
|
||||
private:
|
||||
friend class FdbReader;
|
||||
friend class Table;
|
||||
Row(const FdbReader* reader, uint32_t fieldsOffset, uint32_t fieldCount)
|
||||
: m_Reader(reader), m_FieldsOffset(fieldsOffset), m_FieldCount(fieldCount) {}
|
||||
|
||||
bool ReadField(uint32_t column, eSqliteDataType& type, uint32_t& value) const;
|
||||
|
||||
const FdbReader* m_Reader;
|
||||
uint32_t m_FieldsOffset;
|
||||
uint32_t m_FieldCount;
|
||||
};
|
||||
|
||||
class Table {
|
||||
public:
|
||||
[[nodiscard]] const std::string& GetName() const { return m_Name; }
|
||||
[[nodiscard]] const std::vector<Column>& GetColumns() const { return m_Columns; }
|
||||
[[nodiscard]] uint32_t GetBucketCount() const { return m_BucketCount; }
|
||||
|
||||
// The column's index, or -1 when the table has no such column
|
||||
[[nodiscard]] int32_t GetColumnIndex(std::string_view name) const;
|
||||
|
||||
// Calls f(const Row&) for each row whose first column equals key, in file order
|
||||
template<typename F>
|
||||
void ForEachRowWithKey(int64_t key, F&& f) const {
|
||||
if (m_BucketCount == 0) return;
|
||||
const uint32_t bucket = static_cast<uint32_t>(static_cast<uint64_t>(key) % m_BucketCount);
|
||||
WalkBucket(bucket, [&](const Row& row) {
|
||||
if (KeyOf(row) == key) f(row);
|
||||
});
|
||||
}
|
||||
|
||||
// The first row whose first column equals key
|
||||
[[nodiscard]] std::optional<Row> FindFirst(int64_t key) const;
|
||||
|
||||
// Calls f(const Row&) for every row, bucket by bucket
|
||||
template<typename F>
|
||||
void ForEachRow(F&& f) const {
|
||||
for (uint32_t bucket = 0; bucket < m_BucketCount; bucket++) WalkBucket(bucket, f);
|
||||
}
|
||||
|
||||
// The first column as an integer, the value rows are bucketed by
|
||||
[[nodiscard]] static std::optional<int64_t> KeyOf(const Row& row);
|
||||
|
||||
private:
|
||||
friend class FdbReader;
|
||||
|
||||
template<typename F>
|
||||
void WalkBucket(uint32_t bucket, F&& f) const {
|
||||
uint32_t rowInfo = 0;
|
||||
if (!m_Reader->ReadU32(m_BucketArrayOffset + static_cast<uint64_t>(bucket) * 4, rowInfo)) return;
|
||||
// A bound on the chain length so a damaged file can't loop forever
|
||||
uint64_t guard = m_Reader->GetSize() / 8;
|
||||
while (rowInfo != NO_POINTER && guard-- > 0) {
|
||||
uint32_t rowData = 0;
|
||||
uint32_t next = NO_POINTER;
|
||||
if (!m_Reader->ReadU32(rowInfo, rowData) || !m_Reader->ReadU32(static_cast<uint64_t>(rowInfo) + 4, next)) return;
|
||||
uint32_t fieldCount = 0;
|
||||
uint32_t fields = 0;
|
||||
if (m_Reader->ReadU32(rowData, fieldCount) && m_Reader->ReadU32(static_cast<uint64_t>(rowData) + 4, fields) &&
|
||||
m_Reader->InBounds(fields, static_cast<uint64_t>(fieldCount) * 8)) {
|
||||
f(Row(m_Reader, fields, fieldCount));
|
||||
}
|
||||
rowInfo = next;
|
||||
}
|
||||
}
|
||||
|
||||
const FdbReader* m_Reader = nullptr;
|
||||
std::string m_Name;
|
||||
std::vector<Column> m_Columns;
|
||||
uint32_t m_BucketCount = 0;
|
||||
uint32_t m_BucketArrayOffset = 0;
|
||||
};
|
||||
|
||||
static constexpr uint32_t NO_POINTER = 0xFFFFFFFF;
|
||||
|
||||
FdbReader() = default;
|
||||
FdbReader(const FdbReader&) = delete;
|
||||
FdbReader& operator=(const FdbReader&) = delete;
|
||||
|
||||
/**
|
||||
* Opens an fdb file and reads its table and column headers (the rows stay in the file).
|
||||
*
|
||||
* @param path The fdb file
|
||||
* @param allowMapping false reads the file into memory instead of mapping it
|
||||
* @return true if the file was opened and its headers are sound
|
||||
*/
|
||||
bool Open(const std::filesystem::path& path, bool allowMapping = true);
|
||||
|
||||
void Close();
|
||||
|
||||
[[nodiscard]] bool IsOpen() const { return m_File.IsOpen(); }
|
||||
[[nodiscard]] bool IsMapped() const { return m_File.IsMapped(); }
|
||||
[[nodiscard]] uint64_t GetSize() const { return m_File.GetSize(); }
|
||||
|
||||
// The table with this exact name, or nullptr
|
||||
[[nodiscard]] const Table* GetTable(std::string_view name) const;
|
||||
|
||||
[[nodiscard]] const std::vector<Table>& GetTables() const { return m_Tables; }
|
||||
|
||||
// Little-endian reads that fail instead of reading past the end of the file
|
||||
[[nodiscard]] bool ReadU32(uint64_t offset, uint32_t& out) const;
|
||||
[[nodiscard]] bool ReadI64(uint64_t offset, int64_t& out) const;
|
||||
[[nodiscard]] bool InBounds(uint64_t offset, uint64_t length) const;
|
||||
// The null-terminated string at offset, without the terminator
|
||||
[[nodiscard]] std::optional<std::string_view> ReadCString(uint64_t offset) const;
|
||||
|
||||
private:
|
||||
bool ReadHeaders();
|
||||
|
||||
FdbMappedFile m_File;
|
||||
std::vector<Table> m_Tables;
|
||||
std::unordered_map<std::string_view, uint32_t> m_TableIndex;
|
||||
};
|
||||
|
||||
#endif // FDBREADER_H
|
||||
Reference in New Issue
Block a user