diff --git a/dCommon/CMakeLists.txt b/dCommon/CMakeLists.txt index 1080d3143..f1f10270c 100644 --- a/dCommon/CMakeLists.txt +++ b/dCommon/CMakeLists.txt @@ -19,6 +19,8 @@ set(DCOMMON_SOURCES "BrickByBrickFix.cpp" "BinaryPathFinder.cpp" "FdbToSqlite.cpp" + "FdbMappedFile.cpp" + "FdbReader.cpp" "JSONUtils.cpp" "TinyXmlUtils.cpp" "Sd0.cpp" diff --git a/dCommon/FdbMappedFile.cpp b/dCommon/FdbMappedFile.cpp new file mode 100644 index 000000000..51e3fc307 --- /dev/null +++ b/dCommon/FdbMappedFile.cpp @@ -0,0 +1,145 @@ +#include "FdbMappedFile.h" + +#include +#include +#include + +#ifdef _WIN32 +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#include +#include +#include +#endif + +FdbMappedFile::~FdbMappedFile() { + Close(); +} + +FdbMappedFile::FdbMappedFile(FdbMappedFile&& other) noexcept { + *this = std::move(other); +} + +FdbMappedFile& FdbMappedFile::operator=(FdbMappedFile&& other) noexcept { + if (this == &other) return *this; + Close(); + m_Mapped = std::exchange(other.m_Mapped, false); + m_Size = std::exchange(other.m_Size, 0); + m_Buffer = std::move(other.m_Buffer); + other.m_Buffer.clear(); + const uint8_t* data = std::exchange(other.m_Data, nullptr); + // A moved vector keeps its heap block, so the pointer stays valid; set it again anyway for clarity + m_Data = m_Mapped ? data : (m_Buffer.empty() ? nullptr : m_Buffer.data()); +#ifdef _WIN32 + m_FileHandle = std::exchange(other.m_FileHandle, nullptr); + m_MappingHandle = std::exchange(other.m_MappingHandle, nullptr); +#endif + return *this; +} + +bool FdbMappedFile::Open(const std::filesystem::path& path, bool allowMapping) { + Close(); + if (allowMapping && Map(path)) return true; + return Read(path); +} + +void FdbMappedFile::Close() { + if (m_Mapped && m_Data) { +#ifdef _WIN32 + UnmapViewOfFile(m_Data); +#else + munmap(const_cast(m_Data), static_cast(m_Size)); +#endif + } +#ifdef _WIN32 + if (m_MappingHandle) CloseHandle(static_cast(m_MappingHandle)); + if (m_FileHandle) CloseHandle(static_cast(m_FileHandle)); + m_MappingHandle = nullptr; + m_FileHandle = nullptr; +#endif + m_Data = nullptr; + m_Size = 0; + m_Mapped = false; + m_Buffer.clear(); + m_Buffer.shrink_to_fit(); +} + +bool FdbMappedFile::Map(const std::filesystem::path& path) { +#ifdef _WIN32 + HANDLE file = CreateFileW(path.wstring().c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (file == INVALID_HANDLE_VALUE) return false; + + LARGE_INTEGER size{}; + if (!GetFileSizeEx(file, &size) || size.QuadPart <= 0) { + CloseHandle(file); + return false; + } + + // Size 0/0 maps the whole file + HANDLE mapping = CreateFileMappingW(file, nullptr, PAGE_READONLY, 0, 0, nullptr); + if (!mapping) { + CloseHandle(file); + return false; + } + + const void* view = MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0); + if (!view) { + CloseHandle(mapping); + CloseHandle(file); + return false; + } + + m_FileHandle = file; + m_MappingHandle = mapping; + m_Data = static_cast(view); + m_Size = static_cast(size.QuadPart); + m_Mapped = true; + return true; +#else + const int fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (fd < 0) return false; + + struct stat info {}; + if (fstat(fd, &info) != 0 || info.st_size <= 0 || + static_cast(info.st_size) > static_cast(std::numeric_limits::max())) { + close(fd); + return false; + } + + const auto length = static_cast(info.st_size); + void* view = mmap(nullptr, length, PROT_READ, MAP_SHARED, fd, 0); + // The mapping keeps its own reference to the file + close(fd); + if (view == MAP_FAILED) return false; + + m_Data = static_cast(view); + m_Size = static_cast(info.st_size); + m_Mapped = true; + return true; +#endif +} + +bool FdbMappedFile::Read(const std::filesystem::path& path) { + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file) return false; + + const std::streamoff size = file.tellg(); + if (size <= 0) return false; + + std::vector buffer(static_cast(size)); + file.seekg(0); + if (!file.read(reinterpret_cast(buffer.data()), size)) return false; + + m_Buffer = std::move(buffer); + m_Data = m_Buffer.data(); + m_Size = static_cast(size); + m_Mapped = false; + return true; +} diff --git a/dCommon/FdbMappedFile.h b/dCommon/FdbMappedFile.h new file mode 100644 index 000000000..34fd96624 --- /dev/null +++ b/dCommon/FdbMappedFile.h @@ -0,0 +1,58 @@ +#ifndef FDBMAPPEDFILE_H +#define FDBMAPPEDFILE_H + +#include +#include +#include + +/** + * A read-only view of a whole file. The file is memory-mapped when the platform allows it + * (CreateFileMapping/MapViewOfFile on Windows, mmap elsewhere), so every process that maps the + * same file shares one copy through the OS page cache. If mapping fails the file is read into + * memory instead, and the view works the same way. + */ +class FdbMappedFile { +public: + FdbMappedFile() = default; + ~FdbMappedFile(); + + FdbMappedFile(const FdbMappedFile&) = delete; + FdbMappedFile& operator=(const FdbMappedFile&) = delete; + FdbMappedFile(FdbMappedFile&& other) noexcept; + FdbMappedFile& operator=(FdbMappedFile&& other) noexcept; + + /** + * Opens a file. Closes whatever was open before. + * + * @param path The file to open + * @param allowMapping false skips the mapping and reads the file into memory + * @return true if the file is open and not empty + */ + bool Open(const std::filesystem::path& path, bool allowMapping = true); + + void Close(); + + [[nodiscard]] bool IsOpen() const { return m_Data != nullptr; } + + // true if the view is a mapping, false if the file was read into memory + [[nodiscard]] bool IsMapped() const { return m_Mapped; } + + [[nodiscard]] const uint8_t* GetData() const { return m_Data; } + + [[nodiscard]] uint64_t GetSize() const { return m_Size; } + +private: + bool Map(const std::filesystem::path& path); + bool Read(const std::filesystem::path& path); + + const uint8_t* m_Data = nullptr; + uint64_t m_Size = 0; + bool m_Mapped = false; + std::vector m_Buffer; +#ifdef _WIN32 + void* m_FileHandle = nullptr; + void* m_MappingHandle = nullptr; +#endif +}; + +#endif // FDBMAPPEDFILE_H diff --git a/dCommon/FdbReader.cpp b/dCommon/FdbReader.cpp new file mode 100644 index 000000000..576522e22 --- /dev/null +++ b/dCommon/FdbReader.cpp @@ -0,0 +1,260 @@ +#include "FdbReader.h" + +#include +#include +#include +#include + +#include "GeneralUtils.h" + +namespace { + // Little-endian decode from bytes, independent of host byte order and alignment + uint32_t DecodeU32(const uint8_t* bytes) { + return static_cast(bytes[0]) | + (static_cast(bytes[1]) << 8) | + (static_cast(bytes[2]) << 16) | + (static_cast(bytes[3]) << 24); + } + + uint64_t DecodeU64(const uint8_t* bytes) { + return static_cast(DecodeU32(bytes)) | (static_cast(DecodeU32(bytes + 4)) << 32); + } + + float BitsToFloat(uint32_t bits) { + float value; + static_assert(sizeof(value) == sizeof(bits)); + std::memcpy(&value, &bits, sizeof(value)); + return value; + } + + // sqlite3_column_int on text: the leading integer, 0 if there is none + int64_t LeadingInteger(std::string_view text) { + size_t start = 0; + while (start < text.size() && (text[start] == ' ' || text[start] == '\t')) start++; + if (start < text.size() && text[start] == '+') start++; + int64_t value = 0; + std::from_chars(text.data() + start, text.data() + text.size(), value); + return value; + } + + bool IsText(eSqliteDataType type) { + return type == eSqliteDataType::TEXT_4 || type == eSqliteDataType::TEXT_8; + } +} + +bool FdbReader::Open(const std::filesystem::path& path, bool allowMapping) { + Close(); + if (!m_File.Open(path, allowMapping)) return false; + if (!ReadHeaders()) { + Close(); + return false; + } + return true; +} + +void FdbReader::Close() { + m_TableIndex.clear(); + m_Tables.clear(); + m_File.Close(); +} + +bool FdbReader::InBounds(uint64_t offset, uint64_t length) const { + const uint64_t size = m_File.GetSize(); + return offset <= size && length <= size - offset; +} + +bool FdbReader::ReadU32(uint64_t offset, uint32_t& out) const { + if (!InBounds(offset, 4)) return false; + out = DecodeU32(m_File.GetData() + offset); + return true; +} + +bool FdbReader::ReadI64(uint64_t offset, int64_t& out) const { + if (!InBounds(offset, 8)) return false; + out = static_cast(DecodeU64(m_File.GetData() + offset)); + return true; +} + +std::optional FdbReader::ReadCString(uint64_t offset) const { + if (!InBounds(offset, 0) || offset == m_File.GetSize()) return std::nullopt; + const auto* start = m_File.GetData() + offset; + const auto* end = static_cast(std::memchr(start, 0, static_cast(m_File.GetSize() - offset))); + if (!end) return std::nullopt; + return std::string_view(reinterpret_cast(start), static_cast(end - start)); +} + +bool FdbReader::ReadHeaders() { + uint32_t tableCount = 0; + uint32_t tableHeaders = 0; + if (!ReadU32(0, tableCount) || !ReadU32(4, tableHeaders)) return false; + if (!InBounds(tableHeaders, static_cast(tableCount) * 8)) return false; + + m_Tables.reserve(tableCount); + for (uint32_t i = 0; i < tableCount; i++) { + const uint64_t entry = static_cast(tableHeaders) + static_cast(i) * 8; + uint32_t columnHeader = 0; + uint32_t rowTop = 0; + if (!ReadU32(entry, columnHeader) || !ReadU32(entry + 4, rowTop)) return false; + + Table table; + table.m_Reader = this; + + uint32_t columnCount = 0; + uint32_t namePointer = 0; + uint32_t columns = 0; + if (!ReadU32(columnHeader, columnCount) || !ReadU32(static_cast(columnHeader) + 4, namePointer) || + !ReadU32(static_cast(columnHeader) + 8, columns)) return false; + const auto name = ReadCString(namePointer); + if (!name || !InBounds(columns, static_cast(columnCount) * 8)) return false; + table.m_Name = std::string(*name); + + table.m_Columns.reserve(columnCount); + for (uint32_t c = 0; c < columnCount; c++) { + const uint64_t column = static_cast(columns) + static_cast(c) * 8; + uint32_t type = 0; + uint32_t columnName = 0; + if (!ReadU32(column, type) || !ReadU32(column + 4, columnName)) return false; + const auto columnNameString = ReadCString(columnName); + if (!columnNameString) return false; + table.m_Columns.push_back({ GeneralUtils::Latin1ToUTF8(std::u8string_view(reinterpret_cast(columnNameString->data()), columnNameString->size())), static_cast(type) }); + } + + if (!ReadU32(rowTop, table.m_BucketCount) || !ReadU32(static_cast(rowTop) + 4, table.m_BucketArrayOffset)) return false; + if (!InBounds(table.m_BucketArrayOffset, static_cast(table.m_BucketCount) * 4)) return false; + + m_Tables.push_back(std::move(table)); + } + + // Built after the vector stops growing, since the keys point into the table names + for (uint32_t i = 0; i < m_Tables.size(); i++) m_TableIndex.emplace(m_Tables[i].m_Name, i); + return true; +} + +const FdbReader::Table* FdbReader::GetTable(std::string_view name) const { + const auto it = m_TableIndex.find(name); + return it == m_TableIndex.end() ? nullptr : &m_Tables[it->second]; +} + +int32_t FdbReader::Table::GetColumnIndex(std::string_view name) const { + for (uint32_t i = 0; i < m_Columns.size(); i++) { + if (m_Columns[i].name == name) return static_cast(i); + } + return -1; +} + +std::optional FdbReader::Table::FindFirst(int64_t key) const { + std::optional found; + ForEachRowWithKey(key, [&found](const Row& row) { + if (!found) found = row; + }); + return found; +} + +std::optional FdbReader::Table::KeyOf(const Row& row) { + switch (row.GetType(0)) { + case eSqliteDataType::INT32: + case eSqliteDataType::INT_BOOL: + return row.GetInt(0); + case eSqliteDataType::INT64: + return row.GetInt64(0); + default: + return std::nullopt; + } +} + +bool FdbReader::Row::ReadField(uint32_t column, eSqliteDataType& type, uint32_t& value) const { + if (column >= m_FieldCount) return false; + const uint64_t field = static_cast(m_FieldsOffset) + static_cast(column) * 8; + uint32_t rawType = 0; + if (!m_Reader->ReadU32(field, rawType) || !m_Reader->ReadU32(field + 4, value)) return false; + type = static_cast(rawType); + return true; +} + +eSqliteDataType FdbReader::Row::GetType(uint32_t column) const { + eSqliteDataType type{}; + uint32_t value = 0; + return ReadField(column, type, value) ? type : eSqliteDataType::NONE; +} + +int64_t FdbReader::Row::GetInt64(uint32_t column, int64_t nullValue) const { + eSqliteDataType type{}; + uint32_t value = 0; + if (!ReadField(column, type, value)) return nullValue; + switch (type) { + case eSqliteDataType::INT32: + return static_cast(value); + case eSqliteDataType::INT_BOOL: + // The conversion stores bools as 0 or 1 + return value != 0 ? 1 : 0; + case eSqliteDataType::INT64: { + int64_t wide = 0; + return m_Reader->ReadI64(value, wide) ? wide : nullValue; + } + case eSqliteDataType::REAL: + return static_cast(BitsToFloat(value)); + case eSqliteDataType::TEXT_4: + case eSqliteDataType::TEXT_8: + return LeadingInteger(GetRawString(column)); + default: + return nullValue; + } +} + +int32_t FdbReader::Row::GetInt(uint32_t column, int32_t nullValue) const { + // sqlite3_column_int keeps the low 32 bits of a wider integer + if (IsNull(column)) return nullValue; + return static_cast(static_cast(static_cast(GetInt64(column, nullValue)))); +} + +float FdbReader::Row::GetFloat(uint32_t column, float nullValue) const { + eSqliteDataType type{}; + uint32_t value = 0; + if (!ReadField(column, type, value)) return nullValue; + switch (type) { + case eSqliteDataType::REAL: + return BitsToFloat(value); + case eSqliteDataType::INT32: + case eSqliteDataType::INT_BOOL: + case eSqliteDataType::INT64: + return static_cast(GetInt64(column, 0)); + case eSqliteDataType::TEXT_4: + case eSqliteDataType::TEXT_8: { + const std::string text(GetRawString(column)); + return static_cast(std::strtod(text.c_str(), nullptr)); + } + default: + return nullValue; + } +} + +std::string_view FdbReader::Row::GetRawString(uint32_t column) const { + eSqliteDataType type{}; + uint32_t value = 0; + if (!ReadField(column, type, value) || !IsText(type)) return {}; + return m_Reader->ReadCString(value).value_or(std::string_view{}); +} + +std::string FdbReader::Row::GetString(uint32_t column, std::string_view nullValue) const { + eSqliteDataType type{}; + uint32_t value = 0; + if (!ReadField(column, type, value)) return std::string(nullValue); + switch (type) { + case eSqliteDataType::TEXT_4: + case eSqliteDataType::TEXT_8: { + const auto raw = GetRawString(column); + return GeneralUtils::Latin1ToUTF8(std::u8string_view(reinterpret_cast(raw.data()), raw.size())); + } + case eSqliteDataType::INT32: + case eSqliteDataType::INT_BOOL: + case eSqliteDataType::INT64: + return std::to_string(GetInt64(column, 0)); + case eSqliteDataType::REAL: { + char buffer[32]; + std::snprintf(buffer, sizeof(buffer), "%.15g", static_cast(BitsToFloat(value))); + return buffer; + } + default: + return std::string(nullValue); + } +} diff --git a/dCommon/FdbReader.h b/dCommon/FdbReader.h new file mode 100644 index 000000000..630bc4e31 --- /dev/null +++ b/dCommon/FdbReader.h @@ -0,0 +1,181 @@ +#ifndef FDBREADER_H +#define FDBREADER_H + +#include +#include +#include +#include +#include +#include +#include + +#include "FdbMappedFile.h" +#include "eSqliteDataType.h" + +/** + * Reads the client's cdclient.fdb in place, without loading it. + * + * Layout (all integers little-endian, every pointer an absolute u32 file offset, -1 meaning none): + * file header: u32 table count, ptr to table header array + * table header: ptr to column header, ptr to row top header (one pair per table, sorted by name) + * column header: u32 column count, ptr to table name, ptr to column array + * column: u32 data type, ptr to column name + * row top: u32 bucket count (a power of two, may be 0), ptr to bucket array + * bucket array: one ptr per bucket to the first row info of the bucket + * row info: ptr to row data, ptr to the next row info in the same bucket + * row data: u32 field count, ptr to field array + * field: u32 data type, u32 value + * + * A field value holds the number itself for int32, bool and float, and a pointer for int64 + * (to 8 bytes) and text (to a null-terminated latin-1 string). A row sits in bucket + * (u32)(first field) % bucket count, for int32 first columns. Tables are found by name and + * rows by their first column, so a lookup only touches a few pages of the file. + */ +class FdbReader { +public: + class Row; + class Table; + + struct Column { + std::string name; + eSqliteDataType type{}; + }; + + class Row { + public: + [[nodiscard]] uint32_t GetFieldCount() const { return m_FieldCount; } + + // NONE for a null field or a column past the end of the row + [[nodiscard]] eSqliteDataType GetType(uint32_t column) const; + + [[nodiscard]] bool IsNull(uint32_t column) const { return GetType(column) == eSqliteDataType::NONE; } + + // These convert between types the way SQLite does for the converted CDServer.sqlite, + // and return nullValue for a null field. + [[nodiscard]] int32_t GetInt(uint32_t column, int32_t nullValue = 0) const; + [[nodiscard]] int64_t GetInt64(uint32_t column, int64_t nullValue = 0) const; + [[nodiscard]] float GetFloat(uint32_t column, float nullValue = 0.0f) const; + [[nodiscard]] bool GetBool(uint32_t column, bool nullValue = false) const { return GetInt(column, nullValue ? 1 : 0) != 0; } + + // UTF-8 + [[nodiscard]] std::string GetString(uint32_t column, std::string_view nullValue = "") const; + + // The latin-1 bytes as stored, for text fields; empty otherwise + [[nodiscard]] std::string_view GetRawString(uint32_t column) const; + + private: + friend class FdbReader; + friend class Table; + Row(const FdbReader* reader, uint32_t fieldsOffset, uint32_t fieldCount) + : m_Reader(reader), m_FieldsOffset(fieldsOffset), m_FieldCount(fieldCount) {} + + bool ReadField(uint32_t column, eSqliteDataType& type, uint32_t& value) const; + + const FdbReader* m_Reader; + uint32_t m_FieldsOffset; + uint32_t m_FieldCount; + }; + + class Table { + public: + [[nodiscard]] const std::string& GetName() const { return m_Name; } + [[nodiscard]] const std::vector& GetColumns() const { return m_Columns; } + [[nodiscard]] uint32_t GetBucketCount() const { return m_BucketCount; } + + // The column's index, or -1 when the table has no such column + [[nodiscard]] int32_t GetColumnIndex(std::string_view name) const; + + // Calls f(const Row&) for each row whose first column equals key, in file order + template + void ForEachRowWithKey(int64_t key, F&& f) const { + if (m_BucketCount == 0) return; + const uint32_t bucket = static_cast(static_cast(key) % m_BucketCount); + WalkBucket(bucket, [&](const Row& row) { + if (KeyOf(row) == key) f(row); + }); + } + + // The first row whose first column equals key + [[nodiscard]] std::optional FindFirst(int64_t key) const; + + // Calls f(const Row&) for every row, bucket by bucket + template + void ForEachRow(F&& f) const { + for (uint32_t bucket = 0; bucket < m_BucketCount; bucket++) WalkBucket(bucket, f); + } + + // The first column as an integer, the value rows are bucketed by + [[nodiscard]] static std::optional KeyOf(const Row& row); + + private: + friend class FdbReader; + + template + void WalkBucket(uint32_t bucket, F&& f) const { + uint32_t rowInfo = 0; + if (!m_Reader->ReadU32(m_BucketArrayOffset + static_cast(bucket) * 4, rowInfo)) return; + // A bound on the chain length so a damaged file can't loop forever + uint64_t guard = m_Reader->GetSize() / 8; + while (rowInfo != NO_POINTER && guard-- > 0) { + uint32_t rowData = 0; + uint32_t next = NO_POINTER; + if (!m_Reader->ReadU32(rowInfo, rowData) || !m_Reader->ReadU32(static_cast(rowInfo) + 4, next)) return; + uint32_t fieldCount = 0; + uint32_t fields = 0; + if (m_Reader->ReadU32(rowData, fieldCount) && m_Reader->ReadU32(static_cast(rowData) + 4, fields) && + m_Reader->InBounds(fields, static_cast(fieldCount) * 8)) { + f(Row(m_Reader, fields, fieldCount)); + } + rowInfo = next; + } + } + + const FdbReader* m_Reader = nullptr; + std::string m_Name; + std::vector m_Columns; + uint32_t m_BucketCount = 0; + uint32_t m_BucketArrayOffset = 0; + }; + + static constexpr uint32_t NO_POINTER = 0xFFFFFFFF; + + FdbReader() = default; + FdbReader(const FdbReader&) = delete; + FdbReader& operator=(const FdbReader&) = delete; + + /** + * Opens an fdb file and reads its table and column headers (the rows stay in the file). + * + * @param path The fdb file + * @param allowMapping false reads the file into memory instead of mapping it + * @return true if the file was opened and its headers are sound + */ + bool Open(const std::filesystem::path& path, bool allowMapping = true); + + void Close(); + + [[nodiscard]] bool IsOpen() const { return m_File.IsOpen(); } + [[nodiscard]] bool IsMapped() const { return m_File.IsMapped(); } + [[nodiscard]] uint64_t GetSize() const { return m_File.GetSize(); } + + // The table with this exact name, or nullptr + [[nodiscard]] const Table* GetTable(std::string_view name) const; + + [[nodiscard]] const std::vector& GetTables() const { return m_Tables; } + + // Little-endian reads that fail instead of reading past the end of the file + [[nodiscard]] bool ReadU32(uint64_t offset, uint32_t& out) const; + [[nodiscard]] bool ReadI64(uint64_t offset, int64_t& out) const; + [[nodiscard]] bool InBounds(uint64_t offset, uint64_t length) const; + // The null-terminated string at offset, without the terminator + [[nodiscard]] std::optional ReadCString(uint64_t offset) const; + +private: + bool ReadHeaders(); + + FdbMappedFile m_File; + std::vector
m_Tables; + std::unordered_map m_TableIndex; +}; + +#endif // FDBREADER_H diff --git a/tests/FdbTestWriter.h b/tests/FdbTestWriter.h new file mode 100644 index 000000000..316cd055f --- /dev/null +++ b/tests/FdbTestWriter.h @@ -0,0 +1,157 @@ +#ifndef FDBTESTWRITER_H +#define FDBTESTWRITER_H + +// Writes small fdb files for tests, in the client's layout (see FdbReader.h) + +#include +#include +#include +#include +#include +#include + +#include "eSqliteDataType.h" + +namespace FdbTestWriter { + struct Value { + eSqliteDataType type = eSqliteDataType::NONE; + int64_t integer = 0; + float real = 0.0f; + std::string text; // latin-1 bytes + + static Value Null() { return {}; } + static Value Int(int32_t v) { Value r; r.type = eSqliteDataType::INT32; r.integer = v; return r; } + static Value Bool(bool v) { Value r; r.type = eSqliteDataType::INT_BOOL; r.integer = v ? 1 : 0; return r; } + static Value Int64(int64_t v) { Value r; r.type = eSqliteDataType::INT64; r.integer = v; return r; } + static Value Real(float v) { Value r; r.type = eSqliteDataType::REAL; r.real = v; return r; } + static Value Text(std::string v, bool wide = false) { + Value r; r.type = wide ? eSqliteDataType::TEXT_8 : eSqliteDataType::TEXT_4; r.text = std::move(v); return r; + } + }; + + struct Column { + std::string name; + eSqliteDataType type; + }; + + struct Table { + std::string name; + std::vector columns; + uint32_t bucketCount = 0; + std::vector> rows; + }; + + class Buffer { + public: + uint32_t Alloc(uint32_t size) { + const auto offset = static_cast(m_Bytes.size()); + m_Bytes.resize(m_Bytes.size() + size); + return offset; + } + + void PutU32(uint32_t offset, uint32_t value) { + for (uint32_t i = 0; i < 4; i++) m_Bytes[offset + i] = static_cast(value >> (i * 8)); + } + + uint32_t String(const std::string& text) { + const auto offset = Alloc(static_cast(text.size()) + 1); + if (!text.empty()) std::memcpy(m_Bytes.data() + offset, text.data(), text.size()); + return offset; + } + + uint32_t I64(int64_t value) { + const auto offset = Alloc(8); + PutU32(offset, static_cast(static_cast(value))); + PutU32(offset + 4, static_cast(static_cast(value) >> 32)); + return offset; + } + + std::vector& Bytes() { return m_Bytes; } + + private: + std::vector m_Bytes; + }; + + inline std::vector Write(const std::vector
& tables) { + constexpr uint32_t NONE = 0xFFFFFFFF; + Buffer out; + out.Alloc(8); + const auto headers = out.Alloc(static_cast(tables.size()) * 8); + out.PutU32(0, static_cast(tables.size())); + out.PutU32(4, headers); + + for (uint32_t t = 0; t < tables.size(); t++) { + const auto& table = tables[t]; + + const auto columnHeader = out.Alloc(12); + const auto columns = out.Alloc(static_cast(table.columns.size()) * 8); + out.PutU32(columnHeader, static_cast(table.columns.size())); + out.PutU32(columnHeader + 4, out.String(table.name)); + out.PutU32(columnHeader + 8, columns); + for (uint32_t c = 0; c < table.columns.size(); c++) { + out.PutU32(columns + c * 8, static_cast(table.columns[c].type)); + out.PutU32(columns + c * 8 + 4, out.String(table.columns[c].name)); + } + + const auto rowTop = out.Alloc(8); + const auto buckets = out.Alloc(table.bucketCount * 4); + out.PutU32(rowTop, table.bucketCount); + out.PutU32(rowTop + 4, buckets); + std::vector lastInBucket(table.bucketCount, NONE); + for (uint32_t b = 0; b < table.bucketCount; b++) out.PutU32(buckets + b * 4, NONE); + + for (const auto& row : table.rows) { + const auto fields = out.Alloc(static_cast(row.size()) * 8); + for (uint32_t c = 0; c < row.size(); c++) { + const auto& value = row[c]; + uint32_t raw = 0; + switch (value.type) { + case eSqliteDataType::INT32: + case eSqliteDataType::INT_BOOL: + raw = static_cast(static_cast(value.integer)); + break; + case eSqliteDataType::REAL: + std::memcpy(&raw, &value.real, sizeof(raw)); + break; + case eSqliteDataType::INT64: + raw = out.I64(value.integer); + break; + case eSqliteDataType::TEXT_4: + case eSqliteDataType::TEXT_8: + raw = out.String(value.text); + break; + default: + break; + } + out.PutU32(fields + c * 8, static_cast(value.type)); + out.PutU32(fields + c * 8 + 4, raw); + } + + const auto rowData = out.Alloc(8); + out.PutU32(rowData, static_cast(row.size())); + out.PutU32(rowData + 4, fields); + + const auto rowInfo = out.Alloc(8); + out.PutU32(rowInfo, rowData); + out.PutU32(rowInfo + 4, NONE); + + // Bucketed by the first column, appended to the end of the bucket's chain + const auto bucket = static_cast(static_cast(row[0].integer) % table.bucketCount); + if (lastInBucket[bucket] == NONE) out.PutU32(buckets + bucket * 4, rowInfo); + else out.PutU32(lastInBucket[bucket] + 4, rowInfo); + lastInBucket[bucket] = rowInfo; + } + + out.PutU32(headers + t * 8, columnHeader); + out.PutU32(headers + t * 8 + 4, rowTop); + } + return out.Bytes(); + } + + inline void WriteFile(const std::filesystem::path& path, const std::vector& bytes) { + std::ofstream file(path, std::ios::binary | std::ios::trunc); + file.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + } +}; + +#endif // FDBTESTWRITER_H diff --git a/tests/dCommonTests/CMakeLists.txt b/tests/dCommonTests/CMakeLists.txt index 1abaee7b9..98873d095 100644 --- a/tests/dCommonTests/CMakeLists.txt +++ b/tests/dCommonTests/CMakeLists.txt @@ -34,6 +34,7 @@ set(DCOMMONTEST_SOURCES "BindAddressTests.cpp" "TrafficStatsTests.cpp" "Sd0Tests.cpp" + "FdbReaderTests.cpp" ) add_subdirectory(dEnumsTests) diff --git a/tests/dCommonTests/FdbReaderTests.cpp b/tests/dCommonTests/FdbReaderTests.cpp new file mode 100644 index 000000000..3607b51ee --- /dev/null +++ b/tests/dCommonTests/FdbReaderTests.cpp @@ -0,0 +1,216 @@ +#include + +#include +#include +#include + +#include "FdbReader.h" +#include "../FdbTestWriter.h" + +using FdbTestWriter::Value; + +namespace { + std::vector SampleTables() { + FdbTestWriter::Table items; + items.name = "Items"; + items.columns = { + { "id", eSqliteDataType::INT32 }, + { "name", eSqliteDataType::TEXT_4 }, + { "big", eSqliteDataType::INT64 }, + { "scale", eSqliteDataType::REAL }, + { "flag", eSqliteDataType::INT_BOOL }, + { "notes", eSqliteDataType::TEXT_8 }, + }; + items.bucketCount = 4; + // 1, 5 and -3 all land in bucket 1 + items.rows = { + { Value::Int(1), Value::Text("one"), Value::Int64(0x123456789ALL), Value::Real(1.5f), Value::Bool(true), Value::Text("a", true) }, + { Value::Int(2), Value::Text("two"), Value::Int64(-2), Value::Real(-0.25f), Value::Bool(false), Value::Null() }, + { Value::Int(5), Value::Text("caf\xE9"), Value::Null(), Value::Null(), Value::Null(), Value::Text("") }, + { Value::Int(1), Value::Text("one again"), Value::Int64(1), Value::Real(2.0f), Value::Bool(false), Value::Null() }, + { Value::Int(-3), Value::Text("negative"), Value::Int64(INT64_MIN), Value::Real(3.0f), Value::Bool(true), Value::Text("42abc") }, + }; + + FdbTestWriter::Table empty; + empty.name = "Empty"; + empty.columns = { { "id", eSqliteDataType::INT32 } }; + empty.bucketCount = 0; + + return { empty, items }; + } + + class FdbReaderTest : public ::testing::TestWithParam { + protected: + void SetUp() override { + // Parameterized test names contain '/' + std::string name = ::testing::UnitTest::GetInstance()->current_test_info()->name(); + for (auto& c : name) if (c == '/') c = '_'; + m_Path = std::filesystem::temp_directory_path() / ("dlu_fdb_reader_" + name + ".fdb"); + FdbTestWriter::WriteFile(m_Path, FdbTestWriter::Write(SampleTables())); + } + + void TearDown() override { + m_Reader.Close(); + std::error_code error; + std::filesystem::remove(m_Path, error); + } + + std::vector NamesWithKey(const FdbReader::Table& table, int64_t key) { + std::vector names; + table.ForEachRowWithKey(key, [&](const FdbReader::Row& row) { names.push_back(row.GetString(1)); }); + return names; + } + + std::filesystem::path m_Path; + FdbReader m_Reader; + }; +} + +// true maps the file, false reads it into memory (the fallback) +INSTANTIATE_TEST_SUITE_P(MapOrRead, FdbReaderTest, ::testing::Values(true, false)); + +TEST_P(FdbReaderTest, OpensAndFindsTables) { + ASSERT_TRUE(m_Reader.Open(m_Path, GetParam())); + EXPECT_EQ(m_Reader.IsMapped(), GetParam()); + EXPECT_EQ(m_Reader.GetTables().size(), 2u); + + const auto* items = m_Reader.GetTable("Items"); + ASSERT_NE(items, nullptr); + EXPECT_EQ(items->GetBucketCount(), 4u); + ASSERT_EQ(items->GetColumns().size(), 6u); + EXPECT_EQ(items->GetColumns()[2].name, "big"); + EXPECT_EQ(items->GetColumns()[2].type, eSqliteDataType::INT64); + EXPECT_EQ(items->GetColumnIndex("notes"), 5); + EXPECT_EQ(items->GetColumnIndex("missing"), -1); + + EXPECT_EQ(m_Reader.GetTable("items"), nullptr); + EXPECT_EQ(m_Reader.GetTable("Nope"), nullptr); + + const auto* empty = m_Reader.GetTable("Empty"); + ASSERT_NE(empty, nullptr); + EXPECT_FALSE(empty->FindFirst(0).has_value()); + uint32_t rows = 0; + empty->ForEachRow([&](const FdbReader::Row&) { rows++; }); + EXPECT_EQ(rows, 0u); +} + +TEST_P(FdbReaderTest, LooksUpByKeyThroughCollisions) { + ASSERT_TRUE(m_Reader.Open(m_Path, GetParam())); + const auto& items = *m_Reader.GetTable("Items"); + + // Same bucket, filtered by key, file order kept + EXPECT_EQ(NamesWithKey(items, 1), (std::vector{ "one", "one again" })); + EXPECT_EQ(NamesWithKey(items, 5), (std::vector{ "caf\xC3\xA9" })); + EXPECT_EQ(NamesWithKey(items, -3), (std::vector{ "negative" })); + EXPECT_EQ(NamesWithKey(items, 2), (std::vector{ "two" })); + EXPECT_TRUE(NamesWithKey(items, 9).empty()); + EXPECT_TRUE(NamesWithKey(items, 3).empty()); + + const auto first = items.FindFirst(1); + ASSERT_TRUE(first.has_value()); + EXPECT_EQ(first->GetString(1), "one"); + + uint32_t rows = 0; + items.ForEachRow([&](const FdbReader::Row&) { rows++; }); + EXPECT_EQ(rows, 5u); +} + +TEST_P(FdbReaderTest, ReadsTypedValues) { + ASSERT_TRUE(m_Reader.Open(m_Path, GetParam())); + const auto& items = *m_Reader.GetTable("Items"); + + const auto one = items.FindFirst(1); + ASSERT_TRUE(one.has_value()); + EXPECT_EQ(one->GetFieldCount(), 6u); + EXPECT_EQ(one->GetInt(0), 1); + EXPECT_EQ(one->GetInt64(2), 0x123456789ALL); + // Low 32 bits, as sqlite3_column_int + EXPECT_EQ(one->GetInt(2), 0x3456789A); + EXPECT_FLOAT_EQ(one->GetFloat(3), 1.5f); + EXPECT_TRUE(one->GetBool(4)); + EXPECT_EQ(one->GetType(5), eSqliteDataType::TEXT_8); + EXPECT_EQ(one->GetString(5), "a"); + + const auto two = items.FindFirst(2); + ASSERT_TRUE(two.has_value()); + EXPECT_EQ(two->GetInt64(2), -2); + EXPECT_FLOAT_EQ(two->GetFloat(3), -0.25f); + EXPECT_FALSE(two->GetBool(4, true)); + EXPECT_TRUE(two->IsNull(5)); + EXPECT_EQ(two->GetString(5, "fallback"), "fallback"); + + const auto negative = items.FindFirst(-3); + ASSERT_TRUE(negative.has_value()); + EXPECT_EQ(negative->GetInt64(2), INT64_MIN); + // Text read as a number, as SQLite does + EXPECT_EQ(negative->GetInt(5), 42); + // A number read as text + EXPECT_EQ(negative->GetString(0), "-3"); + + // Nulls give the caller's default + const auto five = items.FindFirst(5); + ASSERT_TRUE(five.has_value()); + EXPECT_TRUE(five->IsNull(2)); + EXPECT_EQ(five->GetInt(2, -1), -1); + EXPECT_EQ(five->GetInt64(2, -7), -7); + EXPECT_FLOAT_EQ(five->GetFloat(3, -1.0f), -1.0f); + EXPECT_TRUE(five->GetBool(4, true)); + EXPECT_EQ(five->GetRawString(1), "caf\xE9"); + EXPECT_EQ(five->GetString(5, "x"), ""); + + // Past the last column reads as null + EXPECT_TRUE(five->IsNull(99)); + EXPECT_EQ(five->GetInt(99, 11), 11); +} + +TEST(FdbReaderFailureTest, MissingEmptyAndTruncatedFiles) { + const auto dir = std::filesystem::temp_directory_path(); + FdbReader reader; + EXPECT_FALSE(reader.Open(dir / "dlu_fdb_reader_does_not_exist.fdb")); + EXPECT_FALSE(reader.IsOpen()); + + const auto emptyPath = dir / "dlu_fdb_reader_empty.fdb"; + FdbTestWriter::WriteFile(emptyPath, {}); + EXPECT_FALSE(reader.Open(emptyPath)); + EXPECT_FALSE(reader.Open(emptyPath, false)); + + // Cut before the row data: the headers are still sound, the rows read as missing instead of past the end + auto bytes = FdbTestWriter::Write(SampleTables()); + const auto truncatedPath = dir / "dlu_fdb_reader_truncated.fdb"; + FdbTestWriter::WriteFile(truncatedPath, std::vector(bytes.begin(), bytes.begin() + 20)); + EXPECT_FALSE(reader.Open(truncatedPath)); + + // A table count far past the end of the file + bytes[0] = 0xFF; + bytes[1] = 0xFF; + const auto badCountPath = dir / "dlu_fdb_reader_bad_count.fdb"; + FdbTestWriter::WriteFile(badCountPath, bytes); + EXPECT_FALSE(reader.Open(badCountPath)); + + std::error_code error; + std::filesystem::remove(emptyPath, error); + std::filesystem::remove(truncatedPath, error); + std::filesystem::remove(badCountPath, error); +} + +TEST(FdbReaderFailureTest, MappedFileMovesAndCloses) { + const auto path = std::filesystem::temp_directory_path() / "dlu_fdb_mapped_move.fdb"; + FdbTestWriter::WriteFile(path, { 1, 2, 3, 4 }); + for (const bool map : { true, false }) { + FdbMappedFile file; + ASSERT_TRUE(file.Open(path, map)); + EXPECT_EQ(file.IsMapped(), map); + EXPECT_EQ(file.GetSize(), 4u); + + FdbMappedFile moved(std::move(file)); + EXPECT_FALSE(file.IsOpen()); + ASSERT_TRUE(moved.IsOpen()); + EXPECT_EQ(moved.GetData()[3], 4); + + moved.Close(); + EXPECT_FALSE(moved.IsOpen()); + EXPECT_EQ(moved.GetSize(), 0u); + } + std::error_code error; + std::filesystem::remove(path, error); +}