From ba41bf0311406483c55e5577bc3c8451f2808684 Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Wed, 4 Nov 2026 21:14:28 -0600 Subject: [PATCH] feat: read the bounding boxes of the client's bricks from its brickdb BrickBounds reads the of every Primitives/.xml in the client's res/brickdb.zip (through the asset manager, so packed and unpacked clients both work), by design ID. A design listed only as another primitive's alias gets that primitive's box. Get() reads the file the first time it is called and keeps the boxes for every later call. The zip is read by a small reader in dCommon (Zip::ForEachEntry) that handles stored and deflated entries, with a raw inflate added to ZCompression. Co-Authored-By: Claude Opus 5.5 --- dCommon/CMakeLists.txt | 1 + dCommon/Zip.cpp | 77 ++++++++++++ dCommon/Zip.h | 18 +++ dCommon/dClient/BrickBounds.cpp | 99 +++++++++++++++ dCommon/dClient/BrickBounds.h | 42 +++++++ dCommon/dClient/CMakeLists.txt | 1 + tests/dCommonTests/BrickBoundsTests.cpp | 154 ++++++++++++++++++++++++ tests/dCommonTests/CMakeLists.txt | 1 + 8 files changed, 393 insertions(+) create mode 100644 dCommon/Zip.cpp create mode 100644 dCommon/Zip.h create mode 100644 dCommon/dClient/BrickBounds.cpp create mode 100644 dCommon/dClient/BrickBounds.h create mode 100644 tests/dCommonTests/BrickBoundsTests.cpp diff --git a/dCommon/CMakeLists.txt b/dCommon/CMakeLists.txt index 0aafc0622..66636b435 100644 --- a/dCommon/CMakeLists.txt +++ b/dCommon/CMakeLists.txt @@ -17,6 +17,7 @@ set(DCOMMON_SOURCES "NiQuaternion.cpp" "Demangler.cpp" "ZCompression.cpp" + "Zip.cpp" "BrickByBrickFix.cpp" "BinaryPathFinder.cpp" "FdbToSqlite.cpp" diff --git a/dCommon/Zip.cpp b/dCommon/Zip.cpp new file mode 100644 index 000000000..81b08c2f3 --- /dev/null +++ b/dCommon/Zip.cpp @@ -0,0 +1,77 @@ +#include "Zip.h" + +#include "ZCompression.h" + +#include +#include + +namespace { + constexpr uint32_t END_SIGNATURE = 0x06054b50; + constexpr uint32_t CENTRAL_SIGNATURE = 0x02014b50; + constexpr uint32_t LOCAL_SIGNATURE = 0x04034b50; + constexpr uint64_t END_SIZE = 22; + constexpr uint64_t MAX_COMMENT_SIZE = 65535; + + // Zip numbers are little endian + template + bool ReadAt(const std::string_view data, const uint64_t offset, T& value) { + if (offset > data.size() || sizeof(T) > data.size() - offset) return false; + value = 0; + for (uint64_t i = 0; i < sizeof(T); i++) value |= static_cast(static_cast(data[offset + i])) << (8 * i); + return true; + } +} + +bool Zip::ForEachEntry(const std::string_view zip, const std::function& wanted, + const std::function& visit) { + if (zip.size() < END_SIZE) return false; + + // The end of central directory record: the last one in the file, after which only its comment can come + uint64_t end = 0; + bool found = false; + const uint64_t lowest = zip.size() > END_SIZE + MAX_COMMENT_SIZE ? zip.size() - END_SIZE - MAX_COMMENT_SIZE : 0; + for (uint64_t i = zip.size() - END_SIZE + 1; i-- > lowest;) { + uint32_t signature{}; + if (ReadAt(zip, i, signature) && signature == END_SIGNATURE) { + end = i; + found = true; + break; + } + } + if (!found) return false; + + uint16_t entries{}; + uint32_t directoryOffset{}; + if (!ReadAt(zip, end + 10, entries) || !ReadAt(zip, end + 16, directoryOffset)) return false; + + uint64_t offset = directoryOffset; + for (uint16_t i = 0; i < entries; i++) { + uint32_t signature{}, compressedSize{}, size{}, localOffset{}; + uint16_t method{}, nameLength{}, extraLength{}, commentLength{}; + if (!ReadAt(zip, offset, signature) || signature != CENTRAL_SIGNATURE || !ReadAt(zip, offset + 10, method) || + !ReadAt(zip, offset + 20, compressedSize) || !ReadAt(zip, offset + 24, size) || !ReadAt(zip, offset + 28, nameLength) || + !ReadAt(zip, offset + 30, extraLength) || !ReadAt(zip, offset + 32, commentLength) || !ReadAt(zip, offset + 42, localOffset) || + offset + 46 + nameLength > zip.size()) { + return false; + } + const auto name = zip.substr(offset + 46, nameLength); + offset += 46 + nameLength + extraLength + commentLength; + if (!wanted(name)) continue; + + // The data follows the local header, whose name and extra field can differ in length from the central one's + uint16_t localNameLength{}, localExtraLength{}; + if (!ReadAt(zip, localOffset, signature) || signature != LOCAL_SIGNATURE || !ReadAt(zip, localOffset + 26, localNameLength) || + !ReadAt(zip, localOffset + 28, localExtraLength)) { + continue; + } + const uint64_t dataStart = static_cast(localOffset) + 30 + localNameLength + localExtraLength; + if (dataStart > zip.size() || compressedSize > zip.size() - dataStart) continue; + const auto data = zip.substr(dataStart, compressedSize); + if (method == 0) { + visit(name, data); + } else if (method == 8) { + if (const auto inflated = ZCompression::InflateRaw(data, size)) visit(name, *inflated); + } + } + return true; +} diff --git a/dCommon/Zip.h b/dCommon/Zip.h new file mode 100644 index 000000000..e28d6e57d --- /dev/null +++ b/dCommon/Zip.h @@ -0,0 +1,18 @@ +// Darkflame Universe +// Copyright 2025 + +#ifndef ZIP_H +#define ZIP_H + +#include +#include + +namespace Zip { + // Calls `visit` with the name and contents of every entry of the zip archive `zip` whose name `wanted` accepts, in + // the archive's order. Stored and deflated entries are read; others, and entries that fail to read, are skipped. + // Returns false when the archive's directory can't be read. + bool ForEachEntry(std::string_view zip, const std::function& wanted, + const std::function& visit); +}; + +#endif //!ZIP_H diff --git a/dCommon/dClient/BrickBounds.cpp b/dCommon/dClient/BrickBounds.cpp new file mode 100644 index 000000000..10d9d83cb --- /dev/null +++ b/dCommon/dClient/BrickBounds.cpp @@ -0,0 +1,99 @@ +#include "BrickBounds.h" + +#include "AssetManager.h" +#include "GeneralUtils.h" +#include "Logger.h" +#include "Zip.h" + +#include "tinyxml2.h" + +#include +#include +#include + +namespace { + constexpr std::string_view PRIMITIVES = "primitives/"; + constexpr std::string_view XML = ".xml"; + + // The design ID of a Primitives/.xml entry + std::optional PrimitiveDesign(const std::string_view name) { + if (name.size() <= PRIMITIVES.size() + XML.size()) return std::nullopt; + std::string lower(name); + std::transform(lower.begin(), lower.end(), lower.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); + if (!lower.starts_with(PRIMITIVES) || !lower.ends_with(XML)) return std::nullopt; + const auto id = std::string_view(lower).substr(PRIMITIVES.size(), lower.size() - PRIMITIVES.size() - XML.size()); + if (id.empty() || !std::all_of(id.begin(), id.end(), [](unsigned char c) { return std::isdigit(c); })) return std::nullopt; + return GeneralUtils::TryParse(id); + } +} + +namespace { + std::optional ReadBounds(const tinyxml2::XMLElement* primitive) { + const auto* bounding = primitive ? primitive->FirstChildElement("Bounding") : nullptr; + const auto* aabb = bounding ? bounding->FirstChildElement("AABB") : nullptr; + if (!aabb) return std::nullopt; + + BrickBounds::Bounds bounds; + if (aabb->QueryFloatAttribute("minX", &bounds.min.x) != tinyxml2::XML_SUCCESS || + aabb->QueryFloatAttribute("minY", &bounds.min.y) != tinyxml2::XML_SUCCESS || + aabb->QueryFloatAttribute("minZ", &bounds.min.z) != tinyxml2::XML_SUCCESS || + aabb->QueryFloatAttribute("maxX", &bounds.max.x) != tinyxml2::XML_SUCCESS || + aabb->QueryFloatAttribute("maxY", &bounds.max.y) != tinyxml2::XML_SUCCESS || + aabb->QueryFloatAttribute("maxZ", &bounds.max.z) != tinyxml2::XML_SUCCESS) { + return std::nullopt; + } + return bounds; + } +} + +std::optional BrickBounds::ParsePrimitive(const std::string_view xml) { + tinyxml2::XMLDocument doc; + if (doc.Parse(xml.data(), xml.size()) != tinyxml2::XML_SUCCESS) return std::nullopt; + return ReadBounds(doc.FirstChildElement("LEGOPrimitive")); +} + +BrickBounds::DesignBounds BrickBounds::ParseBrickDb(const std::string_view zip) { + DesignBounds bounds; + DesignBounds aliases; + Zip::ForEachEntry(zip, [](std::string_view name) { return PrimitiveDesign(name).has_value(); }, + [&bounds, &aliases](std::string_view name, std::string_view data) { + const auto design = PrimitiveDesign(name); + tinyxml2::XMLDocument doc; + if (!design || doc.Parse(data.data(), data.size()) != tinyxml2::XML_SUCCESS) return; + const auto* primitive = doc.FirstChildElement("LEGOPrimitive"); + const auto box = ReadBounds(primitive); + if (!box) return; + bounds[*design] = *box; + + // : other design IDs of the same brick + const auto* annotations = primitive->FirstChildElement("Annotations"); + for (const auto* annotation = annotations ? annotations->FirstChildElement("Annotation") : nullptr; annotation; annotation = annotation->NextSiblingElement("Annotation")) { + const char* list = annotation->Attribute("aliases"); + if (!list) continue; + for (const auto& alias : GeneralUtils::SplitString(list, ';')) { + if (const auto id = GeneralUtils::TryParse(alias)) aliases.emplace(*id, *box); + } + } + }); + // A primitive of its own wins over being another's alias + for (const auto& [design, box] : aliases) bounds.emplace(design, box); + return bounds; +} + +BrickBounds::DesignBounds BrickBounds::Load(const AssetManager& assets) { + char* data = nullptr; + uint32_t length = 0; + if (!assets.GetFile("brickdb.zip", &data, &length)) { + LOG("Failed to read brickdb.zip; models are placed by their bricks' origins"); + return {}; + } + auto bounds = ParseBrickDb(std::string_view(data, length)); + free(data); + LOG("Read the bounds of %u brick designs", static_cast(bounds.size())); + return bounds; +} + +const BrickBounds::DesignBounds& BrickBounds::Get(const AssetManager* assets) { + static const DesignBounds bounds = assets ? Load(*assets) : DesignBounds{}; + return bounds; +} diff --git a/dCommon/dClient/BrickBounds.h b/dCommon/dClient/BrickBounds.h new file mode 100644 index 000000000..a723c8ae8 --- /dev/null +++ b/dCommon/dClient/BrickBounds.h @@ -0,0 +1,42 @@ +// Darkflame Universe +// Copyright 2025 + +#ifndef BRICKBOUNDS_H +#define BRICKBOUNDS_H + +#include +#include +#include +#include + +#include "NiPoint3.h" + +class AssetManager; + +// The bounding boxes of the client's bricks, from its brickdb (res/brickdb.zip) +namespace BrickBounds { + // A brick's box around its own origin + struct Bounds { + NiPoint3 min; + NiPoint3 max; + }; + + // Design ID -> its box + using DesignBounds = std::unordered_map; + + // The of one primitive (Primitives/.xml) + [[nodiscard]] std::optional ParsePrimitive(std::string_view xml); + + // The box of every primitive in a brickdb.zip. A design listed only as another primitive's alias gets that + // primitive's box. + [[nodiscard]] DesignBounds ParseBrickDb(std::string_view zip); + + // Reads brickdb.zip through `assets`; empty if it can't be read + [[nodiscard]] DesignBounds Load(const AssetManager& assets); + + // The boxes read by Load the first time this is called, kept for every later call (`assets` is only used then). + // Empty without `assets`. + [[nodiscard]] const DesignBounds& Get(const AssetManager* assets); +}; + +#endif //!BRICKBOUNDS_H diff --git a/dCommon/dClient/CMakeLists.txt b/dCommon/dClient/CMakeLists.txt index 017a22a52..83dc640c0 100644 --- a/dCommon/dClient/CMakeLists.txt +++ b/dCommon/dClient/CMakeLists.txt @@ -1,5 +1,6 @@ set(DCOMMON_DCLIENT_SOURCES "AssetManager.cpp" + "BrickBounds.cpp" "PackIndex.cpp" "Pack.cpp" PARENT_SCOPE diff --git a/tests/dCommonTests/BrickBoundsTests.cpp b/tests/dCommonTests/BrickBoundsTests.cpp new file mode 100644 index 000000000..a3e2515cf --- /dev/null +++ b/tests/dCommonTests/BrickBoundsTests.cpp @@ -0,0 +1,154 @@ +#include "gtest/gtest.h" + +#include "AssetManager.h" +#include "BrickBounds.h" +#include "ZCompression.h" +#include "Zip.h" +#include "dCommonDependencies.h" + +#include +#include +#include +#include + +namespace { + std::string Primitive(const std::string& aliases, const std::string& aabb) { + return R"()" + R"()" + R"()" + aabb + R"()" + R"()"; + } + + const std::string BRICK_2X2_AABB = R"()"; + + struct Entry { + std::string name; + std::string data; + bool deflate; + }; + + void Write16(std::string& out, uint16_t value) { + out += static_cast(value & 0xFF); + out += static_cast(value >> 8); + } + + void Write32(std::string& out, uint32_t value) { + Write16(out, static_cast(value & 0xFFFF)); + Write16(out, static_cast(value >> 16)); + } + + // Raw deflate data: zlib's stream without its 2 byte header and 4 byte checksum + std::string Deflate(const std::string& data) { + std::string out(ZCompression::GetMaxCompressedLength(static_cast(data.size())), '\0'); + const auto size = ZCompression::Compress(reinterpret_cast(data.data()), static_cast(data.size()), reinterpret_cast(out.data()), static_cast(out.size())); + return out.substr(2, size - 6); + } + + // A zip archive (crc left at 0: it isn't checked) + std::string MakeZip(const std::vector& entries) { + std::string zip; + std::string directory; + for (const auto& entry : entries) { + const auto data = entry.deflate ? Deflate(entry.data) : entry.data; + const auto method = static_cast(entry.deflate ? 8 : 0); + const auto offset = static_cast(zip.size()); + Write32(zip, 0x04034b50); Write16(zip, 20); Write16(zip, 0); Write16(zip, method); Write32(zip, 0); Write32(zip, 0); + Write32(zip, static_cast(data.size())); Write32(zip, static_cast(entry.data.size())); + Write16(zip, static_cast(entry.name.size())); Write16(zip, 0); + zip += entry.name + data; + + Write32(directory, 0x02014b50); Write16(directory, 20); Write16(directory, 20); Write16(directory, 0); Write16(directory, method); + Write32(directory, 0); Write32(directory, 0); Write32(directory, static_cast(data.size())); Write32(directory, static_cast(entry.data.size())); + Write16(directory, static_cast(entry.name.size())); Write16(directory, 0); Write16(directory, 0); Write16(directory, 0); Write16(directory, 0); + Write32(directory, 0); Write32(directory, offset); + directory += entry.name; + } + const auto directoryOffset = static_cast(zip.size()); + zip += directory; + Write32(zip, 0x06054b50); Write16(zip, 0); Write16(zip, 0); + Write16(zip, static_cast(entries.size())); Write16(zip, static_cast(entries.size())); + Write32(zip, static_cast(directory.size())); Write32(zip, directoryOffset); Write16(zip, 0); + return zip; + } + + const std::string BRICKDB = MakeZip({ + { "info.xml", R"()", false }, + { "Primitives/3003.xml", Primitive("6223;3003", BRICK_2X2_AABB), true }, + { "Primitives/3001.xml", Primitive("3001;3003", R"()"), false }, + { "Primitives/4000.xml", Primitive("4000", ""), true }, + { "Primitives/notanumber.xml", Primitive("", BRICK_2X2_AABB), false }, + { "Assemblies/5000.lxfml", "", false }, + }); + + class BrickBoundsTests : public dCommonDependenciesTest { + protected: + void SetUp() override { SetUpDependencies(); } + void TearDown() override { + TearDownDependencies(); + Game::logger = nullptr; + } + }; +} + +TEST(BrickBoundsParseTests, ReadsTheBoundingBoxNotTheGeometryBox) { + const auto bounds = BrickBounds::ParsePrimitive(Primitive("3003", BRICK_2X2_AABB)); + ASSERT_TRUE(bounds); + EXPECT_EQ(bounds->min, NiPoint3(-0.4f, 0.0f, -1.2f)); + EXPECT_EQ(bounds->max, NiPoint3(1.2f, 1.138f, 0.4f)); +} + +TEST(BrickBoundsParseTests, APrimitiveWithoutABoxHasNoBounds) { + EXPECT_FALSE(BrickBounds::ParsePrimitive(Primitive("3003", ""))); + EXPECT_FALSE(BrickBounds::ParsePrimitive(Primitive("3003", R"()"))); + EXPECT_FALSE(BrickBounds::ParsePrimitive("not xml")); +} + +TEST(BrickBoundsParseTests, ZipEntriesAreReadInOrder) { + std::vector names; + std::vector contents; + EXPECT_TRUE(Zip::ForEachEntry(BRICKDB, [](std::string_view name) { return name != "Assemblies/5000.lxfml"; }, + [&](std::string_view name, std::string_view data) { names.emplace_back(name); contents.emplace_back(data); })); + EXPECT_EQ(names, (std::vector{ "info.xml", "Primitives/3003.xml", "Primitives/3001.xml", "Primitives/4000.xml", "Primitives/notanumber.xml" })); + // Stored and deflated entries both come out as written + EXPECT_EQ(contents[0], R"()"); + EXPECT_EQ(contents[1], Primitive("6223;3003", BRICK_2X2_AABB)); +} + +TEST(BrickBoundsParseTests, NotAZip) { + EXPECT_FALSE(Zip::ForEachEntry("", [](std::string_view) { return true; }, [](std::string_view, std::string_view) {})); + EXPECT_FALSE(Zip::ForEachEntry(std::string(100, 'x'), [](std::string_view) { return true; }, [](std::string_view, std::string_view) {})); + EXPECT_TRUE(BrickBounds::ParseBrickDb("not a zip").empty()); +} + +TEST(BrickBoundsParseTests, BrickDbHasEveryPrimitiveWithABox) { + const auto bounds = BrickBounds::ParseBrickDb(BRICKDB); + ASSERT_EQ(bounds.size(), 3); + EXPECT_EQ(bounds.at(3003).min, NiPoint3(-0.4f, 0.0f, -1.2f)); + EXPECT_EQ(bounds.at(3003).max, NiPoint3(1.2f, 1.138f, 0.4f)); + EXPECT_EQ(bounds.at(3001).max, NiPoint3(2.8f, 1.138f, 0.4f)); + // 6223 is only an alias of 3003; 3003 being listed as an alias of 3001 doesn't replace its own box + EXPECT_EQ(bounds.at(6223).max, NiPoint3(1.2f, 1.138f, 0.4f)); + EXPECT_FALSE(bounds.contains(4000)); +} + +TEST_F(BrickBoundsTests, LoadsTheClientsBrickDb) { + const auto root = std::filesystem::temp_directory_path() / ("dlu_brickbounds_" + std::to_string(::testing::UnitTest::GetInstance()->random_seed())); + std::filesystem::remove_all(root); + std::filesystem::create_directories(root / "res"); + std::ofstream(root / "res" / "CDServer.sqlite").put('\0'); + { + AssetManager assets(root); + EXPECT_TRUE(BrickBounds::Load(assets).empty()); + } + std::ofstream(root / "res" / "brickdb.zip", std::ios::binary) << BRICKDB; + AssetManager assets(root); + const auto loaded = BrickBounds::Load(assets); + EXPECT_EQ(loaded.size(), 3); + + // Read once, then kept + const auto& kept = BrickBounds::Get(&assets); + EXPECT_EQ(kept.size(), 3); + std::filesystem::remove_all(root); + EXPECT_EQ(&BrickBounds::Get(&assets), &kept); + EXPECT_EQ(BrickBounds::Get(nullptr).size(), 3); +} diff --git a/tests/dCommonTests/CMakeLists.txt b/tests/dCommonTests/CMakeLists.txt index 463596f94..25cb2adf5 100644 --- a/tests/dCommonTests/CMakeLists.txt +++ b/tests/dCommonTests/CMakeLists.txt @@ -40,6 +40,7 @@ set(DCOMMONTEST_SOURCES "FdbSnapshotTests.cpp" "WorldFileWatchTests.cpp" "ChatFilterCoreTests.cpp" + "BrickBoundsTests.cpp" ) add_subdirectory(dEnumsTests)