diff --git a/dCommon/CMakeLists.txt b/dCommon/CMakeLists.txt index 0aafc0622..66636b435 100644 --- a/dCommon/CMakeLists.txt +++ b/dCommon/CMakeLists.txt @@ -17,6 +17,7 @@ set(DCOMMON_SOURCES "NiQuaternion.cpp" "Demangler.cpp" "ZCompression.cpp" + "Zip.cpp" "BrickByBrickFix.cpp" "BinaryPathFinder.cpp" "FdbToSqlite.cpp" diff --git a/dCommon/Zip.cpp b/dCommon/Zip.cpp new file mode 100644 index 000000000..81b08c2f3 --- /dev/null +++ b/dCommon/Zip.cpp @@ -0,0 +1,77 @@ +#include "Zip.h" + +#include "ZCompression.h" + +#include +#include + +namespace { + constexpr uint32_t END_SIGNATURE = 0x06054b50; + constexpr uint32_t CENTRAL_SIGNATURE = 0x02014b50; + constexpr uint32_t LOCAL_SIGNATURE = 0x04034b50; + constexpr uint64_t END_SIZE = 22; + constexpr uint64_t MAX_COMMENT_SIZE = 65535; + + // Zip numbers are little endian + template + bool ReadAt(const std::string_view data, const uint64_t offset, T& value) { + if (offset > data.size() || sizeof(T) > data.size() - offset) return false; + value = 0; + for (uint64_t i = 0; i < sizeof(T); i++) value |= static_cast(static_cast(data[offset + i])) << (8 * i); + return true; + } +} + +bool Zip::ForEachEntry(const std::string_view zip, const std::function& wanted, + const std::function& visit) { + if (zip.size() < END_SIZE) return false; + + // The end of central directory record: the last one in the file, after which only its comment can come + uint64_t end = 0; + bool found = false; + const uint64_t lowest = zip.size() > END_SIZE + MAX_COMMENT_SIZE ? zip.size() - END_SIZE - MAX_COMMENT_SIZE : 0; + for (uint64_t i = zip.size() - END_SIZE + 1; i-- > lowest;) { + uint32_t signature{}; + if (ReadAt(zip, i, signature) && signature == END_SIGNATURE) { + end = i; + found = true; + break; + } + } + if (!found) return false; + + uint16_t entries{}; + uint32_t directoryOffset{}; + if (!ReadAt(zip, end + 10, entries) || !ReadAt(zip, end + 16, directoryOffset)) return false; + + uint64_t offset = directoryOffset; + for (uint16_t i = 0; i < entries; i++) { + uint32_t signature{}, compressedSize{}, size{}, localOffset{}; + uint16_t method{}, nameLength{}, extraLength{}, commentLength{}; + if (!ReadAt(zip, offset, signature) || signature != CENTRAL_SIGNATURE || !ReadAt(zip, offset + 10, method) || + !ReadAt(zip, offset + 20, compressedSize) || !ReadAt(zip, offset + 24, size) || !ReadAt(zip, offset + 28, nameLength) || + !ReadAt(zip, offset + 30, extraLength) || !ReadAt(zip, offset + 32, commentLength) || !ReadAt(zip, offset + 42, localOffset) || + offset + 46 + nameLength > zip.size()) { + return false; + } + const auto name = zip.substr(offset + 46, nameLength); + offset += 46 + nameLength + extraLength + commentLength; + if (!wanted(name)) continue; + + // The data follows the local header, whose name and extra field can differ in length from the central one's + uint16_t localNameLength{}, localExtraLength{}; + if (!ReadAt(zip, localOffset, signature) || signature != LOCAL_SIGNATURE || !ReadAt(zip, localOffset + 26, localNameLength) || + !ReadAt(zip, localOffset + 28, localExtraLength)) { + continue; + } + const uint64_t dataStart = static_cast(localOffset) + 30 + localNameLength + localExtraLength; + if (dataStart > zip.size() || compressedSize > zip.size() - dataStart) continue; + const auto data = zip.substr(dataStart, compressedSize); + if (method == 0) { + visit(name, data); + } else if (method == 8) { + if (const auto inflated = ZCompression::InflateRaw(data, size)) visit(name, *inflated); + } + } + return true; +} diff --git a/dCommon/Zip.h b/dCommon/Zip.h new file mode 100644 index 000000000..e28d6e57d --- /dev/null +++ b/dCommon/Zip.h @@ -0,0 +1,18 @@ +// Darkflame Universe +// Copyright 2025 + +#ifndef ZIP_H +#define ZIP_H + +#include +#include + +namespace Zip { + // Calls `visit` with the name and contents of every entry of the zip archive `zip` whose name `wanted` accepts, in + // the archive's order. Stored and deflated entries are read; others, and entries that fail to read, are skipped. + // Returns false when the archive's directory can't be read. + bool ForEachEntry(std::string_view zip, const std::function& wanted, + const std::function& visit); +}; + +#endif //!ZIP_H diff --git a/dCommon/dClient/BrickBounds.cpp b/dCommon/dClient/BrickBounds.cpp new file mode 100644 index 000000000..10d9d83cb --- /dev/null +++ b/dCommon/dClient/BrickBounds.cpp @@ -0,0 +1,99 @@ +#include "BrickBounds.h" + +#include "AssetManager.h" +#include "GeneralUtils.h" +#include "Logger.h" +#include "Zip.h" + +#include "tinyxml2.h" + +#include +#include +#include + +namespace { + constexpr std::string_view PRIMITIVES = "primitives/"; + constexpr std::string_view XML = ".xml"; + + // The design ID of a Primitives/.xml entry + std::optional PrimitiveDesign(const std::string_view name) { + if (name.size() <= PRIMITIVES.size() + XML.size()) return std::nullopt; + std::string lower(name); + std::transform(lower.begin(), lower.end(), lower.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); + if (!lower.starts_with(PRIMITIVES) || !lower.ends_with(XML)) return std::nullopt; + const auto id = std::string_view(lower).substr(PRIMITIVES.size(), lower.size() - PRIMITIVES.size() - XML.size()); + if (id.empty() || !std::all_of(id.begin(), id.end(), [](unsigned char c) { return std::isdigit(c); })) return std::nullopt; + return GeneralUtils::TryParse(id); + } +} + +namespace { + std::optional ReadBounds(const tinyxml2::XMLElement* primitive) { + const auto* bounding = primitive ? primitive->FirstChildElement("Bounding") : nullptr; + const auto* aabb = bounding ? bounding->FirstChildElement("AABB") : nullptr; + if (!aabb) return std::nullopt; + + BrickBounds::Bounds bounds; + if (aabb->QueryFloatAttribute("minX", &bounds.min.x) != tinyxml2::XML_SUCCESS || + aabb->QueryFloatAttribute("minY", &bounds.min.y) != tinyxml2::XML_SUCCESS || + aabb->QueryFloatAttribute("minZ", &bounds.min.z) != tinyxml2::XML_SUCCESS || + aabb->QueryFloatAttribute("maxX", &bounds.max.x) != tinyxml2::XML_SUCCESS || + aabb->QueryFloatAttribute("maxY", &bounds.max.y) != tinyxml2::XML_SUCCESS || + aabb->QueryFloatAttribute("maxZ", &bounds.max.z) != tinyxml2::XML_SUCCESS) { + return std::nullopt; + } + return bounds; + } +} + +std::optional BrickBounds::ParsePrimitive(const std::string_view xml) { + tinyxml2::XMLDocument doc; + if (doc.Parse(xml.data(), xml.size()) != tinyxml2::XML_SUCCESS) return std::nullopt; + return ReadBounds(doc.FirstChildElement("LEGOPrimitive")); +} + +BrickBounds::DesignBounds BrickBounds::ParseBrickDb(const std::string_view zip) { + DesignBounds bounds; + DesignBounds aliases; + Zip::ForEachEntry(zip, [](std::string_view name) { return PrimitiveDesign(name).has_value(); }, + [&bounds, &aliases](std::string_view name, std::string_view data) { + const auto design = PrimitiveDesign(name); + tinyxml2::XMLDocument doc; + if (!design || doc.Parse(data.data(), data.size()) != tinyxml2::XML_SUCCESS) return; + const auto* primitive = doc.FirstChildElement("LEGOPrimitive"); + const auto box = ReadBounds(primitive); + if (!box) return; + bounds[*design] = *box; + + // : other design IDs of the same brick + const auto* annotations = primitive->FirstChildElement("Annotations"); + for (const auto* annotation = annotations ? annotations->FirstChildElement("Annotation") : nullptr; annotation; annotation = annotation->NextSiblingElement("Annotation")) { + const char* list = annotation->Attribute("aliases"); + if (!list) continue; + for (const auto& alias : GeneralUtils::SplitString(list, ';')) { + if (const auto id = GeneralUtils::TryParse(alias)) aliases.emplace(*id, *box); + } + } + }); + // A primitive of its own wins over being another's alias + for (const auto& [design, box] : aliases) bounds.emplace(design, box); + return bounds; +} + +BrickBounds::DesignBounds BrickBounds::Load(const AssetManager& assets) { + char* data = nullptr; + uint32_t length = 0; + if (!assets.GetFile("brickdb.zip", &data, &length)) { + LOG("Failed to read brickdb.zip; models are placed by their bricks' origins"); + return {}; + } + auto bounds = ParseBrickDb(std::string_view(data, length)); + free(data); + LOG("Read the bounds of %u brick designs", static_cast(bounds.size())); + return bounds; +} + +const BrickBounds::DesignBounds& BrickBounds::Get(const AssetManager* assets) { + static const DesignBounds bounds = assets ? Load(*assets) : DesignBounds{}; + return bounds; +} diff --git a/dCommon/dClient/BrickBounds.h b/dCommon/dClient/BrickBounds.h new file mode 100644 index 000000000..a723c8ae8 --- /dev/null +++ b/dCommon/dClient/BrickBounds.h @@ -0,0 +1,42 @@ +// Darkflame Universe +// Copyright 2025 + +#ifndef BRICKBOUNDS_H +#define BRICKBOUNDS_H + +#include +#include +#include +#include + +#include "NiPoint3.h" + +class AssetManager; + +// The bounding boxes of the client's bricks, from its brickdb (res/brickdb.zip) +namespace BrickBounds { + // A brick's box around its own origin + struct Bounds { + NiPoint3 min; + NiPoint3 max; + }; + + // Design ID -> its box + using DesignBounds = std::unordered_map; + + // The of one primitive (Primitives/.xml) + [[nodiscard]] std::optional ParsePrimitive(std::string_view xml); + + // The box of every primitive in a brickdb.zip. A design listed only as another primitive's alias gets that + // primitive's box. + [[nodiscard]] DesignBounds ParseBrickDb(std::string_view zip); + + // Reads brickdb.zip through `assets`; empty if it can't be read + [[nodiscard]] DesignBounds Load(const AssetManager& assets); + + // The boxes read by Load the first time this is called, kept for every later call (`assets` is only used then). + // Empty without `assets`. + [[nodiscard]] const DesignBounds& Get(const AssetManager* assets); +}; + +#endif //!BRICKBOUNDS_H diff --git a/dCommon/dClient/CMakeLists.txt b/dCommon/dClient/CMakeLists.txt index 017a22a52..83dc640c0 100644 --- a/dCommon/dClient/CMakeLists.txt +++ b/dCommon/dClient/CMakeLists.txt @@ -1,5 +1,6 @@ set(DCOMMON_DCLIENT_SOURCES "AssetManager.cpp" + "BrickBounds.cpp" "PackIndex.cpp" "Pack.cpp" PARENT_SCOPE diff --git a/tests/dCommonTests/BrickBoundsTests.cpp b/tests/dCommonTests/BrickBoundsTests.cpp new file mode 100644 index 000000000..a3e2515cf --- /dev/null +++ b/tests/dCommonTests/BrickBoundsTests.cpp @@ -0,0 +1,154 @@ +#include "gtest/gtest.h" + +#include "AssetManager.h" +#include "BrickBounds.h" +#include "ZCompression.h" +#include "Zip.h" +#include "dCommonDependencies.h" + +#include +#include +#include +#include + +namespace { + std::string Primitive(const std::string& aliases, const std::string& aabb) { + return R"()" + R"()" + R"()" + aabb + R"()" + R"()"; + } + + const std::string BRICK_2X2_AABB = R"()"; + + struct Entry { + std::string name; + std::string data; + bool deflate; + }; + + void Write16(std::string& out, uint16_t value) { + out += static_cast(value & 0xFF); + out += static_cast(value >> 8); + } + + void Write32(std::string& out, uint32_t value) { + Write16(out, static_cast(value & 0xFFFF)); + Write16(out, static_cast(value >> 16)); + } + + // Raw deflate data: zlib's stream without its 2 byte header and 4 byte checksum + std::string Deflate(const std::string& data) { + std::string out(ZCompression::GetMaxCompressedLength(static_cast(data.size())), '\0'); + const auto size = ZCompression::Compress(reinterpret_cast(data.data()), static_cast(data.size()), reinterpret_cast(out.data()), static_cast(out.size())); + return out.substr(2, size - 6); + } + + // A zip archive (crc left at 0: it isn't checked) + std::string MakeZip(const std::vector& entries) { + std::string zip; + std::string directory; + for (const auto& entry : entries) { + const auto data = entry.deflate ? Deflate(entry.data) : entry.data; + const auto method = static_cast(entry.deflate ? 8 : 0); + const auto offset = static_cast(zip.size()); + Write32(zip, 0x04034b50); Write16(zip, 20); Write16(zip, 0); Write16(zip, method); Write32(zip, 0); Write32(zip, 0); + Write32(zip, static_cast(data.size())); Write32(zip, static_cast(entry.data.size())); + Write16(zip, static_cast(entry.name.size())); Write16(zip, 0); + zip += entry.name + data; + + Write32(directory, 0x02014b50); Write16(directory, 20); Write16(directory, 20); Write16(directory, 0); Write16(directory, method); + Write32(directory, 0); Write32(directory, 0); Write32(directory, static_cast(data.size())); Write32(directory, static_cast(entry.data.size())); + Write16(directory, static_cast(entry.name.size())); Write16(directory, 0); Write16(directory, 0); Write16(directory, 0); Write16(directory, 0); + Write32(directory, 0); Write32(directory, offset); + directory += entry.name; + } + const auto directoryOffset = static_cast(zip.size()); + zip += directory; + Write32(zip, 0x06054b50); Write16(zip, 0); Write16(zip, 0); + Write16(zip, static_cast(entries.size())); Write16(zip, static_cast(entries.size())); + Write32(zip, static_cast(directory.size())); Write32(zip, directoryOffset); Write16(zip, 0); + return zip; + } + + const std::string BRICKDB = MakeZip({ + { "info.xml", R"()", false }, + { "Primitives/3003.xml", Primitive("6223;3003", BRICK_2X2_AABB), true }, + { "Primitives/3001.xml", Primitive("3001;3003", R"()"), false }, + { "Primitives/4000.xml", Primitive("4000", ""), true }, + { "Primitives/notanumber.xml", Primitive("", BRICK_2X2_AABB), false }, + { "Assemblies/5000.lxfml", "", false }, + }); + + class BrickBoundsTests : public dCommonDependenciesTest { + protected: + void SetUp() override { SetUpDependencies(); } + void TearDown() override { + TearDownDependencies(); + Game::logger = nullptr; + } + }; +} + +TEST(BrickBoundsParseTests, ReadsTheBoundingBoxNotTheGeometryBox) { + const auto bounds = BrickBounds::ParsePrimitive(Primitive("3003", BRICK_2X2_AABB)); + ASSERT_TRUE(bounds); + EXPECT_EQ(bounds->min, NiPoint3(-0.4f, 0.0f, -1.2f)); + EXPECT_EQ(bounds->max, NiPoint3(1.2f, 1.138f, 0.4f)); +} + +TEST(BrickBoundsParseTests, APrimitiveWithoutABoxHasNoBounds) { + EXPECT_FALSE(BrickBounds::ParsePrimitive(Primitive("3003", ""))); + EXPECT_FALSE(BrickBounds::ParsePrimitive(Primitive("3003", R"()"))); + EXPECT_FALSE(BrickBounds::ParsePrimitive("not xml")); +} + +TEST(BrickBoundsParseTests, ZipEntriesAreReadInOrder) { + std::vector names; + std::vector contents; + EXPECT_TRUE(Zip::ForEachEntry(BRICKDB, [](std::string_view name) { return name != "Assemblies/5000.lxfml"; }, + [&](std::string_view name, std::string_view data) { names.emplace_back(name); contents.emplace_back(data); })); + EXPECT_EQ(names, (std::vector{ "info.xml", "Primitives/3003.xml", "Primitives/3001.xml", "Primitives/4000.xml", "Primitives/notanumber.xml" })); + // Stored and deflated entries both come out as written + EXPECT_EQ(contents[0], R"()"); + EXPECT_EQ(contents[1], Primitive("6223;3003", BRICK_2X2_AABB)); +} + +TEST(BrickBoundsParseTests, NotAZip) { + EXPECT_FALSE(Zip::ForEachEntry("", [](std::string_view) { return true; }, [](std::string_view, std::string_view) {})); + EXPECT_FALSE(Zip::ForEachEntry(std::string(100, 'x'), [](std::string_view) { return true; }, [](std::string_view, std::string_view) {})); + EXPECT_TRUE(BrickBounds::ParseBrickDb("not a zip").empty()); +} + +TEST(BrickBoundsParseTests, BrickDbHasEveryPrimitiveWithABox) { + const auto bounds = BrickBounds::ParseBrickDb(BRICKDB); + ASSERT_EQ(bounds.size(), 3); + EXPECT_EQ(bounds.at(3003).min, NiPoint3(-0.4f, 0.0f, -1.2f)); + EXPECT_EQ(bounds.at(3003).max, NiPoint3(1.2f, 1.138f, 0.4f)); + EXPECT_EQ(bounds.at(3001).max, NiPoint3(2.8f, 1.138f, 0.4f)); + // 6223 is only an alias of 3003; 3003 being listed as an alias of 3001 doesn't replace its own box + EXPECT_EQ(bounds.at(6223).max, NiPoint3(1.2f, 1.138f, 0.4f)); + EXPECT_FALSE(bounds.contains(4000)); +} + +TEST_F(BrickBoundsTests, LoadsTheClientsBrickDb) { + const auto root = std::filesystem::temp_directory_path() / ("dlu_brickbounds_" + std::to_string(::testing::UnitTest::GetInstance()->random_seed())); + std::filesystem::remove_all(root); + std::filesystem::create_directories(root / "res"); + std::ofstream(root / "res" / "CDServer.sqlite").put('\0'); + { + AssetManager assets(root); + EXPECT_TRUE(BrickBounds::Load(assets).empty()); + } + std::ofstream(root / "res" / "brickdb.zip", std::ios::binary) << BRICKDB; + AssetManager assets(root); + const auto loaded = BrickBounds::Load(assets); + EXPECT_EQ(loaded.size(), 3); + + // Read once, then kept + const auto& kept = BrickBounds::Get(&assets); + EXPECT_EQ(kept.size(), 3); + std::filesystem::remove_all(root); + EXPECT_EQ(&BrickBounds::Get(&assets), &kept); + EXPECT_EQ(BrickBounds::Get(nullptr).size(), 3); +} diff --git a/tests/dCommonTests/CMakeLists.txt b/tests/dCommonTests/CMakeLists.txt index 463596f94..25cb2adf5 100644 --- a/tests/dCommonTests/CMakeLists.txt +++ b/tests/dCommonTests/CMakeLists.txt @@ -40,6 +40,7 @@ set(DCOMMONTEST_SOURCES "FdbSnapshotTests.cpp" "WorldFileWatchTests.cpp" "ChatFilterCoreTests.cpp" + "BrickBoundsTests.cpp" ) add_subdirectory(dEnumsTests)