From ceec638e52583801bb148cd61cb12359d0b571d1 Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Sun, 27 Sep 2026 17:46:34 -0500 Subject: [PATCH] feat(common): Sd0::Compress, a default Sd0 and room for chunks that don't shrink Sd0 gets a default constructor and Sd0::Compress(data), the raw sd0 bytes of some data (what the client reads for UGC files it downloads without 3D services). FromData compresses into a heap buffer big enough for a chunk that grows when deflated (it used a 256 KiB stack buffer, so random data failed and workers carried a large stack frame), and drops a half-written result. Tests: chunks as the 1.10.64 client inflates them, incompressible data, empty. Co-Authored-By: Claude Opus 5.5 --- dCommon/Sd0.cpp | 14 ++++++- dCommon/Sd0.h | 9 +++++ tests/dCommonTests/CMakeLists.txt | 1 + tests/dCommonTests/Sd0Tests.cpp | 67 +++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 tests/dCommonTests/Sd0Tests.cpp diff --git a/dCommon/Sd0.cpp b/dCommon/Sd0.cpp index e06f242eb..8fbc4232f 100644 --- a/dCommon/Sd0.cpp +++ b/dCommon/Sd0.cpp @@ -98,17 +98,20 @@ void Sd0::FromData(const uint8_t* data, size_t bufferSize) { if (bufferSize == 0) return; m_Chunks.clear(); + // Room for a chunk that doesn't compress (zlib makes it slightly bigger), on the heap: this runs on worker threads + std::vector compressedChunk(ZCompression::GetMaxCompressedLength(MAX_UNCOMPRESSED_CHUNK_SIZE)); while (bufferSize > 0) { const auto numToCopy = std::min(MAX_UNCOMPRESSED_CHUNK_SIZE, bufferSize); const auto* startOffset = data + originalBufferSize - bufferSize; bufferSize -= numToCopy; - std::array compressedChunk; const auto compressedSize = ZCompression::Compress( startOffset, numToCopy, compressedChunk.data(), compressedChunk.size()); if (compressedSize == -1) { LOG("Failed to compress chunk, aborting"); + // Part of the data isn't a usable buffer + m_Chunks.clear(); break; } @@ -128,6 +131,15 @@ void Sd0::FromData(const uint8_t* data, size_t bufferSize) { } +std::string Sd0::Compress(std::string_view data) { + Sd0 sd0; + if (data.empty()) return std::string(SD0_HEADER, 5); + sd0.FromData(reinterpret_cast(data.data()), data.size()); + std::string out; + for (const auto& chunk : sd0.GetAsVector()) out.append(reinterpret_cast(chunk.data()), chunk.size()); + return out; +} + std::string Sd0::GetAsStringUncompressed() const { std::string toReturn; bool first = true; diff --git a/dCommon/Sd0.h b/dCommon/Sd0.h index 40bf99308..b4dae02ff 100644 --- a/dCommon/Sd0.h +++ b/dCommon/Sd0.h @@ -5,6 +5,8 @@ #define SD0_H #include +#include +#include #include // Sd0 is comprised of multiple zlib compressed buffers stored in a row. @@ -21,9 +23,16 @@ public: */ static constexpr inline size_t MAX_UNCOMPRESSED_CHUNK_SIZE = 1024 * 256; + // Empty, for FromData + Sd0() = default; + // Read the input buffer into an internal chunk stream to be used later Sd0(std::istream& buffer); + // `data` compressed into an sd0 buffer, returned in its raw compressed form (what the client reads, e.g. the UGC + // files it downloads without 3D services); only the header when `data` is empty, empty when it can't be compressed + [[nodiscard]] static std::string Compress(std::string_view data); + // Uncompresses the entire Sd0 buffer and returns it as a string [[nodiscard]] std::string GetAsStringUncompressed() const; diff --git a/tests/dCommonTests/CMakeLists.txt b/tests/dCommonTests/CMakeLists.txt index 1e18043e1..6c7377aaa 100644 --- a/tests/dCommonTests/CMakeLists.txt +++ b/tests/dCommonTests/CMakeLists.txt @@ -29,6 +29,7 @@ set(DCOMMONTEST_SOURCES "PropertyReputationRulesTests.cpp" "BindAddressTests.cpp" "TrafficStatsTests.cpp" + "Sd0Tests.cpp" ) add_subdirectory(dEnumsTests) diff --git a/tests/dCommonTests/Sd0Tests.cpp b/tests/dCommonTests/Sd0Tests.cpp new file mode 100644 index 000000000..1e0b53e55 --- /dev/null +++ b/tests/dCommonTests/Sd0Tests.cpp @@ -0,0 +1,67 @@ +#include + +#include +#include +#include +#include + +#include "Sd0.h" +#include "ZCompression.h" + +namespace { + // The chunks as the 1.10.64 client reads them (Sd0Decompress): the header, then a u32 size and zlib data per chunk, + // each inflating to at most 256 KiB + std::string InflateLikeTheClient(const std::string& sd0, size_t& chunks) { + EXPECT_TRUE(sd0.starts_with(std::string(Sd0::SD0_HEADER, 5))); + size_t offset = 5; + std::string inflated; + chunks = 0; + while (offset < sd0.size()) { + if (offset + 4 > sd0.size()) return {}; + uint32_t length{}; + std::memcpy(&length, sd0.data() + offset, 4); + offset += 4; + if (offset + length > sd0.size()) return {}; + std::string chunk(Sd0::MAX_UNCOMPRESSED_CHUNK_SIZE, '\0'); + int32_t error{}; + const auto size = ZCompression::Decompress(reinterpret_cast(sd0.data() + offset), length, + reinterpret_cast(chunk.data()), static_cast(chunk.size()), error); + if (size < 0) return {}; + inflated.append(chunk.data(), static_cast(size)); + offset += length; + chunks++; + } + return inflated; + } +} + +TEST(Sd0, CompressesInChunksTheClientReads) { + // 600 KiB: chunks of 256, 256 and 88 KiB + std::string data(600 * 1024, '\0'); + for (size_t i = 0; i < data.size(); i++) data[i] = static_cast((i * 7) % 251); + const auto sd0 = Sd0::Compress(data); + size_t chunks{}; + EXPECT_EQ(InflateLikeTheClient(sd0, chunks), data); + EXPECT_EQ(chunks, 3u); + + // And back through the reader + std::istringstream stream(sd0); + EXPECT_EQ(Sd0(stream).GetAsStringUncompressed(), data); +} + +TEST(Sd0, CompressesDataThatDoesNotShrink) { + // Random bytes grow a little when deflated: a full chunk must still fit + std::string data(Sd0::MAX_UNCOMPRESSED_CHUNK_SIZE + 1000, '\0'); + std::mt19937 random(1234); + for (auto& c : data) c = static_cast(random()); + const auto sd0 = Sd0::Compress(data); + size_t chunks{}; + EXPECT_EQ(InflateLikeTheClient(sd0, chunks), data); + EXPECT_EQ(chunks, 2u); +} + +TEST(Sd0, EmptyDataIsTheHeaderOnly) { + EXPECT_EQ(Sd0::Compress(""), std::string(Sd0::SD0_HEADER, 5)); + Sd0 empty; + EXPECT_TRUE(empty.GetAsVector().empty()); +}