feat(common): Sd0::Compress, a default Sd0 and room for chunks that don't shrink

Sd0 gets a default constructor and Sd0::Compress(data), the raw sd0 bytes of
some data (what the client reads for UGC files it downloads without 3D
services). FromData compresses into a heap buffer big enough for a chunk that
grows when deflated (it used a 256 KiB stack buffer, so random data failed and
workers carried a large stack frame), and drops a half-written result.

Tests: chunks as the 1.10.64 client inflates them, incompressible data, empty.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dCommon/Sd0.cpp

@@ -98,17 +98,20 @@ void Sd0::FromData(const uint8_t* data, size_t bufferSize) {
 	if (bufferSize == 0) return;
 
 	m_Chunks.clear();
+	// Room for a chunk that doesn't compress (zlib makes it slightly bigger), on the heap: this runs on worker threads
+	std::vector<uint8_t> compressedChunk(ZCompression::GetMaxCompressedLength(MAX_UNCOMPRESSED_CHUNK_SIZE));
 	while (bufferSize > 0) {
 		const auto numToCopy = std::min(MAX_UNCOMPRESSED_CHUNK_SIZE, bufferSize);
 		const auto* startOffset = data + originalBufferSize - bufferSize;
 		bufferSize -= numToCopy;
-		std::array<uint8_t, MAX_UNCOMPRESSED_CHUNK_SIZE> compressedChunk;
 		const auto compressedSize = ZCompression::Compress(
 			startOffset, numToCopy,
 			compressedChunk.data(), compressedChunk.size());
 
 		if (compressedSize == -1) {
 			LOG("Failed to compress chunk, aborting");
+			// Part of the data isn't a usable buffer
+			m_Chunks.clear();
 			break;
 		}
 
@@ -128,6 +131,15 @@ void Sd0::FromData(const uint8_t* data, size_t bufferSize) {
 
 }
 
+std::string Sd0::Compress(std::string_view data) {
+	Sd0 sd0;
+	if (data.empty()) return std::string(SD0_HEADER, 5);
+	sd0.FromData(reinterpret_cast<const uint8_t*>(data.data()), data.size());
+	std::string out;
+	for (const auto& chunk : sd0.GetAsVector()) out.append(reinterpret_cast<const char*>(chunk.data()), chunk.size());
+	return out;
+}
+
 std::string Sd0::GetAsStringUncompressed() const {
 	std::string toReturn;
 	bool first = true;

dCommon/Sd0.h

@@ -5,6 +5,8 @@
 #define SD0_H
 
 #include <fstream>
+#include <string>
+#include <string_view>
 #include <vector>
 
 // Sd0 is comprised of multiple zlib compressed buffers stored in a row.
@@ -21,9 +23,16 @@ public:
 	 */
 	static constexpr inline size_t MAX_UNCOMPRESSED_CHUNK_SIZE = 1024 * 256;
 
+	// Empty, for FromData
+	Sd0() = default;
+
 	// Read the input buffer into an internal chunk stream to be used later
 	Sd0(std::istream& buffer);
 
+	// `data` compressed into an sd0 buffer, returned in its raw compressed form (what the client reads, e.g. the UGC
+	// files it downloads without 3D services); only the header when `data` is empty, empty when it can't be compressed
+	[[nodiscard]] static std::string Compress(std::string_view data);
+
 	// Uncompresses the entire Sd0 buffer and returns it as a string
 	[[nodiscard]] std::string GetAsStringUncompressed() const;
 

tests/dCommonTests/CMakeLists.txt

@@ -29,6 +29,7 @@ set(DCOMMONTEST_SOURCES
 	"PropertyReputationRulesTests.cpp"
 	"BindAddressTests.cpp"
 	"TrafficStatsTests.cpp"
+	"Sd0Tests.cpp"
 )
 
 add_subdirectory(dEnumsTests)

tests/dCommonTests/Sd0Tests.cpp

@@ -0,0 +1,67 @@
+#include <gtest/gtest.h>
+
+#include <cstring>
+#include <random>
+#include <sstream>
+#include <string>
+
+#include "Sd0.h"
+#include "ZCompression.h"
+
+namespace {
+	// The chunks as the 1.10.64 client reads them (Sd0Decompress): the header, then a u32 size and zlib data per chunk,
+	// each inflating to at most 256 KiB
+	std::string InflateLikeTheClient(const std::string& sd0, size_t& chunks) {
+		EXPECT_TRUE(sd0.starts_with(std::string(Sd0::SD0_HEADER, 5)));
+		size_t offset = 5;
+		std::string inflated;
+		chunks = 0;
+		while (offset < sd0.size()) {
+			if (offset + 4 > sd0.size()) return {};
+			uint32_t length{};
+			std::memcpy(&length, sd0.data() + offset, 4);
+			offset += 4;
+			if (offset + length > sd0.size()) return {};
+			std::string chunk(Sd0::MAX_UNCOMPRESSED_CHUNK_SIZE, '\0');
+			int32_t error{};
+			const auto size = ZCompression::Decompress(reinterpret_cast<const uint8_t*>(sd0.data() + offset), length,
+				reinterpret_cast<uint8_t*>(chunk.data()), static_cast<uint32_t>(chunk.size()), error);
+			if (size < 0) return {};
+			inflated.append(chunk.data(), static_cast<size_t>(size));
+			offset += length;
+			chunks++;
+		}
+		return inflated;
+	}
+}
+
+TEST(Sd0, CompressesInChunksTheClientReads) {
+	// 600 KiB: chunks of 256, 256 and 88 KiB
+	std::string data(600 * 1024, '\0');
+	for (size_t i = 0; i < data.size(); i++) data[i] = static_cast<char>((i * 7) % 251);
+	const auto sd0 = Sd0::Compress(data);
+	size_t chunks{};
+	EXPECT_EQ(InflateLikeTheClient(sd0, chunks), data);
+	EXPECT_EQ(chunks, 3u);
+
+	// And back through the reader
+	std::istringstream stream(sd0);
+	EXPECT_EQ(Sd0(stream).GetAsStringUncompressed(), data);
+}
+
+TEST(Sd0, CompressesDataThatDoesNotShrink) {
+	// Random bytes grow a little when deflated: a full chunk must still fit
+	std::string data(Sd0::MAX_UNCOMPRESSED_CHUNK_SIZE + 1000, '\0');
+	std::mt19937 random(1234);
+	for (auto& c : data) c = static_cast<char>(random());
+	const auto sd0 = Sd0::Compress(data);
+	size_t chunks{};
+	EXPECT_EQ(InflateLikeTheClient(sd0, chunks), data);
+	EXPECT_EQ(chunks, 2u);
+}
+
+TEST(Sd0, EmptyDataIsTheHeaderOnly) {
+	EXPECT_EQ(Sd0::Compress(""), std::string(Sd0::SD0_HEADER, 5));
+	Sd0 empty;
+	EXPECT_TRUE(empty.GetAsVector().empty());
+}