feat(ugc): answer the client's UGC manifest requests without 3D services

With UGCUSE3DSERVICES=7:0 (the client's default) the client asks its world for
a blueprint file's MD5 and size (REQUEST_UGC_MANIFEST_INFO, world 27) and then
downloads BrickModels/UserMade/<id % 1000>/<id>.<ext>.sd0. Layouts checked in
the 1.10.64 client: the request is a u64 blueprint id and a u8 resource type;
the answer (UGC_MANIFEST_RESPONSE, client 60) repeats them and adds a u8 valid,
the u32 size and the 16 byte MD5 of the inflated file, 37 bytes after the 0x53
exactly or the client drops it.

- dNet: WorldPackets::RequestUgcManifestInfo, ClientPackets::UgcManifestResponse
  (eUgcResourceType), with byte tests against the client's layouts.
- Database: ugc_file_checksums (per model or module combination and file) and
  ugc_modular_build.combination_id (migrations 89 / 72), GetUgcFileChecksum
  looks a blueprint up as a model, else as a build through its combination.
- UGC server: every download is also written as .sd0 (Sd0::Compress); workers
  hand the checksums back and the main thread stores them; old items get their
  sd0 icon and checksum, and builds their combination id, once at start-up, a
  few per tick; serves <dir>/BrickModels/UserMade/<bucket>/<id>.<ext>.sd0 under
  client_path, /<folder>/UserBrickModels and the root (.hkx 404).
- World: UgcManifest answers on the main thread with one indexed query per
  request; files not made yet are answered when they are (looked at again
  every 5 seconds), and a model in its quiet period is made right away. No
  worker threads, HTTP or file reads in the world.

Off by default (ugc_manifest=0): checked in game, the client then downloads
from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says
and logs the player out when it can't connect, so icons need the UGC server on
port 80 of each player's machine. docs/UgcServer.md has the details.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dUgcServer/UgcFormats.cpp

@@ -372,4 +372,26 @@ namespace UgcFormats {
 	std::string ChecksumXml(std::string_view data) {
 		return "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Checksum><MD5>" + Md5Hex(data) + "</MD5><Filesize>" + std::to_string(data.size()) + "</Filesize></Checksum>\n";
 	}
+
+	bool ReadChecksumXml(std::string_view xml, std::string& md5, uint32_t& size) {
+		const auto between = [xml](std::string_view open, std::string_view close) -> std::string_view {
+			const auto start = xml.find(open);
+			if (start == std::string_view::npos) return {};
+			const auto end = xml.find(close, start + open.size());
+			if (end == std::string_view::npos) return {};
+			return xml.substr(start + open.size(), end - start - open.size());
+		};
+		const auto hash = between("<MD5>", "</MD5>");
+		const auto length = between("<Filesize>", "</Filesize>");
+		if (hash.size() != 32 || length.empty()) return false;
+		uint64_t parsed = 0;
+		for (const char c : length) {
+			if (c < '0' || c > '9') return false;
+			parsed = parsed * 10 + static_cast<uint64_t>(c - '0');
+			if (parsed > std::numeric_limits<uint32_t>::max()) return false;
+		}
+		md5 = hash;
+		size = static_cast<uint32_t>(parsed);
+		return true;
+	}
 }

dUgcServer/UgcFormats.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <cstdint>
 #include <string>
 #include <string_view>
 #include <vector>
@@ -57,4 +58,7 @@ namespace UgcFormats {
 
 	// What the client reads from a UGC file's .checksum: the MD5 and size of the file as it is after inflating it
 	std::string ChecksumXml(std::string_view data);
+
+	// The MD5 (lowercase hex) and size a .checksum (ChecksumXml) holds; false when it doesn't hold both
+	bool ReadChecksumXml(std::string_view xml, std::string& md5, uint32_t& size);
 }

dUgcServer/UgcJobs.cpp

@@ -17,6 +17,9 @@ namespace UgcJobs {
 	void AddDownload(UgcStorage::Files& files, const std::string& name, const std::string& data) {
 		files[name + ".gz"] = ZCompression::Gzip(data);
 		files[name + ".checksum"] = UgcFormats::ChecksumXml(data);
+		// Without 3D services the client downloads it as sd0 (after asking its world for the checksum)
+		auto sd0 = Sd0::Compress(data);
+		if (!sd0.empty()) files[name + ".sd0"] = std::move(sd0);
 	}
 
 	std::string LxfmlFromBlob(const std::string& blob) {

dUgcServer/UgcProcessor.cpp

@@ -5,7 +5,10 @@
 #include "Logger.h"
 #include "UgcBricks.h"
 #include "UgcCdClient.h"
+#include "Sd0.h"
+#include "UgcFormats.h"
 #include "UgcKeys.h"
+#include "ZCompression.h"
 #include "UgcThrottle.h"
 #include "json.hpp"
 
@@ -26,6 +29,12 @@ namespace {
 	constexpr size_t LOG_LENGTH = 50;
 	constexpr auto EVICTION_INTERVAL = std::chrono::minutes(5);
 	constexpr auto RECENT_ANSWER_TIME = std::chrono::seconds(10);
+	// In the storage folder once every item stored has its sd0 icon and checksums (Backfill)
+	constexpr auto BACKFILL_MARKER = ".checksums-stored";
+	constexpr size_t BACKFILL_ITEMS_PER_UPDATE = 16;
+	constexpr uint32_t BACKFILL_BUILDS_PER_UPDATE = 200;
+	// Combination id of a build whose modules can't be told (so it isn't looked at again)
+	constexpr LWOOBJID NO_COMBINATION = -1;
 
 	const char* KindName(UgcStorage::Kind kind) {
 		return kind == UgcStorage::Kind::MODEL ? "model" : "modular";
@@ -95,7 +104,12 @@ UgcProcessor::~UgcProcessor() {
 }
 
 void UgcProcessor::Start() {
-	for (const auto& entry : m_Storage.List()) m_StoredBytes += entry.bytes;
+	const auto stored = m_Storage.List();
+	for (const auto& entry : stored) m_StoredBytes += entry.bytes;
+	// Items made before the checksums were stored get their sd0 icon and checksums once (Backfill)
+	std::error_code error;
+	m_BackfillItemsDone = std::filesystem::exists(m_Storage.GetRoot() / BACKFILL_MARKER, error);
+	if (!m_BackfillItemsDone) m_BackfillItems.assign(stored.begin(), stored.end());
 	m_NextEviction = std::chrono::steady_clock::now();
 	m_Stopping = false;
 	for (size_t i = 0; i < std::max<size_t>(m_Config.threads, 1); i++) m_Threads.emplace_back(&UgcProcessor::Worker, this);
@@ -233,6 +247,14 @@ void UgcProcessor::Worker() {
 			const auto bytes = job.iconOnly ? m_Storage.Update(job.kind, job.id, done.outcome.files, error) : m_Storage.Write(job.kind, job.id, done.outcome.files, error);
 			if (bytes) {
 				done.bytes = *bytes;
+				// The checksums of what the client downloads as sd0, for the main thread to store (the worlds answer
+				// the clients' manifest requests with them)
+				for (const std::string name : { "icon.dds", "model.nif" }) {
+					const auto checksum = done.outcome.files.find(name + ".checksum");
+					if (checksum == done.outcome.files.end() || !done.outcome.files.contains(name + ".sd0")) continue;
+					Checksum parsed{ name };
+					if (UgcFormats::ReadChecksumXml(checksum->second, parsed.md5, parsed.size)) done.checksums.push_back(std::move(parsed));
+				}
 			} else {
 				done.outcome.ok = false;
 				done.outcome.error = error;
@@ -355,6 +377,8 @@ void UgcProcessor::Record(const Done& done) {
 		}
 	} else {
 		Database::Get()->SetModularBuildProcessed(done.id, state, attempts, error);
+		// Which combination's files it shares, for the worlds' manifest answers
+		if (const auto combo = m_ComboOf.find(done.id); combo != m_ComboOf.end()) Database::Get()->SetModularBuildCombination(done.id, combo->second);
 	}
 	m_Recent.erase({ done.kind, done.id });
 
@@ -381,6 +405,8 @@ void UgcProcessor::Collect() {
 		finished.swap(m_Done);
 	}
 	for (const auto& done : finished) {
+		// done.id is the model, or the combination
+		if (done.outcome.ok) StoreChecksums(done.kind, done.id, done.checksums);
 		if (done.kind == Kind::MODEL) {
 			m_InFlight.erase({ done.kind, done.id });
 			if (!done.iconOnly) {
@@ -413,8 +439,58 @@ void UgcProcessor::Collect() {
 	if (!finished.empty()) m_NextPoll = std::min(m_NextPoll, std::chrono::steady_clock::now() + std::chrono::milliseconds(100));
 }
 
+void UgcProcessor::StoreChecksums(Kind kind, LWOOBJID storageId, const std::vector<Checksum>& checksums) {
+	const auto owner = kind == Kind::MODEL ? IUgc::eFileOwner::MODEL : IUgc::eFileOwner::COMBINATION;
+	for (const auto& checksum : checksums) Database::Get()->SetUgcFileChecksum(owner, storageId, checksum.file, checksum.md5, checksum.size);
+}
+
+void UgcProcessor::Backfill() {
+	if (!m_BackfillBuildsDone) {
+		const auto builds = Database::Get()->GetModularBuildsWithoutCombination(BACKFILL_BUILDS_PER_UPDATE);
+		for (const auto& build : builds) {
+			const auto key = UgcModularKey::Normalize(build.modules);
+			const auto combo = key.empty() ? NO_COMBINATION : UgcModularKey::StorageId(key);
+			m_ComboOf[build.id] = combo;
+			Database::Get()->SetModularBuildCombination(build.id, combo);
+		}
+		if (builds.size() < BACKFILL_BUILDS_PER_UPDATE) m_BackfillBuildsDone = true;
+	}
+	if (m_BackfillItemsDone) return;
+	const auto read = [](const std::filesystem::path& path) {
+		std::ifstream in(path, std::ios::binary);
+		return std::string(std::istreambuf_iterator<char>(in), {});
+	};
+	for (size_t i = 0; i < BACKFILL_ITEMS_PER_UPDATE && !m_BackfillItems.empty(); i++) {
+		const auto entry = m_BackfillItems.front();
+		m_BackfillItems.pop_front();
+		if (m_InFlight.contains({ entry.kind, entry.id }) || (entry.kind == Kind::MODULAR && m_ComboJobs.contains(entry.id))) continue;
+		// Only the icon: it is small (a model's .nif isn't asked for without 3D services and gets its sd0 when made again)
+		const auto checksumFile = m_Storage.File(entry.kind, entry.id, "icon.dds.checksum");
+		if (!checksumFile) continue;
+		Checksum checksum{ "icon.dds" };
+		if (!UgcFormats::ReadChecksumXml(read(*checksumFile), checksum.md5, checksum.size)) continue;
+		if (!m_Storage.File(entry.kind, entry.id, "icon.dds.sd0")) {
+			const auto packed = m_Storage.File(entry.kind, entry.id, "icon.dds.gz");
+			const auto icon = packed ? ZCompression::Gunzip(read(*packed)) : std::nullopt;
+			if (!icon || UgcFormats::Md5Hex(*icon) != checksum.md5) continue;
+			std::string error;
+			if (!m_Storage.Update(entry.kind, entry.id, { { "icon.dds.sd0", Sd0::Compress(*icon) } }, error)) {
+				LOG("Couldn't write the sd0 icon of %llu: %s", static_cast<unsigned long long>(entry.id), error.c_str());
+				continue;
+			}
+		}
+		StoreChecksums(entry.kind, entry.id, { checksum });
+	}
+	if (m_BackfillItems.empty()) {
+		m_BackfillItemsDone = true;
+		std::ofstream(m_Storage.GetRoot() / BACKFILL_MARKER) << "1\n";
+		LOG("Stored the checksums of the icons made before");
+	}
+}
+
 void UgcProcessor::Update() {
 	Collect();
+	Backfill();
 	const auto now = std::chrono::steady_clock::now();
 	if (now - m_CpuSampled >= std::chrono::seconds(2)) SampleUsage();
 	if (now >= m_NextPoll) {

dUgcServer/UgcProcessor.h

@@ -155,6 +155,13 @@ private:
 		UgcIconParams::Values iconValues; // models: the preset and override (UgcIconParams)
 	};
 
+	// A written file's MD5 and size as the client has it after inflating it (from its .checksum)
+	struct Checksum {
+		std::string file; // icon.dds, model.nif
+		std::string md5;
+		uint32_t size{};
+	};
+
 	struct Done {
 		Kind kind{};
 		LWOOBJID id{};
@@ -163,8 +170,19 @@ private:
 		uint64_t bytes{};
 		double milliseconds{};
 		bool iconOnly{};
+		std::vector<Checksum> checksums; // of the files written that the client downloads as sd0
 	};
 
+	// Main thread: records the checksums of an item's files (ugc_file_checksums)
+	void StoreChecksums(Kind kind, LWOOBJID storageId, const std::vector<Checksum>& checksums);
+
+	// Main thread: fills in what was made before the worlds answered manifest requests, a little per Update: the
+	// icons' sd0 files and checksums of the items stored, and the builds' combination ids
+	void Backfill();
+	std::deque<UgcStorage::Entry> m_BackfillItems;
+	bool m_BackfillItemsDone{ true };
+	bool m_BackfillBuildsDone{ false };
+
 	void Poll();
 	// Main thread: the icon values for a kind and an item (the kind's preset, then the item's override)
 	UgcIconParams::Values IconValues(const std::string& kind, const std::string& itemTarget);

dUgcServer/UgcServer.cpp

@@ -34,6 +34,7 @@
 #include "UgcCdClient.h"
 #include "UgcIconParams.h"
 #include "UgcJobs.h"
+#include "Sd0.h"
 #include "UgcFormats.h"
 #include "ZCompression.h"
 #include "UgcModel.h"
@@ -228,6 +229,49 @@ namespace {
 		// .hkx: no physics is made
 	}
 
+	/**
+	 * A game client download without 3D services (UGCUSE3DSERVICES=7:0, the client's default):
+	 * BrickModels/UserMade/<id % 1000, 3 digits>/<id, 20 digits><.lxfml|.nif|.hkx|.dds>.sd0 under its UGCSERVERDIR. The
+	 * client asked its world for the file's checksum first (UGC_MANIFEST_RESPONSE) and checks the inflated download
+	 * against it.
+	 */
+	void ServeClientSd0(HTTPReply& reply, const std::string& bucket, const std::string& name) {
+		NotFound(reply);
+		auto file = Lower(name);
+		if (!file.ends_with(".sd0")) return;
+		file.resize(file.size() - 4);
+		const auto dot = file.find('.');
+		if (dot == std::string::npos) return;
+		const auto extension = file.substr(dot);
+		const auto number = file.substr(0, dot);
+		const auto id = GeneralUtils::TryParse<uint64_t>(number);
+		if (!id || number.empty() || GeneralUtils::TryParse<uint64_t>(bucket) != *id % 1000) return;
+		const auto blueprint = static_cast<LWOOBJID>(*id);
+		if (extension == ".lxfml") {
+			// Straight from the ugc row, like the 3D services download; the last few are kept
+			static std::map<LWOOBJID, std::string> cache; // main thread only
+			auto it = cache.find(blueprint);
+			if (it == cache.end()) {
+				const auto model = Database::Get()->GetUgcModel(blueprint);
+				const auto lxfml = model ? UgcJobs::LxfmlFromBlob(model->lxfmlData.str()) : std::string();
+				if (lxfml.empty()) return;
+				if (cache.size() >= 64) cache.erase(cache.begin());
+				it = cache.emplace(blueprint, Sd0::Compress(lxfml)).first;
+			}
+			reply.status = eHTTPStatusCode::OK;
+			reply.contentType = eContentType::APPLICATION_OCTET_STREAM;
+			reply.message = it->second;
+			reply.headers.push_back("Cache-Control: public, max-age=60");
+		} else if (extension == ".nif") {
+			ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "model.nif.sd0", eContentType::APPLICATION_OCTET_STREAM, false);
+		} else if (extension == ".dds") {
+			// A player model's icon, else a car or rocket's (its combination's)
+			ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "icon.dds.sd0", eContentType::APPLICATION_OCTET_STREAM, false);
+			if (reply.status == eHTTPStatusCode::NOT_FOUND) ServeFile(reply, UgcStorage::Kind::MODULAR, blueprint, "icon.dds.sd0", eContentType::APPLICATION_OCTET_STREAM, false);
+		}
+		// .hkx: no physics is made, the client makes its own
+	}
+
 	void RegisterRoutes() {
 		// The configured path, and the one the 1.10.64 client uses whatever its boot.cfg says (its built-in patch server
 		// folder, lwoclient/UserBrickModels; see docs/UgcServer.md)
@@ -253,6 +297,25 @@ namespace {
 			LOG("Serving the client's downloads under %s/UGCC<datacenter>/", base.c_str());
 		}
 
+		// Without 3D services: <UGCSERVERDIR>/BrickModels/UserMade/<bucket>/<file>.sd0, where UGCSERVERDIR is client_path
+		// when boot.cfg sets it, else the client's default <PATCHSERVERDIR>/UserBrickModels (any patch folder), or empty
+		std::set<std::string> sd0Bases = { "/:patchdir/userbrickmodels", "" };
+		{
+			std::string base;
+			for (const auto& segment : Segments(Lower(Game::config->GetValue("client_path").empty() ? std::string("/ugc") : Game::config->GetValue("client_path")))) base += "/" + segment;
+			sd0Bases.insert(base);
+		}
+		for (const auto& base : sd0Bases) {
+			Game::web.RegisterHTTPRoute({ .path = base + "/brickmodels/usermade/:bucket/:file", .method = eHTTPMethod::GET, .middleware = {},
+				.handle = [](HTTPReply& reply, const HTTPContext& context) {
+					const auto segments = Segments(context.originalPath);
+					if (segments.size() < 2) return NotFound(reply);
+					ServeClientSd0(reply, segments[segments.size() - 2], segments.back());
+					LOG("Client download %s -> %i%s", context.originalPath.c_str(), static_cast<int>(reply.status), reply.file.empty() ? "" : " (file)");
+				} });
+			LOG("Serving the client's sd0 downloads under %s/BrickModels/UserMade/", base.c_str());
+		}
+
 		// Previews for the dashboard
 		Game::web.RegisterHTTPRoute({ .path = "/files/:kind/:id/:file", .method = eHTTPMethod::GET, .middleware = {},
 			.handle = [](HTTPReply& reply, const HTTPContext& context) {

dUgcServer/UgcStorage.cpp

@@ -9,9 +9,9 @@
 
 namespace {
 	constexpr std::array KNOWN_FILES = {
-		"model.nif", "model.nif.gz", "model.nif.checksum",
+		"model.nif", "model.nif.gz", "model.nif.checksum", "model.nif.sd0",
 		"model.lxfml.gz", "model.lxfml.checksum",
-		"icon.dds.gz", "icon.dds.checksum", "icon.png",
+		"icon.dds.gz", "icon.dds.checksum", "icon.dds.sd0", "icon.png",
 		"model.noao.nif", "model.noao.nif.gz", "stats.json", "combo.json",
 		"previous.icon.png", "previous.model.nif.gz", "previous.stats.json",
 	};