From 30da6575e3dd12921479eb2cb8385dc7ebe70774 Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Sun, 27 Sep 2026 13:52:38 -0500 Subject: [PATCH] fix(ugc): store meshes compressed, serve LXFML from the database, log client downloads Each model took about 18 MB (the .nif uncompressed beside its .gz, the unbaked .nif for the dashboard), so the 2 GB cap held about 110 models and the server kept evicting and remaking them while clients got 408s. Only the .gz files are stored now (the dashboard's copies are inflated when asked for), and the previous version keeps only its icon, stats and .nif.gz. The LXFML needs no making: its downloads come from the ugc row (kept in memory for the last few), so they're never waited for or evicted. Every client download is logged with its answer. Co-Authored-By: Claude Opus 5.5 --- dUgcServer/UgcJobs.cpp | 6 +++--- dUgcServer/UgcProcessor.cpp | 10 ++++------ dUgcServer/UgcServer.cpp | 38 +++++++++++++++++++++++++++++++++++- dUgcServer/UgcStorage.cpp | 22 ++++++++++++++++++--- dUgcServer/UgcStorage.h | 4 ++++ tests/dUgcTests/UgcTests.cpp | 12 ++++++++---- 6 files changed, 75 insertions(+), 17 deletions(-) diff --git a/dUgcServer/UgcJobs.cpp b/dUgcServer/UgcJobs.cpp index 5e03b58a0..3db9eb4e1 100644 --- a/dUgcServer/UgcJobs.cpp +++ b/dUgcServer/UgcJobs.cpp @@ -168,12 +168,12 @@ namespace UgcJobs { return out; }; const auto nif = UgcFormats::WriteLodNif("SceneNode_Model", groups(lods.size(), opaquePieces, transparentPieces)); - outcome.files["model.nif"] = nif; + // Stored compressed only (the client downloads .gz; the dashboard's copies are inflated when asked for). The + // LXFML is served from the database. AddDownload(outcome.files, "model.nif", nif); - AddDownload(outcome.files, "model.lxfml", lxfml); { const std::vector> opaque{ UgcModel::Divide(preview.opaque) }, transparent{ transparentPieces[0] }; - outcome.files["model.noao.nif"] = UgcFormats::WriteLodNif("SceneNode_Model", groups(1, opaque, transparent)); + outcome.files["model.noao.nif.gz"] = ZCompression::Gzip(UgcFormats::WriteLodNif("SceneNode_Model", groups(1, opaque, transparent))); } // The icon is drawn from the .nif just made (its most detailed LOD, read back like any client .nif), so it diff --git a/dUgcServer/UgcProcessor.cpp b/dUgcServer/UgcProcessor.cpp index 50b8511df..fc8ae4873 100644 --- a/dUgcServer/UgcProcessor.cpp +++ b/dUgcServer/UgcProcessor.cpp @@ -161,8 +161,7 @@ void UgcProcessor::Worker() { outcome = UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings); } else if (!job.preview.Cancelled()) { // A player model's icon from its stored .nif - const auto path = m_Storage.File(Kind::MODEL, job.id, "model.nif"); - const auto nif = path ? UgcBricks::ReadFile(*path) : std::nullopt; + const auto nif = m_Storage.ReadNif(Kind::MODEL, job.id, "model.nif"); auto options = settings.icon; UgcIconParams::Apply(options, job.iconValues); outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, outcome.files, outcome.error); @@ -198,8 +197,7 @@ void UgcProcessor::Worker() { try { if (job.iconOnly) { // Only the icon, from the .nif made before - const auto path = m_Storage.File(Kind::MODEL, job.id, "model.nif"); - const auto nif = path ? UgcBricks::ReadFile(*path) : std::nullopt; + const auto nif = m_Storage.ReadNif(Kind::MODEL, job.id, "model.nif"); auto options = settings.icon; UgcIconParams::Apply(options, job.iconValues); done.outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, done.outcome.files, done.outcome.error); @@ -509,7 +507,7 @@ bool UgcProcessor::QueuePreview(Kind kind, LWOOBJID id, const std::string& modul job.modular.key = UgcModularKey::Normalize(modules); job.modular.iconValues = values; } else { - if (!m_Storage.File(Kind::MODEL, id, "model.nif")) { + if (!m_Storage.File(Kind::MODEL, id, "model.nif.gz") && !m_Storage.File(Kind::MODEL, id, "model.nif")) { error = "the model has no stored .nif yet"; return false; } @@ -535,7 +533,7 @@ size_t UgcProcessor::RegenerateIcons(const std::string& kind) { const bool models = kind == UgcIconParams::ModelKind(); for (const auto& entry : m_Storage.List()) { if (models) { - if (entry.kind != Kind::MODEL || m_InFlight.contains({ Kind::MODEL, entry.id }) || !m_Storage.File(Kind::MODEL, entry.id, "model.nif")) continue; + if (entry.kind != Kind::MODEL || m_InFlight.contains({ Kind::MODEL, entry.id }) || (!m_Storage.File(Kind::MODEL, entry.id, "model.nif.gz") && !m_Storage.File(Kind::MODEL, entry.id, "model.nif"))) continue; Job job{ Kind::MODEL, entry.id, 0 }; job.iconOnly = true; job.iconValues = IconValues(kind, UgcIconParams::ModelTarget(entry.id)); diff --git a/dUgcServer/UgcServer.cpp b/dUgcServer/UgcServer.cpp index fd4ae99ec..3d7293a0a 100644 --- a/dUgcServer/UgcServer.cpp +++ b/dUgcServer/UgcServer.cpp @@ -33,6 +33,8 @@ #include "UgcCdClient.h" #include "UgcIconParams.h" #include "UgcJobs.h" +#include "UgcFormats.h" +#include "ZCompression.h" #include "UgcModel.h" #include "UgcProcessor.h" #include "UgcStorage.h" @@ -163,6 +165,26 @@ namespace { if (crossOrigin) reply.headers.push_back("Access-Control-Allow-Origin: *"); } + /** + * A model's LXFML straight from its ugc row (it needs no making, so it's never waited for or evicted): gzip + * compressed, or its .checksum. The last few are kept in memory. + */ + void ServeLxfml(HTTPReply& reply, LWOOBJID id, bool gz) { + static std::map> cache; // id -> (gz, checksum); main thread only + auto it = cache.find(id); + if (it == cache.end()) { + const auto model = Database::Get()->GetUgcModel(id); + const auto lxfml = model ? UgcJobs::LxfmlFromBlob(model->lxfmlData.str()) : std::string(); + if (lxfml.empty()) return NotFound(reply); + if (cache.size() >= 64) cache.erase(cache.begin()); + it = cache.emplace(id, std::pair{ ZCompression::Gzip(lxfml), UgcFormats::ChecksumXml(lxfml) }).first; + } + reply.status = eHTTPStatusCode::OK; + reply.contentType = gz ? eContentType::APPLICATION_OCTET_STREAM : eContentType::TEXT_PLAIN; + reply.message = gz ? it->second.first : it->second.second; + reply.headers.push_back("Cache-Control: public, max-age=60"); + } + /** * A game client download (3D services): /UGCC/[3DOPTIMIZED/|IMAGE128DDS/]<.ext>(.gz|.checksum). * `segments` are the path's parts after client_path. @@ -194,7 +216,7 @@ namespace { const auto type = suffix == ".gz" ? eContentType::APPLICATION_OCTET_STREAM : eContentType::TEXT_PLAIN; if (extension == ".lxfml" && typeFolder.empty()) { - ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "model.lxfml" + suffix, type, false); + ServeLxfml(reply, blueprint, suffix == ".gz"); } else if (extension == ".nif" && typeFolder == "3doptimized") { ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "model.nif" + suffix, type, false); } else if (extension == ".dds" && typeFolder == "image128dds") { @@ -218,6 +240,8 @@ namespace { } segments.erase(segments.begin(), segments.begin() + static_cast(prefixSize)); ServeClientDownload(reply, segments); + // What the client asked for and what it got, to see how it loads models + LOG("Client download %s -> %i%s", context.originalPath.c_str(), static_cast(reply.status), reply.file.empty() ? "" : " (file)"); }; Game::web.RegisterHTTPRoute({ .path = base + "/:folder/:file", .method = eHTTPMethod::GET, .middleware = {}, .handle = clientRoute }); Game::web.RegisterHTTPRoute({ .path = base + "/:folder/:type/:file", .method = eHTTPMethod::GET, .middleware = {}, .handle = clientRoute }); @@ -237,6 +261,18 @@ namespace { if (!id || !kind || !PREVIEWS.contains(name)) return; reply.headers.clear(); const auto type = name.ends_with(".png") ? eContentType::IMAGE_PNG : name.ends_with(".json") ? eContentType::APPLICATION_JSON : eContentType::APPLICATION_OCTET_STREAM; + if (name.ends_with(".nif")) { + // Stored compressed: inflated for the dashboard + if (g_Processor->Request(*kind, static_cast(*id)) != UgcProcessor::Availability::READY) return; + auto nif = g_Storage->ReadNif(*kind, static_cast(*id), name); + if (!nif) return; + reply.status = eHTTPStatusCode::OK; + reply.contentType = type; + reply.message = std::move(*nif); + reply.headers.push_back("Access-Control-Allow-Origin: *"); + reply.headers.push_back("Cache-Control: no-cache"); + return; + } ServeFile(reply, *kind, static_cast(*id), name, type, true); // Made again since: the newest files differ, so they mustn't be cached as long as the client's if (reply.status == eHTTPStatusCode::OK) { diff --git a/dUgcServer/UgcStorage.cpp b/dUgcServer/UgcStorage.cpp index 00aa00f37..bd9312f9a 100644 --- a/dUgcServer/UgcStorage.cpp +++ b/dUgcServer/UgcStorage.cpp @@ -5,17 +5,19 @@ #include #include +#include "ZCompression.h" + namespace { constexpr std::array KNOWN_FILES = { "model.nif", "model.nif.gz", "model.nif.checksum", "model.lxfml.gz", "model.lxfml.checksum", "icon.dds.gz", "icon.dds.checksum", "icon.png", - "model.noao.nif", "stats.json", "combo.json", - "previous.icon.png", "previous.model.nif", "previous.model.noao.nif", "previous.stats.json", + "model.noao.nif", "model.noao.nif.gz", "stats.json", "combo.json", + "previous.icon.png", "previous.model.nif.gz", "previous.stats.json", }; // What of the files made before is kept when an item is made again, for comparing (as previous.) - constexpr std::array KEPT_FILES = { "icon.png", "model.nif", "model.noao.nif", "stats.json" }; + constexpr std::array KEPT_FILES = { "icon.png", "model.nif.gz", "stats.json" }; const char* KindFolder(UgcStorage::Kind kind) { return kind == UgcStorage::Kind::MODEL ? "models" : "modular"; @@ -180,3 +182,17 @@ std::vector UgcStorage::Evict(uint64_t maxBytes) const { } return removed; } + +std::optional UgcStorage::ReadNif(Kind kind, LWOOBJID id, const std::string& name) const { + const auto read = [](const std::filesystem::path& path) -> std::optional { + std::ifstream in(path, std::ios::binary); + if (!in) return std::nullopt; + return std::string(std::istreambuf_iterator(in), {}); + }; + if (const auto packed = File(kind, id, name + ".gz")) { + const auto data = read(*packed); + return data ? ZCompression::Gunzip(*data) : std::nullopt; + } + if (const auto plain = File(kind, id, name)) return read(*plain); + return std::nullopt; +} diff --git a/dUgcServer/UgcStorage.h b/dUgcServer/UgcStorage.h index c377f769a..ae403ba33 100644 --- a/dUgcServer/UgcStorage.h +++ b/dUgcServer/UgcStorage.h @@ -57,6 +57,10 @@ public: // Removes the least recently used items until the total is at most `maxBytes`; what was removed std::vector Evict(uint64_t maxBytes) const; + // An item's .nif (`name` model.nif, model.noao.nif or previous.model.nif): stored compressed (.gz), or as it is + // (made by older versions); nullopt when there's neither + std::optional ReadNif(Kind kind, LWOOBJID id, const std::string& name) const; + // Names the server writes (and serves); anything else is refused static bool IsKnownFile(const std::string& name); diff --git a/tests/dUgcTests/UgcTests.cpp b/tests/dUgcTests/UgcTests.cpp index 415731f39..8a83ff35b 100644 --- a/tests/dUgcTests/UgcTests.cpp +++ b/tests/dUgcTests/UgcTests.cpp @@ -557,18 +557,22 @@ TEST(UgcJobs, MakesLodsStatsAndIcons) { settings.icon.ao.samples = 4; const auto outcome = UgcJobs::ProcessModel(LXFML5, library, settings, 99); ASSERT_TRUE(outcome.ok) << outcome.error; - for (const auto* name : { "model.nif", "model.nif.gz", "model.nif.checksum", "model.noao.nif", "icon.png", "icon.dds.gz", "stats.json", "model.lxfml.gz" }) { + for (const auto* name : { "model.nif.gz", "model.nif.checksum", "model.noao.nif.gz", "icon.png", "icon.dds.gz", "stats.json" }) { EXPECT_TRUE(outcome.files.contains(name)) << name; } EXPECT_NE(outcome.stats.find("\"lods\""), std::string::npos); EXPECT_NE(outcome.stats.find("\"opaqueAfter\""), std::string::npos); + // Stored compressed only; the LXFML is served from the database + EXPECT_FALSE(outcome.files.contains("model.nif")); + EXPECT_FALSE(outcome.files.contains("model.lxfml.gz")); + const auto nifBytes = *ZCompression::Gunzip(outcome.files.at("model.nif.gz")); std::string error; - const auto nif = NifFile::Parse(outcome.files.at("model.nif"), 0, error); + const auto nif = NifFile::Parse(nifBytes, 0, error); ASSERT_TRUE(nif) << error; EXPECT_TRUE(nif->nodes.contains("S01_Opaque_Model")); EXPECT_TRUE(nif->nodes.contains("S01_Alpha_Model")); EXPECT_TRUE(nif->nodes.contains("LOD_0")); - const auto far = NifFile::Parse(outcome.files.at("model.nif"), 1, error); + const auto far = NifFile::Parse(nifBytes, 1, error); ASSERT_TRUE(far) << error; EXPECT_TRUE(far->nodes.contains("LOD_2")); // The icon is the .nif's LOD 0, drawn with the icon camera and no occlusion of its own @@ -576,7 +580,7 @@ TEST(UgcJobs, MakesLodsStatsAndIcons) { iconOptions.ao.enabled = false; EXPECT_EQ(outcome.files.at("icon.png"), UgcFormats::EncodePng(UgcRender::RenderIcon(UgcModel::FromNif(*nif), iconOptions))); // The same colors when made again - EXPECT_EQ(UgcJobs::ProcessModel(LXFML5, library, settings, 99).files.at("model.nif"), outcome.files.at("model.nif")); + EXPECT_EQ(UgcJobs::ProcessModel(LXFML5, library, settings, 99).files.at("model.nif.checksum"), outcome.files.at("model.nif.checksum")); settings.maxBricks = 2; const auto tooBig = UgcJobs::ProcessModel(LXFML5, library, settings, 99);