Mirrored_Repos/DarkflameServer
1
0
Fork 0
mirror of https://github.com/DarkflameUniverse/DarkflameServer.git synced 2026-10-02 10:53:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ugc): store meshes compressed, serve LXFML from the database, log client downloads

Browse Source 
Each model took about 18 MB (the .nif uncompressed beside its .gz, the
unbaked .nif for the dashboard), so the 2 GB cap held about 110 models and
the server kept evicting and remaking them while clients got 408s. Only the
.gz files are stored now (the dashboard's copies are inflated when asked
for), and the previous version keeps only its icon, stats and .nif.gz.

The LXFML needs no making: its downloads come from the ugc row (kept in
memory for the last few), so they're never waited for or evicted. Every
client download is logged with its answer.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-09-27 13:52:38 -05:00
parent 1a2758feab
commit 30da6575e3
6 changed files with 75 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
dUgcServer/UgcJobs.cpp
View File

@@ -168,12 +168,12 @@ namespace UgcJobs {
return out;
};
const auto nif = UgcFormats::WriteLodNif("SceneNode_Model", groups(lods.size(), opaquePieces, transparentPieces));
outcome.files["model.nif"] = nif;
// Stored compressed only (the client downloads .gz; the dashboard's copies are inflated when asked for). The
// LXFML is served from the database.
AddDownload(outcome.files, "model.nif", nif);
AddDownload(outcome.files, "model.lxfml", lxfml);
{
const std::vector<std::vector<UgcModel::Mesh>> opaque{ UgcModel::Divide(preview.opaque) }, transparent{ transparentPieces[0] };
outcome.files["model.noao.nif"] = UgcFormats::WriteLodNif("SceneNode_Model", groups(1, opaque, transparent));
outcome.files["model.noao.nif.gz"] = ZCompression::Gzip(UgcFormats::WriteLodNif("SceneNode_Model", groups(1, opaque, transparent)));
}
// The icon is drawn from the .nif just made (its most detailed LOD, read back like any client .nif), so it

10
dUgcServer/UgcProcessor.cpp
View File

@@ -161,8 +161,7 @@ void UgcProcessor::Worker() {
outcome = UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings);
} else if (!job.preview.Cancelled()) {
// A player model's icon from its stored .nif
const auto path = m_Storage.File(Kind::MODEL, job.id, "model.nif");
const auto nif = path ? UgcBricks::ReadFile(*path) : std::nullopt;
const auto nif = m_Storage.ReadNif(Kind::MODEL, job.id, "model.nif");
auto options = settings.icon;
UgcIconParams::Apply(options, job.iconValues);
outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, outcome.files, outcome.error);
@@ -198,8 +197,7 @@ void UgcProcessor::Worker() {
try {
if (job.iconOnly) {
// Only the icon, from the .nif made before
const auto path = m_Storage.File(Kind::MODEL, job.id, "model.nif");
const auto nif = path ? UgcBricks::ReadFile(*path) : std::nullopt;
const auto nif = m_Storage.ReadNif(Kind::MODEL, job.id, "model.nif");
auto options = settings.icon;
UgcIconParams::Apply(options, job.iconValues);
done.outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, done.outcome.files, done.outcome.error);
@@ -509,7 +507,7 @@ bool UgcProcessor::QueuePreview(Kind kind, LWOOBJID id, const std::string& modul
job.modular.key = UgcModularKey::Normalize(modules);
job.modular.iconValues = values;
} else {
if (!m_Storage.File(Kind::MODEL, id, "model.nif")) {
if (!m_Storage.File(Kind::MODEL, id, "model.nif.gz") && !m_Storage.File(Kind::MODEL, id, "model.nif")) {
error = "the model has no stored .nif yet";
return false;
}
@@ -535,7 +533,7 @@ size_t UgcProcessor::RegenerateIcons(const std::string& kind) {
const bool models = kind == UgcIconParams::ModelKind();
for (const auto& entry : m_Storage.List()) {
if (models) {
if (entry.kind != Kind::MODEL || m_InFlight.contains({ Kind::MODEL, entry.id }) || !m_Storage.File(Kind::MODEL, entry.id, "model.nif")) continue;
if (entry.kind != Kind::MODEL || m_InFlight.contains({ Kind::MODEL, entry.id }) || (!m_Storage.File(Kind::MODEL, entry.id, "model.nif.gz") && !m_Storage.File(Kind::MODEL, entry.id, "model.nif"))) continue;
Job job{ Kind::MODEL, entry.id, 0 };
job.iconOnly = true;
job.iconValues = IconValues(kind, UgcIconParams::ModelTarget(entry.id));

38
dUgcServer/UgcServer.cpp
View File

@@ -33,6 +33,8 @@
#include "UgcCdClient.h"
#include "UgcIconParams.h"
#include "UgcJobs.h"
#include "UgcFormats.h"
#include "ZCompression.h"
#include "UgcModel.h"
#include "UgcProcessor.h"
#include "UgcStorage.h"
@@ -163,6 +165,26 @@ namespace {
if (crossOrigin) reply.headers.push_back("Access-Control-Allow-Origin: *");
}
/**
* A model's LXFML straight from its ugc row (it needs no making, so it's never waited for or evicted): gzip
* compressed, or its .checksum. The last few are kept in memory.
*/
void ServeLxfml(HTTPReply& reply, LWOOBJID id, bool gz) {
static std::map<LWOOBJID, std::pair<std::string, std::string>> cache; // id -> (gz, checksum); main thread only
auto it = cache.find(id);
if (it == cache.end()) {
const auto model = Database::Get()->GetUgcModel(id);
const auto lxfml = model ? UgcJobs::LxfmlFromBlob(model->lxfmlData.str()) : std::string();
if (lxfml.empty()) return NotFound(reply);
if (cache.size() >= 64) cache.erase(cache.begin());
it = cache.emplace(id, std::pair{ ZCompression::Gzip(lxfml), UgcFormats::ChecksumXml(lxfml) }).first;
}
reply.status = eHTTPStatusCode::OK;
reply.contentType = gz ? eContentType::APPLICATION_OCTET_STREAM : eContentType::TEXT_PLAIN;
reply.message = gz ? it->second.first : it->second.second;
reply.headers.push_back("Cache-Control: public, max-age=60");
}
/**
* A game client download (3D services): <client_path>/UGCC<dc>/[3DOPTIMIZED/|IMAGE128DDS/]<dc><id><.ext>(.gz|.checksum).
* `segments` are the path's parts after client_path.
@@ -194,7 +216,7 @@ namespace {
const auto type = suffix == ".gz" ? eContentType::APPLICATION_OCTET_STREAM : eContentType::TEXT_PLAIN;
if (extension == ".lxfml" && typeFolder.empty()) {
ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "model.lxfml" + suffix, type, false);
ServeLxfml(reply, blueprint, suffix == ".gz");
} else if (extension == ".nif" && typeFolder == "3doptimized") {
ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "model.nif" + suffix, type, false);
} else if (extension == ".dds" && typeFolder == "image128dds") {
@@ -218,6 +240,8 @@ namespace {
}
segments.erase(segments.begin(), segments.begin() + static_cast<std::ptrdiff_t>(prefixSize));
ServeClientDownload(reply, segments);
// What the client asked for and what it got, to see how it loads models
LOG("Client download %s -> %i%s", context.originalPath.c_str(), static_cast<int>(reply.status), reply.file.empty() ? "" : " (file)");
};
Game::web.RegisterHTTPRoute({ .path = base + "/:folder/:file", .method = eHTTPMethod::GET, .middleware = {}, .handle = clientRoute });
Game::web.RegisterHTTPRoute({ .path = base + "/:folder/:type/:file", .method = eHTTPMethod::GET, .middleware = {}, .handle = clientRoute });
@@ -237,6 +261,18 @@ namespace {
if (!id || !kind || !PREVIEWS.contains(name)) return;
reply.headers.clear();
const auto type = name.ends_with(".png") ? eContentType::IMAGE_PNG : name.ends_with(".json") ? eContentType::APPLICATION_JSON : eContentType::APPLICATION_OCTET_STREAM;
if (name.ends_with(".nif")) {
// Stored compressed: inflated for the dashboard
if (g_Processor->Request(*kind, static_cast<LWOOBJID>(*id)) != UgcProcessor::Availability::READY) return;
auto nif = g_Storage->ReadNif(*kind, static_cast<LWOOBJID>(*id), name);
if (!nif) return;
reply.status = eHTTPStatusCode::OK;
reply.contentType = type;
reply.message = std::move(*nif);
reply.headers.push_back("Access-Control-Allow-Origin: *");
reply.headers.push_back("Cache-Control: no-cache");
return;
}
ServeFile(reply, *kind, static_cast<LWOOBJID>(*id), name, type, true);
// Made again since: the newest files differ, so they mustn't be cached as long as the client's
if (reply.status == eHTTPStatusCode::OK) {

22
dUgcServer/UgcStorage.cpp
View File

@@ -5,17 +5,19 @@
#include <fstream>
#include <random>
#include "ZCompression.h"
namespace {
constexpr std::array KNOWN_FILES = {
"model.nif", "model.nif.gz", "model.nif.checksum",
"model.lxfml.gz", "model.lxfml.checksum",
"icon.dds.gz", "icon.dds.checksum", "icon.png",
"model.noao.nif", "stats.json", "combo.json",
"previous.icon.png", "previous.model.nif", "previous.model.noao.nif", "previous.stats.json",
"model.noao.nif", "model.noao.nif.gz", "stats.json", "combo.json",
"previous.icon.png", "previous.model.nif.gz", "previous.stats.json",
};
// What of the files made before is kept when an item is made again, for comparing (as previous.<name>)
constexpr std::array KEPT_FILES = { "icon.png", "model.nif", "model.noao.nif", "stats.json" };
constexpr std::array KEPT_FILES = { "icon.png", "model.nif.gz", "stats.json" };
const char* KindFolder(UgcStorage::Kind kind) {
return kind == UgcStorage::Kind::MODEL ? "models" : "modular";
@@ -180,3 +182,17 @@ std::vector<UgcStorage::Entry> UgcStorage::Evict(uint64_t maxBytes) const {
}
return removed;
}
std::optional<std::string> UgcStorage::ReadNif(Kind kind, LWOOBJID id, const std::string& name) const {
const auto read = [](const std::filesystem::path& path) -> std::optional<std::string> {
std::ifstream in(path, std::ios::binary);
if (!in) return std::nullopt;
return std::string(std::istreambuf_iterator<char>(in), {});
};
if (const auto packed = File(kind, id, name + ".gz")) {
const auto data = read(*packed);
return data ? ZCompression::Gunzip(*data) : std::nullopt;
}
if (const auto plain = File(kind, id, name)) return read(*plain);
return std::nullopt;
}

4
dUgcServer/UgcStorage.h
View File

@@ -57,6 +57,10 @@ public:
// Removes the least recently used items until the total is at most `maxBytes`; what was removed
std::vector<Entry> Evict(uint64_t maxBytes) const;
// An item's .nif (`name` model.nif, model.noao.nif or previous.model.nif): stored compressed (<name>.gz), or as it is
// (made by older versions); nullopt when there's neither
std::optional<std::string> ReadNif(Kind kind, LWOOBJID id, const std::string& name) const;
// Names the server writes (and serves); anything else is refused
static bool IsKnownFile(const std::string& name);

12
tests/dUgcTests/UgcTests.cpp
View File

@@ -557,18 +557,22 @@ TEST(UgcJobs, MakesLodsStatsAndIcons) {
settings.icon.ao.samples = 4;
const auto outcome = UgcJobs::ProcessModel(LXFML5, library, settings, 99);
ASSERT_TRUE(outcome.ok) << outcome.error;
for (const auto* name : { "model.nif", "model.nif.gz", "model.nif.checksum", "model.noao.nif", "icon.png", "icon.dds.gz", "stats.json", "model.lxfml.gz" }) {
for (const auto* name : { "model.nif.gz", "model.nif.checksum", "model.noao.nif.gz", "icon.png", "icon.dds.gz", "stats.json" }) {
EXPECT_TRUE(outcome.files.contains(name)) << name;
}
EXPECT_NE(outcome.stats.find("\"lods\""), std::string::npos);
EXPECT_NE(outcome.stats.find("\"opaqueAfter\""), std::string::npos);
// Stored compressed only; the LXFML is served from the database
EXPECT_FALSE(outcome.files.contains("model.nif"));
EXPECT_FALSE(outcome.files.contains("model.lxfml.gz"));
const auto nifBytes = *ZCompression::Gunzip(outcome.files.at("model.nif.gz"));
std::string error;
const auto nif = NifFile::Parse(outcome.files.at("model.nif"), 0, error);
const auto nif = NifFile::Parse(nifBytes, 0, error);
ASSERT_TRUE(nif) << error;
EXPECT_TRUE(nif->nodes.contains("S01_Opaque_Model"));
EXPECT_TRUE(nif->nodes.contains("S01_Alpha_Model"));
EXPECT_TRUE(nif->nodes.contains("LOD_0"));
const auto far = NifFile::Parse(outcome.files.at("model.nif"), 1, error);
const auto far = NifFile::Parse(nifBytes, 1, error);
ASSERT_TRUE(far) << error;
EXPECT_TRUE(far->nodes.contains("LOD_2"));
// The icon is the .nif's LOD 0, drawn with the icon camera and no occlusion of its own
@@ -576,7 +580,7 @@ TEST(UgcJobs, MakesLodsStatsAndIcons) {
iconOptions.ao.enabled = false;
EXPECT_EQ(outcome.files.at("icon.png"), UgcFormats::EncodePng(UgcRender::RenderIcon(UgcModel::FromNif(*nif), iconOptions)));
// The same colors when made again
EXPECT_EQ(UgcJobs::ProcessModel(LXFML5, library, settings, 99).files.at("model.nif"), outcome.files.at("model.nif"));
EXPECT_EQ(UgcJobs::ProcessModel(LXFML5, library, settings, 99).files.at("model.nif.checksum"), outcome.files.at("model.nif.checksum"));
settings.maxBricks = 2;
const auto tooBig = UgcJobs::ProcessModel(LXFML5, library, settings, 99);