diff --git a/dUgcServer/UgcJobs.cpp b/dUgcServer/UgcJobs.cpp index 5e03b58a0..3db9eb4e1 100644 --- a/dUgcServer/UgcJobs.cpp +++ b/dUgcServer/UgcJobs.cpp @@ -168,12 +168,12 @@ namespace UgcJobs { return out; }; const auto nif = UgcFormats::WriteLodNif("SceneNode_Model", groups(lods.size(), opaquePieces, transparentPieces)); - outcome.files["model.nif"] = nif; + // Stored compressed only (the client downloads .gz; the dashboard's copies are inflated when asked for). The + // LXFML is served from the database. AddDownload(outcome.files, "model.nif", nif); - AddDownload(outcome.files, "model.lxfml", lxfml); { const std::vector> opaque{ UgcModel::Divide(preview.opaque) }, transparent{ transparentPieces[0] }; - outcome.files["model.noao.nif"] = UgcFormats::WriteLodNif("SceneNode_Model", groups(1, opaque, transparent)); + outcome.files["model.noao.nif.gz"] = ZCompression::Gzip(UgcFormats::WriteLodNif("SceneNode_Model", groups(1, opaque, transparent))); } // The icon is drawn from the .nif just made (its most detailed LOD, read back like any client .nif), so it diff --git a/dUgcServer/UgcProcessor.cpp b/dUgcServer/UgcProcessor.cpp index 50b8511df..fc8ae4873 100644 --- a/dUgcServer/UgcProcessor.cpp +++ b/dUgcServer/UgcProcessor.cpp @@ -161,8 +161,7 @@ void UgcProcessor::Worker() { outcome = UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings); } else if (!job.preview.Cancelled()) { // A player model's icon from its stored .nif - const auto path = m_Storage.File(Kind::MODEL, job.id, "model.nif"); - const auto nif = path ? UgcBricks::ReadFile(*path) : std::nullopt; + const auto nif = m_Storage.ReadNif(Kind::MODEL, job.id, "model.nif"); auto options = settings.icon; UgcIconParams::Apply(options, job.iconValues); outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, outcome.files, outcome.error); @@ -198,8 +197,7 @@ void UgcProcessor::Worker() { try { if (job.iconOnly) { // Only the icon, from the .nif made before - const auto path = m_Storage.File(Kind::MODEL, job.id, "model.nif"); - const auto nif = path ? UgcBricks::ReadFile(*path) : std::nullopt; + const auto nif = m_Storage.ReadNif(Kind::MODEL, job.id, "model.nif"); auto options = settings.icon; UgcIconParams::Apply(options, job.iconValues); done.outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, done.outcome.files, done.outcome.error); @@ -509,7 +507,7 @@ bool UgcProcessor::QueuePreview(Kind kind, LWOOBJID id, const std::string& modul job.modular.key = UgcModularKey::Normalize(modules); job.modular.iconValues = values; } else { - if (!m_Storage.File(Kind::MODEL, id, "model.nif")) { + if (!m_Storage.File(Kind::MODEL, id, "model.nif.gz") && !m_Storage.File(Kind::MODEL, id, "model.nif")) { error = "the model has no stored .nif yet"; return false; } @@ -535,7 +533,7 @@ size_t UgcProcessor::RegenerateIcons(const std::string& kind) { const bool models = kind == UgcIconParams::ModelKind(); for (const auto& entry : m_Storage.List()) { if (models) { - if (entry.kind != Kind::MODEL || m_InFlight.contains({ Kind::MODEL, entry.id }) || !m_Storage.File(Kind::MODEL, entry.id, "model.nif")) continue; + if (entry.kind != Kind::MODEL || m_InFlight.contains({ Kind::MODEL, entry.id }) || (!m_Storage.File(Kind::MODEL, entry.id, "model.nif.gz") && !m_Storage.File(Kind::MODEL, entry.id, "model.nif"))) continue; Job job{ Kind::MODEL, entry.id, 0 }; job.iconOnly = true; job.iconValues = IconValues(kind, UgcIconParams::ModelTarget(entry.id)); diff --git a/dUgcServer/UgcServer.cpp b/dUgcServer/UgcServer.cpp index fd4ae99ec..3d7293a0a 100644 --- a/dUgcServer/UgcServer.cpp +++ b/dUgcServer/UgcServer.cpp @@ -33,6 +33,8 @@ #include "UgcCdClient.h" #include "UgcIconParams.h" #include "UgcJobs.h" +#include "UgcFormats.h" +#include "ZCompression.h" #include "UgcModel.h" #include "UgcProcessor.h" #include "UgcStorage.h" @@ -163,6 +165,26 @@ namespace { if (crossOrigin) reply.headers.push_back("Access-Control-Allow-Origin: *"); } + /** + * A model's LXFML straight from its ugc row (it needs no making, so it's never waited for or evicted): gzip + * compressed, or its .checksum. The last few are kept in memory. + */ + void ServeLxfml(HTTPReply& reply, LWOOBJID id, bool gz) { + static std::map> cache; // id -> (gz, checksum); main thread only + auto it = cache.find(id); + if (it == cache.end()) { + const auto model = Database::Get()->GetUgcModel(id); + const auto lxfml = model ? UgcJobs::LxfmlFromBlob(model->lxfmlData.str()) : std::string(); + if (lxfml.empty()) return NotFound(reply); + if (cache.size() >= 64) cache.erase(cache.begin()); + it = cache.emplace(id, std::pair{ ZCompression::Gzip(lxfml), UgcFormats::ChecksumXml(lxfml) }).first; + } + reply.status = eHTTPStatusCode::OK; + reply.contentType = gz ? eContentType::APPLICATION_OCTET_STREAM : eContentType::TEXT_PLAIN; + reply.message = gz ? it->second.first : it->second.second; + reply.headers.push_back("Cache-Control: public, max-age=60"); + } + /** * A game client download (3D services): /UGCC/[3DOPTIMIZED/|IMAGE128DDS/]<.ext>(.gz|.checksum). * `segments` are the path's parts after client_path. @@ -194,7 +216,7 @@ namespace { const auto type = suffix == ".gz" ? eContentType::APPLICATION_OCTET_STREAM : eContentType::TEXT_PLAIN; if (extension == ".lxfml" && typeFolder.empty()) { - ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "model.lxfml" + suffix, type, false); + ServeLxfml(reply, blueprint, suffix == ".gz"); } else if (extension == ".nif" && typeFolder == "3doptimized") { ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "model.nif" + suffix, type, false); } else if (extension == ".dds" && typeFolder == "image128dds") { @@ -218,6 +240,8 @@ namespace { } segments.erase(segments.begin(), segments.begin() + static_cast(prefixSize)); ServeClientDownload(reply, segments); + // What the client asked for and what it got, to see how it loads models + LOG("Client download %s -> %i%s", context.originalPath.c_str(), static_cast(reply.status), reply.file.empty() ? "" : " (file)"); }; Game::web.RegisterHTTPRoute({ .path = base + "/:folder/:file", .method = eHTTPMethod::GET, .middleware = {}, .handle = clientRoute }); Game::web.RegisterHTTPRoute({ .path = base + "/:folder/:type/:file", .method = eHTTPMethod::GET, .middleware = {}, .handle = clientRoute }); @@ -237,6 +261,18 @@ namespace { if (!id || !kind || !PREVIEWS.contains(name)) return; reply.headers.clear(); const auto type = name.ends_with(".png") ? eContentType::IMAGE_PNG : name.ends_with(".json") ? eContentType::APPLICATION_JSON : eContentType::APPLICATION_OCTET_STREAM; + if (name.ends_with(".nif")) { + // Stored compressed: inflated for the dashboard + if (g_Processor->Request(*kind, static_cast(*id)) != UgcProcessor::Availability::READY) return; + auto nif = g_Storage->ReadNif(*kind, static_cast(*id), name); + if (!nif) return; + reply.status = eHTTPStatusCode::OK; + reply.contentType = type; + reply.message = std::move(*nif); + reply.headers.push_back("Access-Control-Allow-Origin: *"); + reply.headers.push_back("Cache-Control: no-cache"); + return; + } ServeFile(reply, *kind, static_cast(*id), name, type, true); // Made again since: the newest files differ, so they mustn't be cached as long as the client's if (reply.status == eHTTPStatusCode::OK) { diff --git a/dUgcServer/UgcStorage.cpp b/dUgcServer/UgcStorage.cpp index 00aa00f37..bd9312f9a 100644 --- a/dUgcServer/UgcStorage.cpp +++ b/dUgcServer/UgcStorage.cpp @@ -5,17 +5,19 @@ #include #include +#include "ZCompression.h" + namespace { constexpr std::array KNOWN_FILES = { "model.nif", "model.nif.gz", "model.nif.checksum", "model.lxfml.gz", "model.lxfml.checksum", "icon.dds.gz", "icon.dds.checksum", "icon.png", - "model.noao.nif", "stats.json", "combo.json", - "previous.icon.png", "previous.model.nif", "previous.model.noao.nif", "previous.stats.json", + "model.noao.nif", "model.noao.nif.gz", "stats.json", "combo.json", + "previous.icon.png", "previous.model.nif.gz", "previous.stats.json", }; // What of the files made before is kept when an item is made again, for comparing (as previous.) - constexpr std::array KEPT_FILES = { "icon.png", "model.nif", "model.noao.nif", "stats.json" }; + constexpr std::array KEPT_FILES = { "icon.png", "model.nif.gz", "stats.json" }; const char* KindFolder(UgcStorage::Kind kind) { return kind == UgcStorage::Kind::MODEL ? "models" : "modular"; @@ -180,3 +182,17 @@ std::vector UgcStorage::Evict(uint64_t maxBytes) const { } return removed; } + +std::optional UgcStorage::ReadNif(Kind kind, LWOOBJID id, const std::string& name) const { + const auto read = [](const std::filesystem::path& path) -> std::optional { + std::ifstream in(path, std::ios::binary); + if (!in) return std::nullopt; + return std::string(std::istreambuf_iterator(in), {}); + }; + if (const auto packed = File(kind, id, name + ".gz")) { + const auto data = read(*packed); + return data ? ZCompression::Gunzip(*data) : std::nullopt; + } + if (const auto plain = File(kind, id, name)) return read(*plain); + return std::nullopt; +} diff --git a/dUgcServer/UgcStorage.h b/dUgcServer/UgcStorage.h index c377f769a..ae403ba33 100644 --- a/dUgcServer/UgcStorage.h +++ b/dUgcServer/UgcStorage.h @@ -57,6 +57,10 @@ public: // Removes the least recently used items until the total is at most `maxBytes`; what was removed std::vector Evict(uint64_t maxBytes) const; + // An item's .nif (`name` model.nif, model.noao.nif or previous.model.nif): stored compressed (.gz), or as it is + // (made by older versions); nullopt when there's neither + std::optional ReadNif(Kind kind, LWOOBJID id, const std::string& name) const; + // Names the server writes (and serves); anything else is refused static bool IsKnownFile(const std::string& name); diff --git a/tests/dUgcTests/UgcTests.cpp b/tests/dUgcTests/UgcTests.cpp index 415731f39..8a83ff35b 100644 --- a/tests/dUgcTests/UgcTests.cpp +++ b/tests/dUgcTests/UgcTests.cpp @@ -557,18 +557,22 @@ TEST(UgcJobs, MakesLodsStatsAndIcons) { settings.icon.ao.samples = 4; const auto outcome = UgcJobs::ProcessModel(LXFML5, library, settings, 99); ASSERT_TRUE(outcome.ok) << outcome.error; - for (const auto* name : { "model.nif", "model.nif.gz", "model.nif.checksum", "model.noao.nif", "icon.png", "icon.dds.gz", "stats.json", "model.lxfml.gz" }) { + for (const auto* name : { "model.nif.gz", "model.nif.checksum", "model.noao.nif.gz", "icon.png", "icon.dds.gz", "stats.json" }) { EXPECT_TRUE(outcome.files.contains(name)) << name; } EXPECT_NE(outcome.stats.find("\"lods\""), std::string::npos); EXPECT_NE(outcome.stats.find("\"opaqueAfter\""), std::string::npos); + // Stored compressed only; the LXFML is served from the database + EXPECT_FALSE(outcome.files.contains("model.nif")); + EXPECT_FALSE(outcome.files.contains("model.lxfml.gz")); + const auto nifBytes = *ZCompression::Gunzip(outcome.files.at("model.nif.gz")); std::string error; - const auto nif = NifFile::Parse(outcome.files.at("model.nif"), 0, error); + const auto nif = NifFile::Parse(nifBytes, 0, error); ASSERT_TRUE(nif) << error; EXPECT_TRUE(nif->nodes.contains("S01_Opaque_Model")); EXPECT_TRUE(nif->nodes.contains("S01_Alpha_Model")); EXPECT_TRUE(nif->nodes.contains("LOD_0")); - const auto far = NifFile::Parse(outcome.files.at("model.nif"), 1, error); + const auto far = NifFile::Parse(nifBytes, 1, error); ASSERT_TRUE(far) << error; EXPECT_TRUE(far->nodes.contains("LOD_2")); // The icon is the .nif's LOD 0, drawn with the icon camera and no occlusion of its own @@ -576,7 +580,7 @@ TEST(UgcJobs, MakesLodsStatsAndIcons) { iconOptions.ao.enabled = false; EXPECT_EQ(outcome.files.at("icon.png"), UgcFormats::EncodePng(UgcRender::RenderIcon(UgcModel::FromNif(*nif), iconOptions))); // The same colors when made again - EXPECT_EQ(UgcJobs::ProcessModel(LXFML5, library, settings, 99).files.at("model.nif"), outcome.files.at("model.nif")); + EXPECT_EQ(UgcJobs::ProcessModel(LXFML5, library, settings, 99).files.at("model.nif.checksum"), outcome.files.at("model.nif.checksum")); settings.maxBricks = 2; const auto tooBig = UgcJobs::ProcessModel(LXFML5, library, settings, 99);