From 7c5d69c1edadf7dc014ae0cfc247d7fb47af73cf Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Sun, 27 Sep 2026 09:19:59 -0500 Subject: [PATCH] feat(ugc): make models like LU Toolbox does, within CPU and memory budgets Parity with LU Toolbox's Process Model, Bake Lighting and icon renderer, with its defaults as the settings' defaults: - Colors from its LU palette (UgcPalette: LU colors, LDD colors mapped to the nearest LU one, unknown ones black), transparent bricks at 58.82% opacity, a brick transparent only when all of its materials are. - Color variation: each brick's material gets its HSV value shifted in a 2.224 gamma by up to 5% (times the color's own amount), from a random number of the model, brick and material, so reprocessing gives the same colors in every LOD. Icons get none, and the icon renderer's color corrections. - LODs 0 and 2 with its distance logic, written as NiLODNode/NiRangeLODData like its exports and the game's own brick models, shapes divided at 65535 vertices along the longest side like divide_mesh. - Ambient occlusion like its AO-only bake: 64 rays per vertex, distance 5, after hidden surface removal, transparent bricks neither baked nor occluding, glow colors added. - Icons from its icon scene: 50 mm lens at 53.4/19.5 degrees, sun of 2.5 at 21/50.3 degrees with soft shadows, grey world light with occlusion; LOD 0's hidden surface removal and occlusion are reused for them. - Optional ground plane for hidden surface removal; stats.json per model and the previous version's previews kept for comparing. Budgets, applied live on config reload: max_cpu_percent (workers account their thread CPU time and sleep to stay under it, long renders included), worker_nice, max_memory_mb (jobs are estimated from their brick count and wait until they fit), max_model_bricks and pause_hours. /status and the traffic report show CPU, memory and throttling. Co-Authored-By: Claude Opus 5.5 --- dUgcServer/CMakeLists.txt | 6 +- dUgcServer/UgcBricks.cpp | 16 +- dUgcServer/UgcBricks.h | 11 +- dUgcServer/UgcFormats.cpp | 151 +++++++++----- dUgcServer/UgcFormats.h | 20 ++ dUgcServer/UgcJobs.cpp | 171 +++++++++++++--- dUgcServer/UgcJobs.h | 26 ++- dUgcServer/UgcModel.cpp | 169 +++++++++++++++- dUgcServer/UgcModel.h | 48 ++++- dUgcServer/UgcPalette.cpp | 219 ++++++++++++++++++++ dUgcServer/UgcPalette.h | 51 +++++ dUgcServer/UgcProcessor.cpp | 157 ++++++++++++++- dUgcServer/UgcProcessor.h | 34 +++- dUgcServer/UgcRender.cpp | 375 ++++++++++++++++++++++++++--------- dUgcServer/UgcRender.h | 53 ++++- dUgcServer/UgcServer.cpp | 91 ++++++++- dUgcServer/UgcStorage.cpp | 13 ++ dUgcServer/UgcThrottle.cpp | 110 ++++++++++ dUgcServer/UgcThrottle.h | 36 ++++ resources/ugcconfig.ini | 73 +++++-- tests/dUgcTests/UgcTests.cpp | 277 +++++++++++++++++++++++++- 21 files changed, 1875 insertions(+), 232 deletions(-) create mode 100644 dUgcServer/UgcPalette.cpp create mode 100644 dUgcServer/UgcPalette.h create mode 100644 dUgcServer/UgcThrottle.cpp create mode 100644 dUgcServer/UgcThrottle.h diff --git a/dUgcServer/CMakeLists.txt b/dUgcServer/CMakeLists.txt index 43b9fdb97..780b5dca7 100644 --- a/dUgcServer/CMakeLists.txt +++ b/dUgcServer/CMakeLists.txt @@ -5,12 +5,14 @@ set(DUGC_SOURCES "UgcJobs.cpp" "UgcModel.cpp" "UgcModular.cpp" + "UgcPalette.cpp" "UgcRender.cpp" "UgcStorage.cpp" + "UgcThrottle.cpp" ) add_library(dUgc STATIC ${DUGC_SOURCES}) -target_include_directories(dUgc PUBLIC "." "${PROJECT_SOURCE_DIR}/thirdparty/MD5") +target_include_directories(dUgc PUBLIC "." "${PROJECT_SOURCE_DIR}/thirdparty/MD5" "${PROJECT_SOURCE_DIR}/thirdparty/nlohmann") target_link_libraries(dUgc PUBLIC dCommon glm::glm tinyxml2 MD5) add_executable(UgcServer "UgcServer.cpp" "UgcProcessor.cpp" "UgcCdClient.cpp") @@ -36,5 +38,5 @@ target_link_libraries(UgcServer ${COMMON_LIBRARIES} dWeb dServer dUgc) # Rendering is far too slow unoptimized (minutes a model instead of seconds), so it is optimized in every build type if(NOT MSVC) - set_source_files_properties("UgcRender.cpp" "UgcModel.cpp" PROPERTIES COMPILE_OPTIONS "-O2") + set_source_files_properties("UgcRender.cpp" "UgcModel.cpp" "UgcPalette.cpp" PROPERTIES COMPILE_OPTIONS "-O2") endif() diff --git a/dUgcServer/UgcBricks.cpp b/dUgcServer/UgcBricks.cpp index 745c0214b..d2d27ccc4 100644 --- a/dUgcServer/UgcBricks.cpp +++ b/dUgcServer/UgcBricks.cpp @@ -185,14 +185,16 @@ namespace UgcBricks { return it != m_Materials.end() ? it->second : Material{}; } - std::shared_ptr> BrickLibrary::GetDesign(uint32_t design) { + std::shared_ptr> BrickLibrary::GetDesign(uint32_t design, std::optional lodLevel) { + const uint32_t lod = std::min(lodLevel.value_or(m_Lod), 2); + const uint64_t key = (static_cast(lod) << 32) | design; { std::lock_guard lock(m_Mutex); - if (const auto it = m_Designs.find(design); it != m_Designs.end()) return it->second; + if (const auto it = m_Designs.find(key); it != m_Designs.end()) return it->second; } // Loaded outside the lock; two threads loading the same design at once is harmless auto parts = std::make_shared>(); - const auto folder = "brickprimitives/lod" + std::to_string(m_Lod) + "/"; + const auto folder = "brickprimitives/lod" + std::to_string(lod) + "/"; for (uint32_t index = 0; index < MAX_GEOMETRY_PARTS; index++) { const auto name = std::to_string(design) + ".g" + (index == 0 ? "" : std::to_string(index)); const auto path = ResolvePath(m_Res, folder + name); @@ -202,7 +204,13 @@ namespace UgcBricks { if (!geometry) break; parts->push_back(std::move(*geometry)); } + if (parts->empty() && lod > 0) return GetDesign(design, lod - 1); std::lock_guard lock(m_Mutex); - return m_Designs.emplace(design, std::move(parts)).first->second; + return m_Designs.emplace(key, std::move(parts)).first->second; + } + + size_t BrickLibrary::CachedDesigns() const { + std::lock_guard lock(m_Mutex); + return m_Designs.size(); } } diff --git a/dUgcServer/UgcBricks.h b/dUgcServer/UgcBricks.h index 2a844b17d..7d4347709 100644 --- a/dUgcServer/UgcBricks.h +++ b/dUgcServer/UgcBricks.h @@ -64,7 +64,12 @@ namespace UgcBricks { Material GetMaterial(uint32_t id) const; // Every geometry file of a design, in order (.g, .g1, ...); empty when the design has none. Loaded once. - std::shared_ptr> GetDesign(uint32_t design); + // `lod`: the brickprimitives level, the library's own when omitted; a design without that level uses the + // next more detailed one. + std::shared_ptr> GetDesign(uint32_t design, std::optional lod = std::nullopt); + + // Designs loaded (for the memory report) + size_t CachedDesigns() const; const std::filesystem::path& GetResPath() const { return m_Res; } @@ -72,7 +77,7 @@ namespace UgcBricks { std::filesystem::path m_Res; uint32_t m_Lod; std::map m_Materials; - std::mutex m_Mutex; - std::map>> m_Designs; + mutable std::mutex m_Mutex; + std::map>> m_Designs; // lod << 32 | design }; } diff --git a/dUgcServer/UgcFormats.cpp b/dUgcServer/UgcFormats.cpp index f6e31530e..af67dad5c 100644 --- a/dUgcServer/UgcFormats.cpp +++ b/dUgcServer/UgcFormats.cpp @@ -187,63 +187,120 @@ namespace { out.U16(0); // match groups return std::move(out.Data()); } + // An NiNode's data: no properties, `children`, no effects + std::string NodeData(int32_t name, const std::vector& children) { + Writer node; + WriteAv(node, name, {}); + node.U32(static_cast(children.size())); + for (const auto child : children) node.I32(child); + node.U32(0); // effects + return std::move(node.Data()); + } + + // The properties every shape shares, and the shapes + class SharedProperties { + public: + explicit SharedProperties(NifBuilder& nif) : m_Nif(nif) { + Writer material; + WriteNet(material, -1); + for (int i = 0; i < 3; i++) material.Float(1.0f); // ambient + for (int i = 0; i < 3; i++) material.Float(1.0f); // diffuse + for (int i = 0; i < 3; i++) material.Float(0.0f); // specular + for (int i = 0; i < 3; i++) material.Float(0.0f); // emissive + material.Float(10.0f); // glossiness + material.Float(1.0f); // alpha + m_Material = nif.Add("NiMaterialProperty", std::move(material.Data())); + + Writer vertexColor; + WriteNet(vertexColor, -1); + vertexColor.U16((2 << 4) | (1 << 3)); // vertex colors are ambient and diffuse; lit + m_VertexColor = nif.Add("NiVertexColorProperty", std::move(vertexColor.Data())); + } + + // An NiTriShape of `mesh` (-1 when it is empty or too big for the format) + int32_t Shape(const std::string& name, const UgcModel::Mesh* mesh, bool transparent) { + if (!mesh || mesh->Empty() || mesh->positions.size() > 65535 || mesh->TriangleCount() > 65535) return -1; + std::vector properties{ m_Material, m_VertexColor }; + if (transparent) { + if (m_Alpha < 0) { + Writer alpha; + WriteNet(alpha, -1); + alpha.U16(1 | (6 << 1) | (7 << 5)); // blend source alpha over one minus source alpha + alpha.U8(0); + m_Alpha = m_Nif.Add("NiAlphaProperty", std::move(alpha.Data())); + } + properties.push_back(m_Alpha); + } + const auto shapeBlock = m_Nif.Reserve("NiTriShape"); + const auto dataBlock = m_Nif.Add("NiTriShapeData", TriShapeData(*mesh)); + Writer tri; + WriteAv(tri, m_Nif.String(name), properties); + tri.I32(dataBlock); + tri.I32(-1); // skin instance + tri.U32(0); // materials + tri.I32(-1); // active material + tri.U8(0); // material needs update + m_Nif.Fill(shapeBlock, std::move(tri.Data())); + return shapeBlock; + } + + private: + NifBuilder& m_Nif; + int32_t m_Material{ -1 }; + int32_t m_VertexColor{ -1 }; + int32_t m_Alpha{ -1 }; + }; } namespace UgcFormats { std::string WriteNif(const std::string& rootName, const std::vector& shapes) { NifBuilder nif; const int32_t root = nif.Reserve("NiNode"); - - // Shared properties - Writer material; - WriteNet(material, -1); - for (int i = 0; i < 3; i++) material.Float(1.0f); // ambient - for (int i = 0; i < 3; i++) material.Float(1.0f); // diffuse - for (int i = 0; i < 3; i++) material.Float(0.0f); // specular - for (int i = 0; i < 3; i++) material.Float(0.0f); // emissive - material.Float(10.0f); // glossiness - material.Float(1.0f); // alpha - const auto materialBlock = nif.Add("NiMaterialProperty", std::move(material.Data())); - - Writer vertexColor; - WriteNet(vertexColor, -1); - vertexColor.U16((2 << 4) | (1 << 3)); // vertex colors are ambient and diffuse; lit - const auto vertexColorBlock = nif.Add("NiVertexColorProperty", std::move(vertexColor.Data())); - - int32_t alphaBlock = -1; + SharedProperties properties(nif); std::vector children; for (const auto& shape : shapes) { - if (!shape.mesh || shape.mesh->Empty() || shape.mesh->positions.size() > 65535 || shape.mesh->TriangleCount() > 65535) continue; - std::vector properties{ materialBlock, vertexColorBlock }; - if (shape.transparent) { - if (alphaBlock < 0) { - Writer alpha; - WriteNet(alpha, -1); - alpha.U16(1 | (6 << 1) | (7 << 5)); // blend source alpha over one minus source alpha - alpha.U8(0); - alphaBlock = nif.Add("NiAlphaProperty", std::move(alpha.Data())); - } - properties.push_back(alphaBlock); - } - const auto shapeBlock = nif.Reserve("NiTriShape"); - const auto dataBlock = nif.Add("NiTriShapeData", TriShapeData(*shape.mesh)); - Writer tri; - WriteAv(tri, nif.String(shape.name), properties); - tri.I32(dataBlock); - tri.I32(-1); // skin instance - tri.U32(0); // materials - tri.I32(-1); // active material - tri.U8(0); // material needs update - nif.Fill(shapeBlock, std::move(tri.Data())); - children.push_back(shapeBlock); + const auto block = properties.Shape(shape.name, shape.mesh, shape.transparent); + if (block >= 0) children.push_back(block); } + nif.Fill(root, NodeData(nif.String(rootName), children)); + return nif.Finish(root); + } - Writer node; - WriteAv(node, nif.String(rootName), {}); - node.U32(static_cast(children.size())); - for (const auto child : children) node.I32(child); - node.U32(0); // effects - nif.Fill(root, std::move(node.Data())); + std::string WriteLodNif(const std::string& rootName, const std::vector& groups) { + NifBuilder nif; + const int32_t root = nif.Reserve("NiNode"); + SharedProperties properties(nif); + std::vector groupBlocks; + for (const auto& group : groups) { + if (group.lods.empty()) continue; + const auto lodNode = nif.Reserve("NiLODNode"); + std::vector levels; + Writer ranges; + for (int i = 0; i < 3; i++) ranges.Float(0.0f); // LOD center + ranges.U32(static_cast(group.lods.size())); + for (const auto& lod : group.lods) { + const auto level = nif.Reserve("NiNode"); + std::vector shapes; + for (const auto* piece : lod.pieces) { + const auto block = properties.Shape(group.name, piece, group.transparent); + if (block >= 0) shapes.push_back(block); + } + nif.Fill(level, NodeData(nif.String(lod.name), shapes)); + levels.push_back(level); + ranges.Float(lod.nearDistance); + ranges.Float(lod.farDistance); + } + const auto rangeData = nif.Add("NiRangeLODData", std::move(ranges.Data())); + auto data = NodeData(nif.String(group.name), levels); + Writer lod; + lod.Raw(data); + lod.U16(3); // switch flags: update only the active child, and controllers (as the game's own files) + lod.U32(0); // index + lod.I32(rangeData); + nif.Fill(lodNode, std::move(lod.Data())); + groupBlocks.push_back(lodNode); + } + nif.Fill(root, NodeData(nif.String(rootName), groupBlocks)); return nif.Finish(root); } diff --git a/dUgcServer/UgcFormats.h b/dUgcServer/UgcFormats.h index a07ded93e..c1f146131 100644 --- a/dUgcServer/UgcFormats.h +++ b/dUgcServer/UgcFormats.h @@ -26,6 +26,26 @@ namespace UgcFormats { */ std::string WriteNif(const std::string& rootName, const std::vector& shapes); + struct NifLod { + float nearDistance{}; + float farDistance{}; + std::string name; // the level's node, e.g. LOD_0 + std::vector pieces; // its shapes (UgcModel::Divide's pieces) + }; + + struct NifLodGroup { + std::string name; // S01_Opaque_Model: the NiLODNode and its shapes + bool transparent{}; + std::vector lods; + }; + + /** + * The layout LU Toolbox exports (setup_lod_data) and the game's own brick models (res/BrickModels/ndmade) have: + * the root node, an NiLODNode per group with NiRangeLODData holding each level's distances, a node per level and + * the level's shapes under it, named like the group. Properties as WriteNif. + */ + std::string WriteLodNif(const std::string& rootName, const std::vector& groups); + // PNG (8-bit RGBA) std::string EncodePng(const UgcRender::Image& image); diff --git a/dUgcServer/UgcJobs.cpp b/dUgcServer/UgcJobs.cpp index 4e26d9a41..cc2e7b986 100644 --- a/dUgcServer/UgcJobs.cpp +++ b/dUgcServer/UgcJobs.cpp @@ -1,7 +1,10 @@ #include "UgcJobs.h" +#include #include +#include "json.hpp" + #include "Sd0.h" #include "UgcFormats.h" #include "UgcModel.h" @@ -27,8 +30,8 @@ namespace UgcJobs { namespace { // The icon files of a model, and false when nothing was drawn - bool AddIcon(UgcStorage::Files& files, const UgcModel::Model& model, const UgcRender::IconOptions& options) { - const auto icon = UgcRender::RenderIcon(model, options); + bool AddIcon(UgcStorage::Files& files, const UgcModel::Model& model, const UgcRender::IconOptions& options, const std::vector* ao = nullptr) { + const auto icon = UgcRender::RenderIcon(model, options, ao); bool drawn = false; for (size_t i = 3; i < icon.rgba.size(); i += 4) drawn = drawn || icon.rgba[i] != 0; files["icon.png"] = UgcFormats::EncodePng(icon); @@ -37,8 +40,34 @@ namespace UgcJobs { } } - Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings) { + size_t CountParts(std::string_view lxfml) { + size_t count = 0; + for (size_t at = lxfml.find("(std::clamp(settings.optimize.resolution, 64, 4096)); + const uint64_t icon = static_cast(settings.icon.size) * settings.icon.supersample; + const uint64_t fixed = resolution * resolution * 8 + icon * icon * 20 + 1024 * 1024 * 4 + 16 * 1024 * 1024; + return fixed + static_cast(parts) * 40 * 1024 * (1 + settings.lods.size()); + } + + namespace { + double Since(std::chrono::steady_clock::time_point start) { + return std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + } + + std::string ShapeName(const std::string& shader, bool transparent) { + return ("S" + (transparent ? std::string("01") : shader) + (transparent ? "_Alpha_" : "_Opaque_") + "Model").substr(0, 60); + } + } + + Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings, uint64_t seed) { Outcome outcome; + const auto started = std::chrono::steady_clock::now(); const auto lxfml = LxfmlFromBlob(blob); if (lxfml.empty()) { outcome.error = "the stored LXFML can't be read"; @@ -50,40 +79,124 @@ namespace UgcJobs { outcome.error = error; return outcome; } - auto model = UgcModel::Build(parts, library); - if (!model.missingDesigns.empty()) { - outcome.note = "no geometry for design(s)"; - for (const auto design : model.missingDesigns) outcome.note += " " + std::to_string(design); - } - if (model.Empty()) { - outcome.error = "none of the model's bricks have geometry"; - if (!outcome.note.empty()) outcome.error += " (" + outcome.note + ")"; + if (settings.maxBricks > 0 && parts.size() > settings.maxBricks) { + outcome.error = "the model has " + std::to_string(parts.size()) + " bricks, more than max_model_bricks (" + std::to_string(settings.maxBricks) + ")"; return outcome; } - const auto optimized = UgcRender::Optimize(model, settings.optimize); - outcome.aoBaked = settings.optimize.bakeAo; - if (optimized.trianglesRemoved > 0) { - if (!outcome.note.empty()) outcome.note += "; "; - outcome.note += "removed " + std::to_string(optimized.trianglesRemoved) + " of " + std::to_string(optimized.trianglesBefore) + " triangles"; - } + auto lods = settings.lods; + std::erase_if(lods, [](uint32_t lod) { return lod > 3; }); + std::sort(lods.begin(), lods.end()); + lods.erase(std::unique(lods.begin(), lods.end()), lods.end()); + if (lods.empty()) lods.push_back(0); + const auto ranges = UgcModel::LodRanges(lods, settings.lodDistances); - // Named like LU Toolbox names its shapes: S__ - std::vector> pieces; - for (const bool transparent : { false, true }) { - for (auto& piece : UgcModel::Split(transparent ? model.transparent : model.opaque)) pieces.emplace_back(std::move(piece), transparent); + nlohmann::json stats; + stats["version"] = 1; + stats["bricks"] = parts.size(); + auto& lodStats = stats["lods"] = nlohmann::json::array(); + double buildMs = 0, hsrMs = 0, aoMs = 0; + + // Every LOD made like LU Toolbox makes each LOD collection: colored, hidden faces removed, lighting baked, divided + std::vector models; + std::vector> opaquePieces, transparentPieces; + UgcModel::Model preview; // LOD 0 before the lighting bake, for the dashboard + // The icon: LU Toolbox's icon renderer imports LOD 0 again, with its own color corrections and no variation. + // The same bricks in the same order, so LOD 0's hidden surface removal and occlusion apply to it too (neither + // changes what the icon's camera sees). + UgcModel::Model iconModel; + std::vector iconAo; + double iconMs = 0; + for (size_t i = 0; i < lods.size(); i++) { + auto options = settings.build; + options.seed = seed; + options.lod = lods[i]; + auto step = std::chrono::steady_clock::now(); + auto model = UgcModel::Build(parts, library, options); + buildMs += Since(step); + if (i == 0) { + if (!model.missingDesigns.empty()) { + outcome.note = "no geometry for design(s)"; + for (const auto design : model.missingDesigns) outcome.note += " " + std::to_string(design); + stats["missingDesigns"] = model.missingDesigns; + } + if (model.Empty()) { + outcome.error = "none of the model's bricks have geometry"; + if (!outcome.note.empty()) outcome.error += " (" + outcome.note + ")"; + return outcome; + } + } + nlohmann::json entry{ { "lod", lods[i] }, { "near", ranges[i].first }, { "far", ranges[i].second }, + { "opaqueBefore", model.opaque.TriangleCount() }, { "transparent", model.transparent.TriangleCount() } }; + step = std::chrono::steady_clock::now(); + const auto optimized = UgcRender::Optimize(model, settings.optimize); + hsrMs += Since(step); + if (i == 0 && optimized.trianglesRemoved > 0) { + if (!outcome.note.empty()) outcome.note += "; "; + outcome.note += "removed " + std::to_string(optimized.trianglesRemoved) + " of " + std::to_string(optimized.trianglesBefore) + " triangles"; + } + if (i == 0) { + preview = model; + step = std::chrono::steady_clock::now(); + auto iconOptions = settings.build; + iconOptions.seed = seed; + iconOptions.lod = 0; + iconOptions.icon = settings.iconCorrectColors; + iconOptions.colorVariation = settings.iconColorVariation; + iconModel = UgcModel::Build(parts, library, iconOptions); + if (!optimized.kept.empty() && iconModel.opaque.TriangleCount() == optimized.kept.size()) UgcModel::KeepTriangles(iconModel.opaque, optimized.kept); + iconMs += Since(step); + } + step = std::chrono::steady_clock::now(); + auto ao = UgcRender::BakeAo(model, settings.ao); + aoMs += Since(step); + if (i == 0) iconAo = std::move(ao); + entry["opaqueAfter"] = model.opaque.TriangleCount(); + entry["vertices"] = model.opaque.positions.size() + model.transparent.positions.size(); + opaquePieces.push_back(UgcModel::Divide(model.opaque)); + transparentPieces.push_back(UgcModel::Divide(model.transparent)); + entry["shapes"] = opaquePieces.back().size() + transparentPieces.back().size(); + lodStats.push_back(entry); } - std::vector shapes; - size_t opaqueIndex = 0, transparentIndex = 0; - for (const auto& [piece, transparent] : pieces) { - const auto index = transparent ? transparentIndex++ : opaqueIndex++; - shapes.push_back({ std::string("S01_") + (transparent ? "Alpha" : "Opaque") + "_Model" + (index > 0 ? std::to_string(index) : ""), &piece, transparent }); - } - const auto nif = UgcFormats::WriteNif("SceneNode_Model", shapes); + outcome.aoBaked = settings.ao.enabled; + + // An NiLODNode for the opaque bricks and one for the transparent ones, as LU Toolbox names them + const auto groups = [&](size_t levels, const std::vector>& opaque, const std::vector>& transparent) { + std::vector out; + for (const bool isTransparent : { false, true }) { + UgcFormats::NifLodGroup group{ ShapeName(settings.shaderOpaque, isTransparent), isTransparent, {} }; + bool any = false; + for (size_t i = 0; i < levels; i++) { + UgcFormats::NifLod lod{ ranges[i].first, ranges[i].second, "LOD_" + std::to_string(lods[i]), {} }; + for (const auto& piece : (isTransparent ? transparent : opaque)[i]) lod.pieces.push_back(&piece); + any = any || !lod.pieces.empty(); + group.lods.push_back(std::move(lod)); + } + if (any) out.push_back(std::move(group)); + } + return out; + }; + const auto nif = UgcFormats::WriteLodNif("SceneNode_Model", groups(lods.size(), opaquePieces, transparentPieces)); outcome.files["model.nif"] = nif; AddDownload(outcome.files, "model.nif", nif); AddDownload(outcome.files, "model.lxfml", lxfml); - AddIcon(outcome.files, model, settings.icon); + { + const std::vector> opaque{ UgcModel::Divide(preview.opaque) }, transparent{ UgcModel::Divide(preview.transparent) }; + outcome.files["model.noao.nif"] = UgcFormats::WriteLodNif("SceneNode_Model", groups(1, opaque, transparent)); + } + + const auto iconStart = std::chrono::steady_clock::now(); + AddIcon(outcome.files, iconModel, settings.icon, iconAo.empty() ? nullptr : &iconAo); + iconMs += Since(iconStart); + + stats["ms"] = { { "build", std::lround(buildMs) }, { "hiddenSurfaces", std::lround(hsrMs) }, { "ambientOcclusion", std::lround(aoMs) }, + { "icon", std::lround(iconMs) }, { "total", std::lround(Since(started)) } }; + stats["settings"] = { { "palette", settings.build.palette == UgcModel::ePalette::LU_TOOLBOX ? "lu_toolbox" : "brickdb" }, + { "colorVariation", settings.build.colorVariation }, { "transparentOpacity", settings.build.transparentOpacity }, + { "removeHiddenFaces", settings.optimize.removeHidden }, { "groundPlane", settings.optimize.groundPlane }, + { "ao", settings.ao.enabled }, { "aoDistance", settings.ao.distance }, { "aoSamples", settings.ao.samples }, { "aoStrength", settings.ao.strength } }; + outcome.stats = stats.dump(); + outcome.files["stats.json"] = outcome.stats; outcome.ok = true; return outcome; } diff --git a/dUgcServer/UgcJobs.h b/dUgcServer/UgcJobs.h index 5b97cd5b4..d9ceaae9a 100644 --- a/dUgcServer/UgcJobs.h +++ b/dUgcServer/UgcJobs.h @@ -15,10 +15,22 @@ * the finished files. No database, network or CDClient: the main thread gathers the input and stores the result. */ namespace UgcJobs { + /** + * How models are made. The defaults are LU Toolbox's (Process Model, Bake Lighting and the icon renderer), see the + * parity table in docs/UgcServer.md. + */ struct Settings { - UgcRender::OptimizeOptions optimize; + UgcModel::BuildOptions build; // palette, color variation, transparent opacity + std::vector lods{ 0, 2 }; // brickprimitives levels made (LU Toolbox imports LOD 0 and 2; the client has no 3) + UgcModel::LodDistances lodDistances; + std::string shaderOpaque{ "01" }; // S_Opaque_...; transparent shapes are always S01 + UgcRender::OptimizeOptions optimize; // hidden surface removal + UgcRender::AoOptions ao; // Bake Lighting (AO Only) UgcRender::IconOptions icon; + bool iconCorrectColors{ true }; // the icon renderer's Correct Colors + float iconColorVariation{ 0.0f }; // percent; its Apply Color Variation is off UgcRender::IconOptions modularIcon; + uint32_t maxBricks{}; // a model with more fails; 0: no limit }; struct Outcome { @@ -27,6 +39,7 @@ namespace UgcJobs { std::string note; // what was odd but not fatal (missing bricks, ...) UgcStorage::Files files; // name -> bytes, when ok bool aoBaked{}; + std::string stats; // stats.json: bricks, triangles before and after per LOD, timings }; // The client's download of `data` (`name` + ".gz" and ".checksum") added to `files` @@ -35,8 +48,15 @@ namespace UgcJobs { // The LXFML of a ugc row's lxfml column (an sd0 stream, or plain LXFML); empty when it can't be read std::string LxfmlFromBlob(const std::string& blob); - // A player model: the optimized .nif, its icon and its LXFML for download - Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings); + // A player model: the optimized .nif, its icon and its LXFML for download, and stats.json. `seed` picks the color + // variation's random numbers (the model's id, so making it again gives the same colors). + Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings, uint64_t seed = 0); + + // How many bricks (parts) an LXFML has, counted cheaply (for the memory estimate before a job starts) + size_t CountParts(std::string_view lxfml); + + // About how much memory making a model of `parts` bricks takes, in bytes + uint64_t EstimateMemory(size_t parts, const Settings& settings); struct ModuleInput { uint32_t lot{}; diff --git a/dUgcServer/UgcModel.cpp b/dUgcServer/UgcModel.cpp index 8407fcadc..81edd0875 100644 --- a/dUgcServer/UgcModel.cpp +++ b/dUgcServer/UgcModel.cpp @@ -10,6 +10,7 @@ #include #include "NifFile.h" +#include "UgcPalette.h" #include "tinyxml2.h" namespace { @@ -146,6 +147,11 @@ namespace UgcModel { positions.insert(positions.end(), other.positions.begin(), other.positions.end()); normals.insert(normals.end(), other.normals.begin(), other.normals.end()); colors.insert(colors.end(), other.colors.begin(), other.colors.end()); + if (!glow.empty() || !other.glow.empty()) { + glow.resize(base, glm::vec3(0.0f)); + if (other.glow.empty()) glow.resize(positions.size(), glm::vec3(0.0f)); + else glow.insert(glow.end(), other.glow.begin(), other.glow.end()); + } indices.reserve(indices.size() + other.indices.size()); for (const auto index : other.indices) indices.push_back(base + index); } @@ -172,24 +178,59 @@ namespace UgcModel { return any; } - Model Build(const std::vector& parts, UgcBricks::BrickLibrary& library) { + Model Build(const std::vector& parts, UgcBricks::BrickLibrary& library, const BuildOptions& options) { Model model; std::set missing; - for (const auto& part : parts) { - const auto design = library.GetDesign(part.designId); + const bool luToolbox = options.palette == ePalette::LU_TOOLBOX; + bool anyGlow = false; + for (uint32_t brick = 0; brick < parts.size(); brick++) { + const auto& part = parts[brick]; + const auto design = library.GetDesign(part.designId, options.lod); if (!design || design->empty()) { missing.insert(part.designId); continue; } model.bricks++; + const auto materialOf = [&part](size_t index) { + auto id = index < part.materials.size() ? part.materials[index] : (part.materials.empty() ? 0 : part.materials[0]); + // Unknown colors are black in LU Toolbox (its name included, so black's variation too) + if (id == 0 || !UgcPalette::Linear(id)) id = UgcPalette::FALLBACK_ID; + return id; + }; + // A brick is transparent only when all of its materials are (LU Toolbox's IS_TRANSPARENT) + bool transparent = true; + for (size_t index = 0; index < design->size(); index++) { + const auto id = index < part.materials.size() ? part.materials[index] : (part.materials.empty() ? 0 : part.materials[0]); + transparent = transparent && (luToolbox ? UgcPalette::IsTransparent(materialOf(index)) : library.GetMaterial(id).Transparent()); + } + auto& mesh = transparent ? model.transparent : model.opaque; const glm::mat3 normalMatrix = glm::transpose(glm::inverse(glm::mat3(part.transform))); for (size_t index = 0; index < design->size(); index++) { const auto& geometry = (*design)[index]; - const auto materialId = index < part.materials.size() ? part.materials[index] : (part.materials.empty() ? 0 : part.materials[0]); - const auto material = library.GetMaterial(materialId); - auto& mesh = material.Transparent() ? model.transparent : model.opaque; + glm::vec3 linear{}; + float alpha = 1.0f; + glm::vec3 glow(0.0f); + uint32_t colorId{}; + if (luToolbox) { + colorId = materialOf(index); + linear = *UgcPalette::Linear(colorId, options.icon); + if (transparent) alpha = std::clamp(options.transparentOpacity / 100.0f, 0.0f, 1.0f); + if (!transparent) { + if (const auto g = UgcPalette::Glow(colorId)) glow = *g; + } + } else { + colorId = index < part.materials.size() ? part.materials[index] : (part.materials.empty() ? 0 : part.materials[0]); + const auto material = library.GetMaterial(colorId); + linear = UgcPalette::SrgbToLinear(glm::vec3(material.r, material.g, material.b) / 255.0f); + if (transparent) alpha = material.a / 255.0f; + } + if (options.colorVariation > 0.0f) { + const float variation = options.colorVariation * (luToolbox ? UgcPalette::VariationScale(colorId) : 1.0f); + linear = UgcPalette::ApplyVariation(linear, variation, UgcPalette::BrickRandom(options.seed, brick, colorId)); + } + const glm::vec4 color(UgcPalette::LinearToSrgb(linear), alpha); + anyGlow = anyGlow || glow != glm::vec3(0.0f); const auto base = static_cast(mesh.positions.size()); - const auto color = ToColor(material); const size_t vertexCount = geometry.positions.size() / 3; for (size_t v = 0; v < vertexCount; v++) { const glm::vec3 position(geometry.positions[v * 3], geometry.positions[v * 3 + 1], geometry.positions[v * 3 + 2]); @@ -200,14 +241,49 @@ namespace UgcModel { mesh.positions.push_back(glm::vec3(part.transform * glm::vec4(position, 1.0f))); mesh.normals.push_back(normal); mesh.colors.push_back(color); + if (&mesh == &model.opaque) model.opaque.glow.push_back(glow); } for (const auto i : geometry.indices) mesh.indices.push_back(base + i); } } + if (!anyGlow) model.opaque.glow.clear(); + else model.opaque.glow.resize(model.opaque.positions.size(), glm::vec3(0.0f)); model.missingDesigns.assign(missing.begin(), missing.end()); return model; } + std::vector> LodRanges(const std::vector& used, const LodDistances& d) { + // LU Toolbox's setup_lod_data ("DYNAMIC LOD HELL"), by the set of levels there are + const std::set set(used.begin(), used.end()); + const auto is = [&set](std::initializer_list levels) { return set == std::set(levels); }; + std::vector> ranges; + for (const auto level : used) { + std::pair range{ 0.0f, 0.0f }; + if (set.size() == 1) { + range = { d.lod0, d.cull }; + } else if (level == 0) { + range.first = d.lod0; + if (is({ 0, 2 }) || is({ 0, 2, 3 })) range.second = d.lod2; + else if (is({ 0, 3 })) range.second = d.lod3; + else range.second = d.lod1; + } else if (level == 1) { + if (is({ 0, 1 })) range = { d.lod1, d.cull }; + else if (is({ 1, 2 }) || is({ 1, 2, 3 })) range = { d.lod0, d.lod2 }; + else if (is({ 0, 1, 3 })) range = { d.lod1, d.lod3 }; + else if (is({ 1, 3 })) range = { d.lod0, d.lod3 }; + else if (is({ 0, 1, 2 }) || is({ 0, 1, 2, 3 })) range = { d.lod1, d.lod2 }; + } else if (level == 2) { + if (is({ 0, 2 }) || is({ 1, 2 }) || is({ 0, 1, 2 })) range = { d.lod2, d.cull }; + else if (is({ 2, 3 })) range = { d.lod0, d.lod3 }; + else if (is({ 0, 2, 3 }) || is({ 1, 2, 3 }) || is({ 0, 1, 2, 3 })) range = { d.lod2, d.lod3 }; + } else if (level == 3) { + range = { d.lod3, d.cull }; + } + ranges.push_back(range); + } + return ranges; + } + Model FromNif(const NifFile::Model& nif) { Model model; for (const auto& source : nif.meshes) { @@ -241,6 +317,26 @@ namespace UgcModel { return model; } + void KeepTriangles(Mesh& mesh, const std::vector& keep) { + Mesh kept; + std::vector remap(mesh.positions.size(), UINT32_MAX); + for (size_t t = 0; t < mesh.TriangleCount(); t++) { + if (t >= keep.size() || !keep[t]) continue; + for (int k = 0; k < 3; k++) { + const auto source = mesh.indices[t * 3 + k]; + if (remap[source] == UINT32_MAX) { + remap[source] = static_cast(kept.positions.size()); + kept.positions.push_back(mesh.positions[source]); + if (source < mesh.normals.size()) kept.normals.push_back(mesh.normals[source]); + if (source < mesh.colors.size()) kept.colors.push_back(mesh.colors[source]); + if (source < mesh.glow.size()) kept.glow.push_back(mesh.glow[source]); + } + kept.indices.push_back(remap[source]); + } + } + mesh = std::move(kept); + } + std::vector Split(const Mesh& mesh, size_t maxVertices, size_t maxTriangles) { std::vector pieces; if (mesh.positions.size() <= maxVertices && mesh.TriangleCount() <= maxTriangles) { @@ -265,6 +361,7 @@ namespace UgcModel { current.positions.push_back(mesh.positions[source]); if (source < mesh.normals.size()) current.normals.push_back(mesh.normals[source]); if (source < mesh.colors.size()) current.colors.push_back(mesh.colors[source]); + if (source < mesh.glow.size()) current.glow.push_back(mesh.glow[source]); } current.indices.push_back(it->second); } @@ -272,4 +369,62 @@ namespace UgcModel { flush(); return pieces; } + + std::vector Divide(const Mesh& mesh, size_t maxVertices, size_t maxTriangles) { + if (mesh.Empty()) return {}; + if (mesh.positions.size() <= maxVertices && mesh.TriangleCount() <= maxTriangles) return { mesh }; + + // Connected pieces (vertices joined by triangles), so a brick's faces stay together + std::vector parent(mesh.positions.size()); + for (uint32_t i = 0; i < parent.size(); i++) parent[i] = i; + const std::function find = [&](uint32_t x) { + while (parent[x] != x) x = parent[x] = parent[parent[x]]; + return x; + }; + for (size_t i = 0; i + 2 < mesh.indices.size(); i += 3) { + const auto a = find(mesh.indices[i]); + parent[find(mesh.indices[i + 1])] = a; + parent[find(mesh.indices[i + 2])] = a; + } + + // divide_mesh: vertices below the mean along the longest side of the bounds, and everything linked to them + glm::vec3 min = mesh.positions[0], max = mesh.positions[0], mean(0.0f); + for (const auto& p : mesh.positions) { + min = glm::min(min, p); + max = glm::max(max, p); + mean += p; + } + mean /= static_cast(mesh.positions.size()); + const auto size = max - min; + const int axis = size.x >= size.y && size.x >= size.z ? 0 : size.y >= size.z ? 1 : 2; + std::vector below(mesh.positions.size(), false); + for (uint32_t v = 0; v < mesh.positions.size(); v++) { + if (mesh.positions[v][axis] < mean[axis]) below[find(v)] = true; + } + + Mesh halves[2]; + std::vector remap(mesh.positions.size(), UINT32_MAX); + for (size_t i = 0; i + 2 < mesh.indices.size(); i += 3) { + auto& half = halves[below[find(mesh.indices[i])] ? 1 : 0]; + for (int k = 0; k < 3; k++) { + const auto source = mesh.indices[i + k]; + if (remap[source] == UINT32_MAX) { + remap[source] = static_cast(half.positions.size()); + half.positions.push_back(mesh.positions[source]); + if (source < mesh.normals.size()) half.normals.push_back(mesh.normals[source]); + if (source < mesh.colors.size()) half.colors.push_back(mesh.colors[source]); + if (source < mesh.glow.size()) half.glow.push_back(mesh.glow[source]); + } + half.indices.push_back(remap[source]); + } + } + // LU Toolbox gives up below a 10% share; this splits the old way then + const float share = static_cast(halves[1].positions.size()) / static_cast(mesh.positions.size()); + if (std::min(share, 1.0f - share) < 0.1f) return Split(mesh, maxVertices, maxTriangles); + std::vector pieces; + for (const auto& half : halves) { + for (auto& piece : Divide(half, maxVertices, maxTriangles)) pieces.push_back(std::move(piece)); + } + return pieces; + } } diff --git a/dUgcServer/UgcModel.h b/dUgcServer/UgcModel.h index 50016d59b..0375ffbe3 100644 --- a/dUgcServer/UgcModel.h +++ b/dUgcServer/UgcModel.h @@ -33,6 +33,7 @@ namespace UgcModel { std::vector positions; std::vector normals; std::vector colors; // sRGB, 0..1, alpha is opacity + std::vector glow; // linear glow color per vertex (LU Toolbox's "Glow" layer); empty when nothing glows std::vector indices; size_t TriangleCount() const { return indices.size() / 3; } @@ -52,12 +53,55 @@ namespace UgcModel { bool Bounds(glm::vec3& min, glm::vec3& max) const; }; - // The mesh of a model's parts - Model Build(const std::vector& parts, UgcBricks::BrickLibrary& library); + enum class ePalette { + LU_TOOLBOX, // LU Toolbox's colors (UgcPalette), what its importer colors models with + BRICKDB, // the brick database's Materials.xml + }; + + struct BuildOptions { + ePalette palette{ ePalette::LU_TOOLBOX }; + float colorVariation{ 5.0f }; // percent, 0: none (LU Toolbox: Apply Color Variation, 5%) + uint64_t seed{}; // of the variation's random numbers + float transparentOpacity{ 58.82f }; // percent, transparent bricks' vertex alpha (LU Toolbox palette only) + bool icon{}; // the icon renderer's color corrections + uint32_t lod{}; // brickprimitives level + }; + + /** + * The mesh of a model's parts, colored as LU Toolbox's Process Model does: a brick is transparent only when all of + * its materials are, each material of each brick has its brightness shifted by the color variation (the same + * random number for a brick's material in every LOD and every time), vertex colors are sRGB with alpha 1 for + * opaque bricks and the transparent opacity for transparent ones. + */ + Model Build(const std::vector& parts, UgcBricks::BrickLibrary& library, const BuildOptions& options = {}); + + /** + * The distance range (near, far) of each LOD LU Toolbox makes, for the brickprimitives levels in `used` (0 to 3), + * from its settings (lod0..lod3, cull): its setup_lod_data, which picks the ranges by which levels are there. + * {0, 0} for a level it has no range for. + */ + struct LodDistances { + float lod0{ 0.0f }; + float lod1{ 50.0f }; + float lod2{ 100.0f }; + float lod3{ 280.0f }; + float cull{ 10000.0f }; + }; + std::vector> LodRanges(const std::vector& used, const LodDistances& distances); + + /** + * Splits a mesh the way LU Toolbox's divide_mesh does while it has too many vertices (or triangles): at the mean + * of its vertices along its longest side, keeping connected pieces whole. Falls back to Split when that can't + * divide it. + */ + std::vector Divide(const Mesh& mesh, size_t maxVertices = 65535, size_t maxTriangles = 65535); // A client .nif's meshes as one model (vertex colors times material color; transparent when blended) Model FromNif(const NifFile::Model& nif); + // Keeps the triangles whose flag is set (and the vertices they use) + void KeepTriangles(Mesh& mesh, const std::vector& keep); + // Splits a mesh into pieces the .nif format can hold (at most `maxVertices` vertices and `maxTriangles` triangles) std::vector Split(const Mesh& mesh, size_t maxVertices = 65535, size_t maxTriangles = 65535); } diff --git a/dUgcServer/UgcPalette.cpp b/dUgcServer/UgcPalette.cpp new file mode 100644 index 000000000..1cbd8105d --- /dev/null +++ b/dUgcServer/UgcPalette.cpp @@ -0,0 +1,219 @@ +#include "UgcPalette.h" + +#include +#include +#include + +namespace { + using Table = std::unordered_map; + + glm::vec3 FromSrgb(float r, float g, float b) { + return UgcPalette::SrgbToLinear(glm::vec3(r, g, b)); + } + + // Colors are linear, as LU Toolbox keeps them + const Table& Opaque() { + static const Table table = [] { + Table t = { + { 1, { 0.904661f, 0.904661f, 0.904661f } }, + { 5, { 0.693872f, 0.491021f, 0.194618f } }, + { 18, { 0.672443f, 0.168269f, 0.051269f } }, + { 21, { 0.730461f, 0.0f, 0.004025f } }, + { 23, { 0.0f, 0.095307f, 0.391572f } }, + { 24, { 0.991102f, 0.552011f, 0.0f } }, + { 26, { 0.006f, 0.006f, 0.006f } }, + { 28, { 0.0f, 0.198069f, 0.021219f } }, + { 37, { 0.0f, 0.304987f, 0.017642f } }, + { 38, { 0.391572f, 0.046665f, 0.007499f } }, + { 50, { 0.7399f, 0.7399f, 0.7399f } }, + { 102, { 0.06301f, 0.262251f, 0.564712f } }, + { 106, { 0.799103f, 0.124772f, 0.009134f } }, + { 107, { 0.002732f, 0.462077f, 0.462077f } }, + { 119, { 0.296138f, 0.47932f, 0.003035f } }, + { 120, { 0.672444f, 0.768151f, 0.262251f } }, + { 124, { 0.332452f, 0.0f, 0.147027f } }, + { 135, { 0.111932f, 0.174647f, 0.262251f } }, + { 138, { 0.262251f, 0.177888f, 0.084376f } }, + { 140, { 0.0f, 0.0185f, 0.052861f } }, + { 141, { 0.0f, 0.03434f, 0.008023f } }, + { 151, { 0.114435f, 0.223228f, 0.130137f } }, + { 154, { 0.215861f, 0.002428f, 0.01096f } }, + { 191, { 0.887923f, 0.318547f, 0.0f } }, + { 192, { 0.104617f, 0.011612f, 0.003677f } }, + { 194, { 0.332452f, 0.283149f, 0.283149f } }, + { 199, { 0.072272f, 0.082283f, 0.093059f } }, + { 208, { 0.768151f, 0.768151f, 0.693872f } }, + { 212, { 0.242281f, 0.520996f, 0.83077f } }, + { 221, { 0.730461f, 0.038204f, 0.258183f } }, + { 222, { 0.846873f, 0.341914f, 0.53948f } }, + { 226, { 1.0f, 0.768151f, 0.141263f } }, + { 268, { 0.066626f, 0.011612f, 0.341914f } }, + { 283, { 0.913099f, 0.533276f, 0.250158f } }, + { 294, { 0.991102f, 0.973445f, 0.665388f } }, + { 308, { 0.03434f, 0.015209f, 0.0f } }, + { 329, { 1.0f, 1.0f, 1.0f } }, + { 330, { 0.184475f, 0.184475f, 0.076185f } }, + { 9013, { 1.0f, 0.009721f, 0.020289f } }, + { 9014, { 0.799103f, 0.088655f, 0.0f } }, + { 9015, { 1.0f, 0.630757f, 0.033105f } }, + { 9016, { 0.012983f, 1.0f, 0.090842f } }, + { 9017, { 0.059511f, 0.768152f, 0.913099f } }, + { 9018, { 0.0f, 0.226966f, 1.0f } }, + { 9019, { 0.40724f, 0.012983f, 1.0f } }, + { 9020, { 0.686686f, 0.000303f, 1.0f } }, + }; + // LDD colors LU doesn't have, as the nearest LU color + const std::pair aliases[] = { + { 0, 26 }, { 2, 194 }, { 3, 5 }, { 4, 106 }, { 6, 119 }, { 9, 222 }, { 11, 212 }, { 12, 106 }, { 13, 191 }, { 19, 191 }, + { 22, 221 }, { 25, 192 }, { 27, 199 }, { 29, 151 }, { 36, 283 }, { 39, 194 }, { 45, 212 }, { 100, 283 }, { 101, 106 }, + { 103, 194 }, { 104, 268 }, { 105, 191 }, { 110, 23 }, { 112, 23 }, { 115, 119 }, { 116, 107 }, { 118, 212 }, { 326, 120 }, + { 125, 283 }, { 128, 38 }, { 133, 106 }, { 134, 119 }, { 136, 135 }, { 153, 138 }, { 180, 191 }, { 195, 23 }, { 196, 23 }, + { 198, 124 }, { 216, 154 }, { 217, 138 }, { 218, 124 }, { 219, 268 }, { 223, 222 }, { 232, 212 }, { 233, 37 }, { 295, 222 }, + { 312, 138 }, { 321, 102 }, { 322, 212 }, { 323, 208 }, { 324, 124 }, { 325, 222 }, + }; + for (const auto& [id, target] : aliases) t[id] = t.at(target); + return t; + }(); + return table; + } + + const Table& Transparent() { + static const Table table = [] { + Table t = { + { 20, { 0.930111f, 0.672443f, 0.250158f } }, + { 40, { 0.854993f, 0.854993f, 0.854993f } }, + { 41, FromSrgb(0.674509f, 0.0f, 0.0f) }, + { 42, FromSrgb(0.244106f, 0.720966f, 0.772058f) }, + { 43, FromSrgb(0.031372f, 0.285668f, 0.643137f) }, + { 44, FromSrgb(0.858f, 0.771375f, 0.0f) }, + { 47, FromSrgb(0.986f, 0.336526f, 0.120035f) }, + { 48, FromSrgb(0.0f, 0.391f, 0.0f) }, + { 49, FromSrgb(0.697f, 1.0f, 0.0f) }, + { 111, FromSrgb(0.741177f, 0.670588f, 0.639216f) }, + { 113, FromSrgb(0.754717f, 0.060520f, 0.541647f) }, + { 126, FromSrgb(0.267974f, 0.196078f, 0.627451f) }, + { 143, FromSrgb(0.325985f, 0.551358f, 0.821f) }, + { 182, FromSrgb(0.913726f, 0.524575f, 0.0156863f) }, + { 311, FromSrgb(0.454640f, 0.788235f, 0.0980392f) }, + }; + const std::pair aliases[] = { { 157, 44 }, { 230, 113 }, { 231, 182 }, { 234, 44 }, { 284, 126 }, { 285, 111 }, { 293, 43 } }; + for (const auto& [id, target] : aliases) t[id] = t.at(target); + return t; + }(); + return table; + } + + const Table& GlowColors() { + static const Table table = [] { + const auto& opaque = Opaque(); + Table t = { { 50, { 0.401978f, 0.401978f, 0.401978f } } }; + for (const uint32_t id : { 329u, 294u, 9013u, 9014u, 9015u, 9016u, 9017u, 9018u, 9019u, 9020u }) t[id] = opaque.at(id); + const std::pair aliases[] = { + { 9000, 9020 }, { 9002, 9016 }, { 9004, 9018 }, { 9008, 9013 }, { 9009, 9014 }, { 9010, 9016 }, { 9011, 9017 }, { 9012, 9019 }, + { 9021, 329 }, { 9022, 50 }, { 9023, 9013 }, { 9024, 9014 }, { 9025, 9016 }, { 9026, 9018 }, { 9027, 329 }, + }; + for (const auto& [id, target] : aliases) t[id] = t.at(target); + return t; + }(); + return table; + } + + const Table& Metallic() { + static const Table table = [] { + Table t; + const auto add = [&t](std::initializer_list ids, glm::vec3 color) { for (const auto id : ids) t[id] = color; }; + add({ 131, 150, 179, 298, 315 }, { 0.262251f, 0.296138f, 0.296138f }); + add({ 139, 187, 300 }, { 0.174648f, 0.066626f, 0.029557f }); + add({ 148 }, { 0.06301f, 0.051269f, 0.043735f }); + add({ 149 }, { 0.006f, 0.006f, 0.006f }); + add({ 184 }, { 0.238095f, 0.00907f, 0.00907f }); + add({ 186, 200 }, { 0.081104f, 0.252379f, 0.045668f }); + add({ 145, 185 }, { 0.104617f, 0.177888f, 0.278894f }); + add({ 309, 183 }, { 0.617207f, 0.617207f, 0.617207f }); + add({ 297, 147, 189 }, { 0.401978f, 0.212231f, 0.027321f }); + add({ 310, 127 }, { 0.737911f, 0.533276f, 0.181164f }); + return t; + }(); + return table; + } + + const std::unordered_map& CustomVariation() { + static const std::unordered_map table = { + { 1, 1.3f }, { 21, 1.4f }, { 23, 1.25f }, { 24, 1.5f }, { 26, 0.4f }, { 28, 0.8f }, { 37, 0.8f }, { 135, 0.85f }, { 141, 0.7f }, + { 199, 0.7f }, { 192, 0.75f }, { 212, 1.25f }, { 222, 1.05f }, { 226, 1.75f }, { 283, 1.15f }, { 308, 0.85f }, { 323, 1.4f }, { 326, 1.75f }, + }; + return table; + } + + uint64_t SplitMix(uint64_t x) { + x += 0x9E3779B97F4A7C15ull; + x = (x ^ (x >> 30)) * 0xBF58476D1CE4E5B9ull; + x = (x ^ (x >> 27)) * 0x94D049BB133111EBull; + return x ^ (x >> 31); + } +} + +namespace UgcPalette { + std::optional Linear(uint32_t id, bool icon) { + if (icon) { + // ICON_MATERIALS_OPAQUE + if (id == 1) return FromSrgb(0.7f, 0.7f, 0.7f); + if (id == 26) return FromSrgb(0.01f, 0.01f, 0.01f); + } + // The importer loads opaque, then transparent, metallic and glow colors, the first one of an id winning + for (const auto* table : { &Opaque(), &Transparent(), &Metallic(), &GlowColors() }) { + if (const auto it = table->find(id); it != table->end()) return it->second; + } + return std::nullopt; + } + + bool IsTransparent(uint32_t id) { + return Transparent().contains(id); + } + + bool IsMetallic(uint32_t id) { + return Metallic().contains(id); + } + + std::optional Glow(uint32_t id) { + const auto it = GlowColors().find(id); + return it != GlowColors().end() ? std::optional(it->second) : std::nullopt; + } + + float VariationScale(uint32_t id) { + const auto it = CustomVariation().find(id); + return it != CustomVariation().end() ? it->second : 1.0f; + } + + float SrgbToLinear(float srgb) { + return srgb <= 0.0404482362771082f ? srgb / 12.92f : std::pow((srgb + 0.055f) / 1.055f, 2.4f); + } + + float LinearToSrgb(float linear) { + return linear > 0.0031308f ? 1.055f * std::pow(linear, 1.0f / 2.4f) - 0.055f : 12.92f * linear; + } + + glm::vec3 SrgbToLinear(const glm::vec3& srgb) { + return { SrgbToLinear(srgb.r), SrgbToLinear(srgb.g), SrgbToLinear(srgb.b) }; + } + + glm::vec3 LinearToSrgb(const glm::vec3& linear) { + return { LinearToSrgb(linear.r), LinearToSrgb(linear.g), LinearToSrgb(linear.b) }; + } + + glm::vec3 ApplyVariation(const glm::vec3& color, float variationPercent, float random) { + const float value = std::max({ color.r, color.g, color.b }); + float gamma = std::pow(std::max(value, 0.0f), 1.0f / 2.224f); + const float range = variationPercent / 200.0f; + gamma += -range + random * 2.0f * range; + const float newValue = std::pow(std::clamp(gamma, 0.0f, 1.0f), 2.224f); + // Setting an HSV value keeps hue and saturation: the channels scale together (a black color becomes grey) + if (value <= 0.0f) return glm::vec3(newValue); + return color * (newValue / value); + } + + float BrickRandom(uint64_t seed, uint32_t brick, uint32_t material) { + const auto bits = SplitMix(SplitMix(seed ^ (static_cast(brick) << 32)) ^ material); + return static_cast(bits >> 40) / static_cast(1ull << 24); + } +} diff --git a/dUgcServer/UgcPalette.h b/dUgcServer/UgcPalette.h new file mode 100644 index 000000000..4a1a053c3 --- /dev/null +++ b/dUgcServer/UgcPalette.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include + +#include + +/** + * LU Toolbox's color palette (its materials/__init__.py), which it colors imported models with instead of the brick + * database's Materials.xml: LU's colors in linear RGB, the LDD colors LU doesn't have mapped onto the nearest LU one, + * which colors are transparent, glow and metallic, how much each color varies from brick to brick, and the icon + * renderer's corrections. Also the brick to brick color variation itself (process_model.py apply_color_variation). + * Pure. + */ +namespace UgcPalette { + // A material id's color in linear RGB; nullopt when LU Toolbox doesn't know it (it uses black, 26, then). + // `icon`: with the icon renderer's corrections (white and black are toned down). + std::optional Linear(uint32_t id, bool icon = false); + + bool IsTransparent(uint32_t id); + bool IsMetallic(uint32_t id); + + // The glow color (linear) of a glowing material, nullopt for the others + std::optional Glow(uint32_t id); + + // How much of the color variation a color gets (CUSTOM_VARIATION: black varies less, orange more); 1 by default + float VariationScale(uint32_t id); + + // The color LU Toolbox falls back to for unknown ids + constexpr uint32_t FALLBACK_ID = 26; + + // sRGB <-> linear, exactly as LU Toolbox converts (color_conversions.py) + float SrgbToLinear(float srgb); + float LinearToSrgb(float linear); + glm::vec3 SrgbToLinear(const glm::vec3& srgb); + glm::vec3 LinearToSrgb(const glm::vec3& linear); + + /** + * LU Toolbox's color variation of one color: its HSV value, taken to a 1/2.224 gamma, shifted by + * `random` (0..1, mapped to -variation/200 .. +variation/200, variation in percent), clamped to 0..1 and taken back. + * Hue and saturation stay. `color` is linear RGB. + */ + glm::vec3 ApplyVariation(const glm::vec3& color, float variationPercent, float random); + + /** + * The random number (0..1) of one brick's material: the same for the same seed, brick and material every time, + * so making a model again gives the same colors, and the same in every LOD (LU Toolbox restarts its random sequence + * for each LOD). `brick` is the brick's index in the LXFML. + */ + float BrickRandom(uint64_t seed, uint32_t brick, uint32_t material); +} diff --git a/dUgcServer/UgcProcessor.cpp b/dUgcServer/UgcProcessor.cpp index b4ec8865e..cf7cc9ef8 100644 --- a/dUgcServer/UgcProcessor.cpp +++ b/dUgcServer/UgcProcessor.cpp @@ -4,8 +4,19 @@ #include "Database.h" #include "Logger.h" #include "UgcCdClient.h" +#include "UgcThrottle.h" #include "json.hpp" +#include +#include +#include +#if defined(__linux__) +#include +#include +#include +#include +#endif + namespace { constexpr size_t MAX_ERROR_LENGTH = 1000; // process_error holds 1024 constexpr size_t LOG_LENGTH = 50; @@ -16,6 +27,22 @@ namespace { return kind == UgcStorage::Kind::MODEL ? "model" : "modular"; } + // The process's CPU time (user and system, every thread), seconds + double ProcessCpuSeconds() { + timespec ts{}; + if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts) == 0) return static_cast(ts.tv_sec) + ts.tv_nsec / 1e9; + return 0.0; + } + + void ApplyNice(int nice) { +#if defined(__linux__) + // Per thread on Linux: only the calling worker's priority changes + setpriority(PRIO_PROCESS, static_cast(syscall(SYS_gettid)), std::clamp(nice, 0, 19)); +#else + (void)nice; +#endif + } + int64_t UnixNow() { return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); } @@ -24,6 +51,41 @@ namespace { UgcProcessor::UgcProcessor(Config config, UgcStorage& storage, UgcBricks::BrickLibrary& library, UgcJobs::Settings settings) : m_Config(config), m_Storage(storage), m_Library(library), m_Settings(std::move(settings)) {} +uint64_t UgcProcessor::ResidentBytes() { +#if defined(__linux__) + std::ifstream statm("/proc/self/statm"); + uint64_t size = 0, resident = 0; + if (statm >> size >> resident) return resident * static_cast(sysconf(_SC_PAGESIZE)); +#endif + return 0; +} + +bool UgcProcessor::Throttled() const { + const auto last = UgcThrottle::GetStats().lastSleepUnixMs; + return last > 0 && UnixNow() * 1000 - last < 5000; +} + +void UgcProcessor::Configure(UgcJobs::Settings settings, Limits limits) { + { + std::lock_guard lock(m_Mutex); + m_Settings = std::move(settings); + m_Limits = limits; + } + UgcThrottle::SetBudget(limits.maxCpus); + m_Wake.notify_all(); +} + +void UgcProcessor::SampleUsage() { + const auto now = std::chrono::steady_clock::now(); + const double cpu = ProcessCpuSeconds(); + if (m_CpuSampled.time_since_epoch().count() != 0) { + const double elapsed = std::chrono::duration(now - m_CpuSampled).count(); + if (elapsed > 0.0) m_CpuPercent = (cpu - m_CpuSeconds) / elapsed * 100.0; + } + m_CpuSampled = now; + m_CpuSeconds = cpu; +} + UgcProcessor::~UgcProcessor() { Stop(); } @@ -50,27 +112,54 @@ void UgcProcessor::Stop() { } void UgcProcessor::Worker() { + int appliedNice = 0; while (true) { Job job; + UgcJobs::Settings settings; + int nice = 0; { std::unique_lock lock(m_Mutex); - m_Wake.wait(lock, [this] { return m_Stopping || !m_Jobs.empty(); }); + // The first job that fits the memory budget beside the ones running; one that is too big alone runs + // when nothing else does + auto pick = m_Jobs.end(); + m_Wake.wait(lock, [this, &pick] { + if (m_Stopping) return true; + pick = m_Jobs.end(); + for (auto it = m_Jobs.begin(); it != m_Jobs.end(); ++it) { + if (m_Limits.maxMemoryBytes == 0 || m_Active == 0 || m_MemoryInUse + it->memory <= m_Limits.maxMemoryBytes) { + pick = it; + break; + } + } + if (pick == m_Jobs.end() && !m_Jobs.empty()) m_Waiting++; + return pick != m_Jobs.end(); + }); if (m_Stopping) return; - job = std::move(m_Jobs.front()); - m_Jobs.pop_front(); + job = std::move(*pick); + m_Jobs.erase(pick); m_Active++; + m_MemoryInUse += job.memory; + settings = m_Settings; + nice = m_Limits.nice; + } + if (nice != appliedNice) { + ApplyNice(nice); + appliedNice = nice; } + UgcThrottle::Begin(); const auto start = std::chrono::steady_clock::now(); Done done{ job.kind, job.id, job.attempts }; try { done.outcome = job.kind == Kind::MODEL - ? UgcJobs::ProcessModel(job.blob, m_Library, m_Settings) - : UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), m_Settings); + ? UgcJobs::ProcessModel(job.blob, m_Library, settings, static_cast(job.id)) + : UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings); } catch (const std::exception& ex) { done.outcome.ok = false; done.outcome.error = std::string("crashed: ") + ex.what(); } + job.blob.clear(); + job.blob.shrink_to_fit(); if (done.outcome.ok) { std::string error; const auto bytes = m_Storage.Write(job.kind, job.id, done.outcome.files, error); @@ -83,20 +172,42 @@ void UgcProcessor::Worker() { } done.outcome.files.clear(); done.milliseconds = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + UgcThrottle::Checkpoint(); - std::lock_guard lock(m_Mutex); - m_Active--; - m_Done.push_back(std::move(done)); + { + std::lock_guard lock(m_Mutex); + m_Active--; + m_MemoryInUse -= std::min(m_MemoryInUse, job.memory); + m_Done.push_back(std::move(done)); + } +#if defined(__linux__) && defined(__GLIBC__) + // Give the big buffers of the job back to the system + malloc_trim(0); +#endif + m_Wake.notify_all(); } } void UgcProcessor::Poll() { size_t queued = 0, active = 0; + Limits limits; + UgcJobs::Settings settings; { std::lock_guard lock(m_Mutex); queued = m_Jobs.size(); active = m_Active; + limits = m_Limits; + settings = m_Settings; } + const auto now = std::time(nullptr); + std::tm local{}; +#if defined(_WIN32) + localtime_s(&local, &now); +#else + localtime_r(&now, &local); +#endif + m_Paused = UgcThrottle::InHours(local.tm_hour, limits.pauseFromHour, limits.pauseToHour); + if (m_Paused) return; // Enough to keep every worker busy until the next poll const size_t wanted = std::max(m_Threads.size() * 2, m_Config.pollBatch); if (queued + active >= wanted) return; @@ -105,11 +216,16 @@ void UgcProcessor::Poll() { std::vector jobs; for (auto& model : Database::Get()->GetUgcModelsToProcess(limit)) { if (m_InFlight.contains({ Kind::MODEL, model.id })) continue; - jobs.push_back(Job{ Kind::MODEL, model.id, model.attempts, std::move(model.lxfml) }); + Job job{ Kind::MODEL, model.id, model.attempts, UgcJobs::LxfmlFromBlob(model.lxfml) }; + if (job.blob.empty()) job.blob = std::move(model.lxfml); // the worker reports it can't be read + job.parts = UgcJobs::CountParts(job.blob); + job.memory = UgcJobs::EstimateMemory(job.parts, settings); + jobs.push_back(std::move(job)); } for (auto& build : Database::Get()->GetModularBuildsToProcess(limit)) { if (m_InFlight.contains({ Kind::MODULAR, build.id })) continue; Job job{ Kind::MODULAR, build.id, build.attempts }; + job.memory = UgcJobs::EstimateMemory(64, settings); std::string error; if (!UgcCdClient::GatherModular(build.modules, job.modular, error)) { // Nothing a worker could do: record it right away @@ -171,6 +287,7 @@ void UgcProcessor::Collect() { void UgcProcessor::Update() { Collect(); const auto now = std::chrono::steady_clock::now(); + if (now - m_CpuSampled >= std::chrono::seconds(2)) SampleUsage(); if (now >= m_NextPoll) { m_NextPoll = now + std::chrono::milliseconds(m_Config.pollIntervalMs); Poll(); @@ -220,7 +337,29 @@ nlohmann::json UgcProcessor::Status() const { status["queued"] = m_Jobs.size(); status["active"] = m_Active; } + Limits limits; + { + std::lock_guard lock(m_Mutex); + limits = m_Limits; + status["jobMemoryBytes"] = m_MemoryInUse; + status["memoryWaits"] = m_Waiting; + } status["workers"] = m_Threads.size(); + const auto throttle = UgcThrottle::GetStats(); + status["usage"] = { + { "cpuPercent", std::lround(m_CpuPercent) }, // of one core + { "cores", std::thread::hardware_concurrency() }, + { "residentBytes", ResidentBytes() }, + { "throttled", Throttled() }, + { "throttledMs", throttle.sleptMs }, + { "paused", m_Paused }, + }; + status["limits"] = { + { "maxCpus", limits.maxCpus }, + { "maxMemoryBytes", limits.maxMemoryBytes }, + { "nice", limits.nice }, + { "pauseHours", limits.pauseFromHour >= 0 ? std::to_string(limits.pauseFromHour) + "-" + std::to_string(limits.pauseToHour) : "" }, + }; status["made"] = m_Made; status["failed"] = m_Failed; status["evicted"] = m_Evicted; diff --git a/dUgcServer/UgcProcessor.h b/dUgcServer/UgcProcessor.h index ca0f34cc8..93bb02adb 100644 --- a/dUgcServer/UgcProcessor.h +++ b/dUgcServer/UgcProcessor.h @@ -36,10 +36,24 @@ public: size_t threads{ 2 }; }; + /** + * What the workers may use; changed while running (Configure) when the settings are reloaded. + */ + struct Limits { + double maxCpus{}; // CPUs the workers may use together on average (UgcThrottle); 0: no limit + uint64_t maxMemoryBytes{}; // estimated memory of the jobs running at once; 0: no limit + int nice{}; // the workers' scheduling priority (Linux nice, 0 to 19) + int pauseFromHour{ -1 }; // local hours in which no new jobs start (from, to; -1: never) + int pauseToHour{ -1 }; + }; + UgcProcessor(Config config, UgcStorage& storage, UgcBricks::BrickLibrary& library, UgcJobs::Settings settings); ~UgcProcessor(); void Start(); + + // Main thread: new settings and limits (config reload) + void Configure(UgcJobs::Settings settings, Limits limits); // Waits for the jobs that are running; queued ones are dropped (they stay pending in the database) void Stop(); @@ -62,6 +76,11 @@ public: size_t Queued() const { std::lock_guard lock(m_Mutex); return m_Jobs.size(); } size_t Busy() const { std::lock_guard lock(m_Mutex); return m_Active; } size_t Threads() const { return m_Config.threads; } + // Main thread: the process's CPU use (percent of one core) and resident memory, and the running jobs' estimate + double CpuPercent() const { return m_CpuPercent; } + static uint64_t ResidentBytes(); + uint64_t JobMemory() const { std::lock_guard lock(m_Mutex); return m_MemoryInUse; } + bool Throttled() const; // Main thread: totals since the start and the files' size (traffic reports, then the dashboard and /metrics) uint64_t Made() const { return m_Made; } @@ -75,8 +94,10 @@ private: Kind kind{}; LWOOBJID id{}; uint32_t attempts{}; - std::string blob; // models: the stored LXFML + std::string blob; // models: the LXFML UgcJobs::ModularInput modular; // modular builds + uint64_t memory{}; // estimated bytes it needs + size_t parts{}; }; struct Done { @@ -92,17 +113,26 @@ private: void Collect(); void Record(const Done& done); void Worker(); + void SampleUsage(); Config m_Config; UgcStorage& m_Storage; UgcBricks::BrickLibrary& m_Library; - UgcJobs::Settings m_Settings; + UgcJobs::Settings m_Settings; // guarded by m_Mutex (workers copy it per job) + Limits m_Limits; // guarded by m_Mutex mutable std::mutex m_Mutex; std::condition_variable m_Wake; std::deque m_Jobs; std::deque m_Done; size_t m_Active{}; + uint64_t m_MemoryInUse{}; // estimates of the running jobs + uint64_t m_Waiting{}; // times a job waited for memory + bool m_Paused{}; // main thread: in the pause hours + // Main thread: process CPU use, measured between status reads + std::chrono::steady_clock::time_point m_CpuSampled{}; + double m_CpuSeconds{}; + double m_CpuPercent{}; bool m_Stopping{}; std::vector m_Threads; diff --git a/dUgcServer/UgcRender.cpp b/dUgcServer/UgcRender.cpp index a1dfd27a2..43d9c1f35 100644 --- a/dUgcServer/UgcRender.cpp +++ b/dUgcServer/UgcRender.cpp @@ -4,9 +4,13 @@ #include #include #include +#include #include +#include "UgcPalette.h" +#include "UgcThrottle.h" + namespace { constexpr float INF = std::numeric_limits::infinity(); @@ -37,8 +41,151 @@ namespace { } } - float ToLinear(float c) { return std::pow(std::clamp(c, 0.0f, 1.0f), 2.2f); } - float ToSrgb(float c) { return std::pow(std::clamp(c, 0.0f, 1.0f), 1.0f / 2.2f); } + float ToLinear(float c) { return UgcPalette::SrgbToLinear(std::clamp(c, 0.0f, 1.0f)); } + float ToSrgb(float c) { return UgcPalette::LinearToSrgb(std::clamp(c, 0.0f, 1.0f)); } + + // A bounding volume hierarchy over a mesh's triangles, for the occlusion rays + class Bvh { + public: + explicit Bvh(const UgcModel::Mesh& mesh) : m_Mesh(mesh) { + const size_t count = mesh.TriangleCount(); + m_Order.resize(count); + std::iota(m_Order.begin(), m_Order.end(), 0u); + m_Centers.resize(count); + for (size_t t = 0; t < count; t++) m_Centers[t] = (Vertex(t, 0) + Vertex(t, 1) + Vertex(t, 2)) / 3.0f; + if (count > 0) Build(0, static_cast(count)); + Flatten(); + } + + // Whether a ray from `origin` along `direction` (unit) hits a triangle nearer than `maxDistance` + bool Hits(const glm::vec3& origin, const glm::vec3& direction, float maxDistance) const { + if (m_Nodes.empty()) return false; + const glm::vec3 inverse(1.0f / (std::abs(direction.x) > 1e-12f ? direction.x : 1e-12f), 1.0f / (std::abs(direction.y) > 1e-12f ? direction.y : 1e-12f), + 1.0f / (std::abs(direction.z) > 1e-12f ? direction.z : 1e-12f)); + uint32_t stack[64]; + int top = 0; + stack[top++] = 0; + while (top > 0) { + const auto& node = m_Nodes[stack[--top]]; + if (!BoxHit(node, origin, inverse, maxDistance)) continue; + if (node.count > 0) { + for (uint32_t i = node.first; i < node.first + node.count; i++) { + if (TriangleHit(m_Triangles[i], origin, direction, maxDistance)) return true; + } + } else if (top < 62) { + stack[top++] = node.first; + stack[top++] = node.first + 1; + } + } + return false; + } + + private: + struct Node { + glm::vec3 min{}; + glm::vec3 max{}; + uint32_t first{}; // leaf: first triangle in m_Order; inner: the first of two children + uint32_t count{}; // triangles, 0 for inner nodes + }; + + glm::vec3 Vertex(size_t t, int k) const { return m_Mesh.positions[m_Mesh.indices[t * 3 + k]]; } + + void Build(uint32_t first, uint32_t count) { + // Iterative, so deep trees don't use the stack + struct Task { uint32_t node, first, count; }; + m_Nodes.push_back({}); + std::vector tasks{ { 0, first, count } }; + while (!tasks.empty()) { + const auto task = tasks.back(); + tasks.pop_back(); + Node node; + node.min = glm::vec3(INF); + node.max = glm::vec3(-INF); + glm::vec3 centerMin(INF), centerMax(-INF); + for (uint32_t i = task.first; i < task.first + task.count; i++) { + for (int k = 0; k < 3; k++) { + node.min = glm::min(node.min, Vertex(m_Order[i], k)); + node.max = glm::max(node.max, Vertex(m_Order[i], k)); + } + centerMin = glm::min(centerMin, m_Centers[m_Order[i]]); + centerMax = glm::max(centerMax, m_Centers[m_Order[i]]); + } + const auto extent = centerMax - centerMin; + const int axis = extent.x >= extent.y && extent.x >= extent.z ? 0 : extent.y >= extent.z ? 1 : 2; + if (task.count <= 4 || extent[axis] <= 0.0f) { + node.first = task.first; + node.count = task.count; + m_Nodes[task.node] = node; + continue; + } + const uint32_t half = task.count / 2; + auto* begin = m_Order.data() + task.first; + std::nth_element(begin, begin + half, begin + task.count, [&](uint32_t a, uint32_t b) { return m_Centers[a][axis] < m_Centers[b][axis]; }); + node.first = static_cast(m_Nodes.size()); + node.count = 0; + m_Nodes[task.node] = node; + m_Nodes.push_back({}); + m_Nodes.push_back({}); + tasks.push_back({ node.first, task.first, half }); + tasks.push_back({ node.first + 1, task.first + half, task.count - half }); + } + } + + static bool BoxHit(const Node& node, const glm::vec3& origin, const glm::vec3& inverse, float maxDistance) { + const auto t0 = (node.min - origin) * inverse; + const auto t1 = (node.max - origin) * inverse; + const auto near = glm::min(t0, t1), far = glm::max(t0, t1); + const float enter = std::max(std::max(near.x, near.y), std::max(near.z, 0.0f)); + const float exit = std::min(std::min(far.x, far.y), std::min(far.z, maxDistance)); + return enter <= exit; + } + + struct Triangle { + glm::vec3 a, e1, e2; + }; + + // The triangles in leaf order, edges worked out once (the rays read them far more often than the tree is built) + void Flatten() { + m_Triangles.reserve(m_Order.size()); + for (const auto t : m_Order) { + const auto a = Vertex(t, 0); + m_Triangles.push_back({ a, Vertex(t, 1) - a, Vertex(t, 2) - a }); + } + } + + static bool TriangleHit(const Triangle& triangle, const glm::vec3& origin, const glm::vec3& direction, float maxDistance) { + const auto& a = triangle.a; + const auto& e1 = triangle.e1; + const auto& e2 = triangle.e2; + const auto p = glm::cross(direction, e2); + const float det = glm::dot(e1, p); + if (std::abs(det) < 1e-12f) return false; + const float inv = 1.0f / det; + const auto s = origin - a; + const float u = glm::dot(s, p) * inv; + if (u < 0.0f || u > 1.0f) return false; + const auto q = glm::cross(s, e1); + const float v = glm::dot(direction, q) * inv; + if (v < 0.0f || u + v > 1.0f) return false; + const float distance = glm::dot(e2, q) * inv; + return distance > 1e-4f && distance < maxDistance; + } + + const UgcModel::Mesh& m_Mesh; + std::vector m_Triangles; + std::vector m_Order; + std::vector m_Centers; + std::vector m_Nodes; + }; + + float RadicalInverse(uint32_t bits) { + bits = (bits << 16u) | (bits >> 16u); + bits = ((bits & 0x55555555u) << 1u) | ((bits & 0xAAAAAAAAu) >> 1u); + bits = ((bits & 0x33333333u) << 2u) | ((bits & 0xCCCCCCCCu) >> 2u); + bits = ((bits & 0x0F0F0F0Fu) << 4u) | ((bits & 0xF0F0F0F0u) >> 4u); + bits = ((bits & 0x00FF00FFu) << 8u) | ((bits & 0xFF00FF00u) >> 8u); + return static_cast(bits) * 2.3283064365386963e-10f; + } // Looking at a sphere (center, radius) from direction `dir` (towards the viewer), square orthographic view struct OrthoView { @@ -102,7 +249,7 @@ namespace UgcRender { OptimizeResult result; auto& opaque = model.opaque; result.trianglesBefore = opaque.TriangleCount() + model.transparent.TriangleCount(); - if (model.Empty() || (!options.removeHidden && !options.bakeAo)) return result; + if (opaque.Empty() || !options.removeHidden) return result; glm::vec3 center{}; float radius{}; @@ -124,10 +271,11 @@ namespace UgcRender { } // Without normals nothing is culled if (opaque.normals.size() != opaque.positions.size()) std::fill(faceNormals.begin(), faceNormals.end(), glm::vec3(0.0f)); - std::vector aoOpaque(opaque.positions.size()), weightOpaque(opaque.positions.size()); - std::vector aoTransparent(model.transparent.positions.size()), weightTransparent(model.transparent.positions.size()); for (const auto& direction : SphereDirections()) { + // A ground plane under the model hides everything from below + if (options.groundPlane && direction.y < -0.05f) continue; + UgcThrottle::Checkpoint(); const OrthoView view(center, radius, direction, resolution); const float bias = view.PixelSize(); std::fill(depth.begin(), depth.end(), INF); @@ -138,6 +286,7 @@ namespace UgcRender { for (size_t t = 0; t < triangles; t++) facing[t] = glm::dot(faceNormals[t], direction) > -0.1f; for (size_t t = 0; t < triangles; t++) { if (!facing[t]) continue; + if ((t & 0x3FFF) == 0) UgcThrottle::Checkpoint(); const auto id = static_cast(t + 1); Rasterize(resolution, resolution, screen[opaque.indices[t * 3]], screen[opaque.indices[t * 3 + 1]], screen[opaque.indices[t * 3 + 2]], [&](int x, int y, float z, float, float, float) { @@ -152,93 +301,96 @@ namespace UgcRender { if (id != 0) visible[id - 1] = true; } - // Whether a point is in front of what was drawn around its pixel - const auto exposed = [&](const glm::vec3& point, float allowance, bool neighbours) { - const auto p = view.Project(point); - const int px = static_cast(std::floor(p.x)), py = static_cast(std::floor(p.y)); - const int reach = neighbours ? 1 : 0; - for (int dy = -reach; dy <= reach; dy++) { - for (int dx = -reach; dx <= reach; dx++) { - const int x = px + dx, y = py + dy; - if (x < 0 || y < 0 || x >= resolution || y >= resolution) return true; - if (p.z <= depth[static_cast(y) * resolution + x] + allowance) return true; - } - } - return false; - }; - // Triangles too small or thin to cover a pixel centre: kept when their centre isn't behind what was drawn. // Bigger ones that show would have covered one. - if (options.removeHidden) { - const float smallArea = 2.0f; // pixels - for (size_t t = 0; t < triangles; t++) { - if (visible[t] || !facing[t]) continue; - const auto& a = screen[opaque.indices[t * 3]]; - const auto& b = screen[opaque.indices[t * 3 + 1]]; - const auto& c = screen[opaque.indices[t * 3 + 2]]; - if (std::abs(Edge(a, b, c.x, c.y)) * 0.5f > smallArea) continue; - const auto centre = (a + b + c) / 3.0f; - const int x = static_cast(centre.x), y = static_cast(centre.y); - if (x < 0 || y < 0 || x >= resolution || y >= resolution || centre.z <= depth[static_cast(y) * resolution + x] + bias) visible[t] = true; - } - } - - if (options.bakeAo) { - const auto accumulate = [&](const UgcModel::Mesh& mesh, std::vector& ao, std::vector& weight) { - for (size_t v = 0; v < mesh.positions.size(); v++) { - const float w = v < mesh.normals.size() ? glm::dot(mesh.normals[v], direction) : 1.0f; - if (w <= 0.0f) continue; - weight[v] += w; - const auto normal = v < mesh.normals.size() ? mesh.normals[v] : glm::vec3(0.0f); - if (exposed(mesh.positions[v] + normal * bias, bias, false)) ao[v] += w; - } - }; - accumulate(opaque, aoOpaque, weightOpaque); - accumulate(model.transparent, aoTransparent, weightTransparent); - } - } - - if (options.bakeAo) { - const float strength = std::clamp(options.aoStrength, 0.0f, 1.0f); - const auto apply = [strength](UgcModel::Mesh& mesh, const std::vector& ao, const std::vector& weight) { - for (size_t v = 0; v < mesh.colors.size() && v < ao.size(); v++) { - if (weight[v] <= 0.0f) continue; - const float factor = 1.0f - strength * (1.0f - ao[v] / weight[v]); - auto& color = mesh.colors[v]; - color.r = ToSrgb(ToLinear(color.r) * factor); - color.g = ToSrgb(ToLinear(color.g) * factor); - color.b = ToSrgb(ToLinear(color.b) * factor); - } - }; - apply(opaque, aoOpaque, weightOpaque); - apply(model.transparent, aoTransparent, weightTransparent); - } - - if (options.removeHidden) { - UgcModel::Mesh kept; - std::vector remap(opaque.positions.size(), UINT32_MAX); + const float smallArea = 2.0f; // pixels for (size_t t = 0; t < triangles; t++) { - if (!visible[t]) { - result.trianglesRemoved++; - continue; - } - for (int k = 0; k < 3; k++) { - const auto source = opaque.indices[t * 3 + k]; - if (remap[source] == UINT32_MAX) { - remap[source] = static_cast(kept.positions.size()); - kept.positions.push_back(opaque.positions[source]); - if (source < opaque.normals.size()) kept.normals.push_back(opaque.normals[source]); - if (source < opaque.colors.size()) kept.colors.push_back(opaque.colors[source]); - } - kept.indices.push_back(remap[source]); - } + if (visible[t] || !facing[t]) continue; + const auto& a = screen[opaque.indices[t * 3]]; + const auto& b = screen[opaque.indices[t * 3 + 1]]; + const auto& c = screen[opaque.indices[t * 3 + 2]]; + if (std::abs(Edge(a, b, c.x, c.y)) * 0.5f > smallArea) continue; + const auto centre = (a + b + c) / 3.0f; + const int x = static_cast(centre.x), y = static_cast(centre.y); + if (x < 0 || y < 0 || x >= resolution || y >= resolution || centre.z <= depth[static_cast(y) * resolution + x] + bias) visible[t] = true; } - opaque = std::move(kept); } + + for (size_t t = 0; t < triangles; t++) result.trianglesRemoved += visible[t] ? 0 : 1; + result.kept = visible; + UgcModel::KeepTriangles(opaque, visible); return result; } - Image RenderIcon(const UgcModel::Model& source, const IconOptions& options) { + std::vector AmbientOcclusion(const UgcModel::Mesh& mesh, const UgcModel::Mesh& occluders, float distance, int samples) { + std::vector ao(mesh.positions.size(), 1.0f); + if (occluders.Empty() || samples <= 0 || distance <= 0.0f || mesh.normals.size() != mesh.positions.size()) return ao; + const Bvh bvh(occluders); + const auto count = static_cast(samples); + // Vertices at the same place facing the same way (bricks' shared corners) are worked out once + struct Key { + int32_t p[3], n[3]; + bool operator==(const Key& o) const { return std::equal(p, p + 3, o.p) && std::equal(n, n + 3, o.n); } + }; + struct KeyHash { + size_t operator()(const Key& k) const { + size_t h = 1469598103934665603ull; + for (int i = 0; i < 3; i++) h = (h ^ static_cast(k.p[i])) * 1099511628211ull ^ static_cast(k.n[i]) * 0x9E3779B97F4A7C15ull; + return h; + } + }; + std::unordered_map known; + known.reserve(mesh.positions.size()); + for (size_t v = 0; v < mesh.positions.size(); v++) { + if ((v & 0xFF) == 0) UgcThrottle::Checkpoint(); + const auto& normal = mesh.normals[v]; + const Key key{ { static_cast(std::lround(mesh.positions[v].x * 1000.0f)), static_cast(std::lround(mesh.positions[v].y * 1000.0f)), + static_cast(std::lround(mesh.positions[v].z * 1000.0f)) }, { static_cast(std::lround(normal.x * 100.0f)), + static_cast(std::lround(normal.y * 100.0f)), static_cast(std::lround(normal.z * 100.0f)) } }; + if (const auto it = known.find(key); it != known.end()) { + ao[v] = it->second; + continue; + } + if (glm::dot(normal, normal) < 0.5f) continue; + // A frame around the normal + const glm::vec3 helper = std::abs(normal.x) < 0.9f ? glm::vec3(1, 0, 0) : glm::vec3(0, 1, 0); + const auto tangent = glm::normalize(glm::cross(helper, normal)); + const auto bitangent = glm::cross(normal, tangent); + // Hammersley points, turned by an amount of the vertex's own (fixed) so neighbours don't band + const float turn = static_cast((v * 0x9E3779B9u) >> 8 & 0xFFFFFF) / 16777216.0f; + const auto origin = mesh.positions[v] + normal * 1e-3f; + uint32_t open = 0; + for (uint32_t i = 0; i < count; i++) { + const float u = (i + 0.5f) / static_cast(count); + const float phi = 2.0f * 3.14159265f * std::fmod(RadicalInverse(i) + turn, 1.0f); + const float r = std::sqrt(u), z = std::sqrt(std::max(0.0f, 1.0f - u)); + const auto direction = tangent * (r * std::cos(phi)) + bitangent * (r * std::sin(phi)) + normal * z; + if (!bvh.Hits(origin, direction, distance)) open++; + } + ao[v] = static_cast(open) / static_cast(count); + known.emplace(key, ao[v]); + } + return ao; + } + + std::vector BakeAo(UgcModel::Model& model, const AoOptions& options) { + auto& opaque = model.opaque; + if (!options.enabled || opaque.Empty()) return {}; + auto ao = AmbientOcclusion(opaque, opaque, options.distance, options.samples); + const float strength = std::clamp(options.strength, 0.0f, 1.0f); + for (size_t v = 0; v < opaque.colors.size() && v < ao.size(); v++) { + glm::vec3 lit(1.0f - strength * (1.0f - ao[v])); + if (v < opaque.glow.size()) lit += opaque.glow[v] * options.glowStrength; + lit = glm::clamp(lit, 0.0f, 1.0f); + auto& color = opaque.colors[v]; + color.r = ToSrgb(ToLinear(color.r) * lit.r); + color.g = ToSrgb(ToLinear(color.g) * lit.g); + color.b = ToSrgb(ToLinear(color.b) * lit.b); + } + return ao; + } + + Image RenderIcon(const UgcModel::Model& source, const IconOptions& options, const std::vector* opaqueAo) { const int size = std::clamp(options.size, 8, 1024); const int supersample = std::clamp(options.supersample, 1, 8); const int n = size * supersample; @@ -284,8 +436,47 @@ namespace UgcRender { // Linear, premultiplied std::vector color(static_cast(n) * n, glm::vec4(0.0f)); std::vector depth(static_cast(n) * n, INF); - const glm::vec3 light = glm::normalize(dir + glm::vec3(-0.35f, 1.1f, 0.25f)); - const auto shade = [&](const UgcModel::Mesh& mesh, uint32_t i0, uint32_t i1, uint32_t i2, float w0, float w1, float w2) { + const float sunYaw = glm::radians(options.sunYawDegrees), sunPitch = glm::radians(options.sunPitchDegrees); + const glm::vec3 light = glm::normalize(glm::vec3(std::sin(sunYaw) * std::cos(sunPitch), std::sin(sunPitch), std::cos(sunYaw) * std::cos(sunPitch))); + + // Ambient occlusion darkens the world light (opaque bricks only, as they are what occludes) + std::vector ao; + if (options.ao.enabled) { + ao = opaqueAo && opaqueAo->size() == model.opaque.positions.size() ? *opaqueAo : AmbientOcclusion(model.opaque, model.opaque, options.ao.distance, options.ao.samples); + } + + // The sun's shadows: a depth map seen from the sun, looked up with a few taps for the sun's soft edge + const int shadowSize = 1024; + const OrthoView sunView(center, radius * 1.05f, light, shadowSize); + std::vector shadowDepth; + if (options.shadows) { + shadowDepth.assign(static_cast(shadowSize) * shadowSize, INF); + const auto& mesh = model.opaque; + std::vector screen(mesh.positions.size()); + for (size_t v = 0; v < mesh.positions.size(); v++) screen[v] = sunView.Project(mesh.positions[v]); + for (size_t i = 0; i + 2 < mesh.indices.size(); i += 3) { + if ((i & 0xFFFF) == 0) UgcThrottle::Checkpoint(); + Rasterize(shadowSize, shadowSize, screen[mesh.indices[i]], screen[mesh.indices[i + 1]], screen[mesh.indices[i + 2]], [&](int x, int y, float z, float, float, float) { + auto& stored = shadowDepth[static_cast(y) * shadowSize + x]; + stored = std::min(stored, z); + }); + } + } + const float shadowBias = sunView.PixelSize() * 2.0f; + const auto sunlit = [&](const glm::vec3& position) { + if (shadowDepth.empty()) return 1.0f; + const auto p = sunView.Project(position); + float lit = 0.0f; + for (int dy = -1; dy <= 1; dy++) { + for (int dx = -1; dx <= 1; dx++) { + const int x = static_cast(p.x) + dx, y = static_cast(p.y) + dy; + if (x < 0 || y < 0 || x >= shadowSize || y >= shadowSize || p.z <= shadowDepth[static_cast(y) * shadowSize + x] + shadowBias) lit += 1.0f; + } + } + return lit / 9.0f; + }; + + const auto shade = [&](const UgcModel::Mesh& mesh, bool isOpaque, uint32_t i0, uint32_t i1, uint32_t i2, float w0, float w1, float w2) { glm::vec3 normal(0.0f, 1.0f, 0.0f); if (mesh.normals.size() == mesh.positions.size()) { normal = mesh.normals[i0] * w0 + mesh.normals[i1] * w1 + mesh.normals[i2] * w2; @@ -295,7 +486,13 @@ namespace UgcRender { } glm::vec4 base(0.63f, 0.63f, 0.63f, 1.0f); if (mesh.colors.size() == mesh.positions.size()) base = mesh.colors[i0] * w0 + mesh.colors[i1] * w1 + mesh.colors[i2] * w2; - const float lighting = 0.55f + 0.6f * std::max(0.0f, glm::dot(normal, light)); + const float occlusion = isOpaque && ao.size() == mesh.positions.size() ? ao[i0] * w0 + ao[i1] * w1 + ao[i2] * w2 : 1.0f; + const float strength = std::clamp(options.ao.strength, 0.0f, 1.0f); + const auto position = mesh.positions[i0] * w0 + mesh.positions[i1] * w1 + mesh.positions[i2] * w2; + const float direct = std::max(0.0f, glm::dot(normal, light)); + // Diffuse: the world's light (radiance `ambient`) and the sun's (irradiance `sunStrength`, over pi) + const float lighting = options.ambient * (1.0f - strength * (1.0f - occlusion)) + + options.sunStrength / 3.14159265f * direct * (direct > 0.0f ? sunlit(position + normal * shadowBias) : 0.0f); return glm::vec4(ToLinear(base.r) * lighting, ToLinear(base.g) * lighting, ToLinear(base.b) * lighting, std::clamp(base.a, 0.0f, 1.0f)); }; @@ -310,7 +507,7 @@ namespace UgcRender { const size_t index = static_cast(y) * n + x; if (z >= depth[index]) return; depth[index] = z; - const auto shaded = shade(mesh, i0, i1, i2, w0, w1, w2); + const auto shaded = shade(mesh, true, i0, i1, i2, w0, w1, w2); color[index] = glm::vec4(glm::vec3(shaded), 1.0f); }); } @@ -330,7 +527,7 @@ namespace UgcRender { Rasterize(n, n, screen[i0], screen[i1], screen[i2], [&](int x, int y, float z, float w0, float w1, float w2) { const size_t index = static_cast(y) * n + x; if (z >= depth[index]) return; - const auto shaded = shade(mesh, i0, i1, i2, w0, w1, w2); + const auto shaded = shade(mesh, false, i0, i1, i2, w0, w1, w2); const float alpha = shaded.a; auto& target = color[index]; target = glm::vec4(glm::vec3(shaded) * alpha + glm::vec3(target) * (1.0f - alpha), alpha + target.a * (1.0f - alpha)); diff --git a/dUgcServer/UgcRender.h b/dUgcServer/UgcRender.h index 222115e4d..c2ec87ecb 100644 --- a/dUgcServer/UgcRender.h +++ b/dUgcServer/UgcRender.h @@ -19,38 +19,71 @@ namespace UgcRender { std::vector rgba; }; + struct AoOptions { + bool enabled{ true }; + float distance{ 5.0f }; // LU Toolbox's AO distance (Bake Lighting, AO Only) + int samples{ 64 }; // rays per vertex (AO Samples) + float strength{ 1.0f }; // 0 leaves the colors, 1 is the full bake + float glowStrength{ 6.0f }; // what glowing colors add to the light (Glow Strength 3 x Glow Multiplier 2) + }; + + /** + * The icon, set up like LU Toolbox's icon renderer (the BrickBuild scene of its UGC render add-on): a 50 mm lens + * (39.6 degree field of view) looking from 53.4 degrees around and 19.5 above, framed at 1.03, a sun of strength + * 2.5 from 21 around and 50.3 above with soft shadows, and a grey (0.192) world light, darkened by ambient occlusion. + */ struct IconOptions { int size{ 128 }; int supersample{ 4 }; - float yawDegrees{ 35.0f }; // camera around the model, from +Z towards +X - float pitchDegrees{ 25.0f }; // camera above the model - float fovDegrees{ 30.0f }; + float yawDegrees{ 53.36f }; // camera around the model, from +Z towards +X + float pitchDegrees{ 19.54f }; // camera above the model + float fovDegrees{ 39.6f }; float margin{ 1.03f }; // 1 fills the icon, more leaves a border glm::mat4 modelRotation{ 1.0f }; // applied to the model before the camera looks at it + float sunYawDegrees{ 21.0f }; + float sunPitchDegrees{ 50.3f }; + float sunStrength{ 2.5f }; + float ambient{ 0.192f }; + bool shadows{ true }; + AoOptions ao{ true, 5.0f, 32, 1.0f, 0.0f }; }; - // The model drawn from the icon's camera, framed to fit, on a transparent background - Image RenderIcon(const UgcModel::Model& model, const IconOptions& options); + // The model drawn from the icon's camera, framed to fit, on a transparent background. `opaqueAo`: the opaque mesh's + // ambient occlusion (AmbientOcclusion) when it is known already, else it is worked out when options.ao wants it. + Image RenderIcon(const UgcModel::Model& model, const IconOptions& options, const std::vector* opaqueAo = nullptr); struct OptimizeOptions { int resolution{ 1024 }; // of each direction's render bool removeHidden{ true }; - bool bakeAo{ true }; - float aoStrength{ 0.6f }; // 0 leaves the colors, 1 makes fully hidden corners black + bool groundPlane{ false }; // LU Toolbox's Use Ground Plane: nothing is seen from below the model }; struct OptimizeResult { size_t trianglesBefore{}; size_t trianglesRemoved{}; + std::vector kept; // per opaque triangle before: whether it stayed (for UgcModel::KeepTriangles) }; /** - * Renders the opaque mesh from 42 directions around it: triangles that show in none of them are removed (with a - * conservative test, so small visible ones stay), and each vertex's share of the directions it can be seen from - * darkens its color (ambient occlusion). Transparent bricks hide nothing but are darkened too. + * Hidden surface removal: renders the opaque mesh from 42 directions around it and removes the triangles that show + * in none of them (with a conservative test, so small visible ones stay). Transparent bricks hide nothing and + * aren't touched, as in LU Toolbox. */ OptimizeResult Optimize(UgcModel::Model& model, const OptimizeOptions& options); + /** + * Ambient occlusion of each vertex of `mesh`: the share of `samples` rays (cosine weighted around the vertex + * normal, the same pattern every time) that leave without hitting a triangle of `occluders` within `distance`. + * 1 is open, 0 fully hidden. + */ + std::vector AmbientOcclusion(const UgcModel::Mesh& mesh, const UgcModel::Mesh& occluders, float distance, int samples); + + /** + * LU Toolbox's Bake Lighting with AO Only (its defaults): the opaque mesh's occlusion (transparent bricks are hidden + * while baking and not baked) plus the glow colors, multiplied into the vertex colors. + */ + std::vector BakeAo(UgcModel::Model& model, const AoOptions& options); // the occlusion used, per opaque vertex + // The 42 directions Optimize renders from (an icosahedron's corners and edge centres), unit length std::vector SphereDirections(); } diff --git a/dUgcServer/UgcServer.cpp b/dUgcServer/UgcServer.cpp index 9b8c6686f..1e5b5ec16 100644 --- a/dUgcServer/UgcServer.cpp +++ b/dUgcServer/UgcServer.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include "BinaryPathFinder.h" @@ -33,6 +35,7 @@ #include "UgcModel.h" #include "UgcProcessor.h" #include "UgcStorage.h" +#include "UgcThrottle.h" namespace Game { Logger* logger = nullptr; @@ -52,22 +55,70 @@ namespace { return GeneralUtils::TryParse(Game::config->GetValue(key)).value_or(fallback); } + std::vector ParseLods(const std::string& text) { + std::vector lods; + std::stringstream stream(text); + std::string item; + while (std::getline(stream, item, ',')) { + std::erase_if(item, [](unsigned char c) { return std::isspace(c); }); + if (const auto lod = GeneralUtils::TryParse(item); lod && *lod <= 3) lods.push_back(*lod); + } + return lods; + } + + // ugcconfig.ini (and the dashboard's settings); the defaults are LU Toolbox's UgcJobs::Settings ReadSettings() { UgcJobs::Settings settings; + settings.build.palette = Game::config->GetValue("color_palette") == "brickdb" ? UgcModel::ePalette::BRICKDB : UgcModel::ePalette::LU_TOOLBOX; + settings.build.colorVariation = std::clamp(Setting("color_variation", 5.0f), 0.0f, 100.0f); + settings.build.transparentOpacity = std::clamp(Setting("transparent_opacity", 58.82f), 0.0f, 100.0f); + const auto lods = ParseLods(Game::config->GetValue("lods").empty() ? "0,2" : Game::config->GetValue("lods")); + if (!lods.empty()) settings.lods = lods; + settings.lodDistances.lod0 = Setting("lod_distance_0", 0.0f); + settings.lodDistances.lod1 = Setting("lod_distance_1", 50.0f); + settings.lodDistances.lod2 = Setting("lod_distance_2", 100.0f); + settings.lodDistances.lod3 = Setting("lod_distance_3", 280.0f); + settings.lodDistances.cull = Setting("lod_cull", 10000.0f); + if (!Game::config->GetValue("shader_opaque").empty()) settings.shaderOpaque = Game::config->GetValue("shader_opaque"); settings.optimize.removeHidden = Setting("remove_hidden_faces", 1) != 0; - settings.optimize.bakeAo = Setting("bake_ao", 1) != 0; - settings.optimize.aoStrength = Setting("ao_strength", 0.6f); + settings.optimize.groundPlane = Setting("hsr_ground_plane", 0) != 0; settings.optimize.resolution = Setting("optimize_resolution", 1024); + settings.ao.enabled = Setting("bake_ao", 1) != 0; + settings.ao.distance = Setting("ao_distance", 5.0f); + settings.ao.samples = std::clamp(Setting("ao_samples", 64), 1, 1024); + settings.ao.strength = Setting("ao_strength", 1.0f); + settings.ao.glowStrength = Setting("glow_strength", 6.0f); settings.icon.size = Setting("icon_size", 128); - settings.icon.yawDegrees = Setting("icon_yaw", 35.0f); - settings.icon.pitchDegrees = Setting("icon_pitch", 25.0f); + settings.icon.yawDegrees = Setting("icon_yaw", 53.36f); + settings.icon.pitchDegrees = Setting("icon_pitch", 19.54f); + settings.icon.fovDegrees = Setting("icon_fov", 39.6f); settings.icon.margin = Setting("icon_margin", 1.03f); + settings.icon.sunYawDegrees = Setting("icon_sun_yaw", 21.0f); + settings.icon.sunPitchDegrees = Setting("icon_sun_pitch", 50.3f); + settings.icon.sunStrength = Setting("icon_sun_strength", 2.5f); + settings.icon.ambient = Setting("icon_ambient", 0.192f); + settings.icon.shadows = Setting("icon_shadows", 1) != 0; + settings.icon.ao.enabled = Setting("icon_ao", 1) != 0; + settings.icon.ao.distance = settings.ao.distance; + settings.iconCorrectColors = Setting("icon_correct_colors", 1) != 0; + settings.iconColorVariation = std::clamp(Setting("icon_color_variation", 0.0f), 0.0f, 100.0f); settings.modularIcon = settings.icon; - settings.modularIcon.yawDegrees = Setting("modular_icon_yaw", 35.0f); - settings.modularIcon.pitchDegrees = Setting("modular_icon_pitch", 20.0f); + settings.modularIcon.yawDegrees = Setting("modular_icon_yaw", 53.36f); + settings.modularIcon.pitchDegrees = Setting("modular_icon_pitch", 19.54f); + settings.maxBricks = Setting("max_model_bricks", 0); return settings; } + UgcProcessor::Limits ReadLimits() { + UgcProcessor::Limits limits; + const auto cores = std::max(std::thread::hardware_concurrency(), 1); + limits.maxCpus = std::clamp(Setting("max_cpu_percent", 0.0), 0.0, 100.0) / 100.0 * cores; + limits.maxMemoryBytes = Setting("max_memory_mb", 0) * 1024 * 1024; + limits.nice = std::clamp(Setting("worker_nice", 0), 0, 19); + UgcThrottle::ParseHours(Game::config->GetValue("pause_hours"), limits.pauseFromHour, limits.pauseToHour); + return limits; + } + std::vector Segments(const std::string& path) { std::vector segments; size_t start = 0; @@ -187,9 +238,17 @@ namespace { const auto id = GeneralUtils::TryParse(segments[2]); const auto kind = segments[1] == "model" ? std::optional(UgcStorage::Kind::MODEL) : segments[1] == "modular" ? std::optional(UgcStorage::Kind::MODULAR) : std::nullopt; const auto& name = segments[3]; - if (!id || !kind || (name != "icon.png" && name != "model.nif")) return; + static const std::set PREVIEWS = { "icon.png", "model.nif", "model.noao.nif", "stats.json", + "previous.icon.png", "previous.model.nif", "previous.model.noao.nif", "previous.stats.json" }; + if (!id || !kind || !PREVIEWS.contains(name)) return; reply.headers.clear(); - ServeFile(reply, *kind, static_cast(*id), name, name == "icon.png" ? eContentType::IMAGE_PNG : eContentType::APPLICATION_OCTET_STREAM, true); + const auto type = name.ends_with(".png") ? eContentType::IMAGE_PNG : name.ends_with(".json") ? eContentType::APPLICATION_JSON : eContentType::APPLICATION_OCTET_STREAM; + ServeFile(reply, *kind, static_cast(*id), name, type, true); + // Made again since: the newest files differ, so they mustn't be cached as long as the client's + if (reply.status == eHTTPStatusCode::OK) { + std::erase_if(reply.headers, [](const std::string& header) { return header.starts_with("Cache-Control"); }); + reply.headers.push_back("Cache-Control: no-cache"); + } } }); Game::web.RegisterHTTPRoute({ .path = "/status", .method = eHTTPMethod::GET, .middleware = {}, @@ -212,7 +271,7 @@ namespace { int MakeFromCommandLine(const std::string& mode, const std::string& input, const std::filesystem::path& output) { const auto res = ResPath(); const auto settings = ReadSettings(); - UgcBricks::BrickLibrary library(res, Setting("brick_lod", 0)); + UgcBricks::BrickLibrary library(res, 0); if (!library.LoadMaterials()) std::cerr << "Couldn't read Materials.xml from " << (res / "brickdb.zip") << "; bricks will be grey\n"; UgcJobs::Outcome outcome; const auto start = std::chrono::steady_clock::now(); @@ -309,7 +368,7 @@ int main(int argc, char** argv) { ServiceType::UGC, Game::config, &Game::lastSignal, masterPassword); Game::server = g_Server; - UgcBricks::BrickLibrary library(res, Setting("brick_lod", 0)); + UgcBricks::BrickLibrary library(res, 0); if (!library.LoadMaterials()) LOG("Couldn't read Materials.xml from %s; bricks will be grey", (res / "brickdb.zip").string().c_str()); auto outputDir = std::filesystem::path(Game::config->GetValue("ugc_output_dir").empty() ? "ugc" : Game::config->GetValue("ugc_output_dir")); @@ -325,6 +384,7 @@ int main(int argc, char** argv) { const auto threads = Setting("worker_threads", 0); processorConfig.threads = threads > 0 ? threads : std::max(std::thread::hardware_concurrency() / 2, 1); UgcProcessor processor(processorConfig, storage, library, ReadSettings()); + processor.Configure(ReadSettings(), ReadLimits()); g_Processor = &processor; // Sent with the traffic reports to the dashboard (Diagnostics) TrafficStats::Local().SetGauge("workers_busy", [&processor] { return static_cast(processor.Busy()); }); @@ -339,6 +399,10 @@ int main(int argc, char** argv) { { "ugc_max_storage_bytes", [&processor] { return static_cast(processor.MaxStorageBytes()); } }, }; for (const auto& [name, gauge] : ugcGauges) TrafficStats::Local().SetGauge(name, gauge); + TrafficStats::Local().SetGauge("cpu_percent", [&processor] { return processor.CpuPercent(); }); + TrafficStats::Local().SetGauge("memory_mb", [] { return static_cast(UgcProcessor::ResidentBytes()) / (1024.0 * 1024.0); }); + TrafficStats::Local().SetGauge("job_memory_mb", [&processor] { return static_cast(processor.JobMemory()) / (1024.0 * 1024.0); }); + TrafficStats::Local().SetGauge("throttled", [&processor] { return processor.Throttled() ? 1.0 : 0.0; }); const auto listenIp = Game::config->GetValue("listen_ip").empty() ? std::string("0.0.0.0") : Game::config->GetValue("listen_ip"); const auto port = Setting("port", 2008); @@ -351,6 +415,7 @@ int main(int argc, char** argv) { LOG("UGC Server started on %s:%u, files in %s", listenIp.c_str(), port, outputDir.string().c_str()); auto lastTick = std::chrono::steady_clock::now(); + auto lastConfigure = lastTick; while (!Game::ShouldShutdown()) { // The poll's wait for network traffic is also this loop's sleep Game::web.ReceiveRequests(10); @@ -364,6 +429,11 @@ int main(int argc, char** argv) { packet = g_Server->ReceiveFromMaster(); } processor.Update(); + // Settings the dashboard changed arrive as a config reload; pick them up + if (now - lastConfigure >= std::chrono::seconds(5)) { + lastConfigure = now; + processor.Configure(ReadSettings(), ReadLimits()); + } } LOG("Stopping the UGC server"); @@ -372,6 +442,7 @@ int main(int argc, char** argv) { TrafficStats::Local().SetGauge("workers_queued", nullptr); TrafficStats::Local().SetGauge("workers_threads", nullptr); for (const auto& [name, gauge] : ugcGauges) TrafficStats::Local().SetGauge(name, nullptr); + for (const auto* gauge : { "cpu_percent", "memory_mb", "job_memory_mb", "throttled" }) TrafficStats::Local().SetGauge(gauge, nullptr); Game::web.Shutdown(); g_Processor = nullptr; Database::Destroy("UgcServer"); diff --git a/dUgcServer/UgcStorage.cpp b/dUgcServer/UgcStorage.cpp index 0aaa4e83f..e3cfd1342 100644 --- a/dUgcServer/UgcStorage.cpp +++ b/dUgcServer/UgcStorage.cpp @@ -10,8 +10,13 @@ namespace { "model.nif", "model.nif.gz", "model.nif.checksum", "model.lxfml.gz", "model.lxfml.checksum", "icon.dds.gz", "icon.dds.checksum", "icon.png", + "model.noao.nif", "stats.json", + "previous.icon.png", "previous.model.nif", "previous.model.noao.nif", "previous.stats.json", }; + // What of the files made before is kept when an item is made again, for comparing (as previous.) + constexpr std::array KEPT_FILES = { "icon.png", "model.nif", "model.noao.nif", "stats.json" }; + const char* KindFolder(UgcStorage::Kind kind) { return kind == UgcStorage::Kind::MODEL ? "models" : "modular"; } @@ -61,6 +66,14 @@ std::optional UgcStorage::Write(Kind kind, LWOOBJID id, const Files& f } bytes += data.size(); } + // Keep the last version's previews to compare with + for (const auto* name : KEPT_FILES) { + if (files.contains(name)) { + std::filesystem::copy_file(folder / name, temporary / (std::string("previous.") + name), std::filesystem::copy_options::overwrite_existing, code); + if (!code) bytes += std::filesystem::file_size(temporary / (std::string("previous.") + name), code); + code.clear(); + } + } // Swap the old folder out and the new one in; the old one is deleted after const auto old = folder.parent_path() / (".old-" + folder.filename().string() + "-" + RandomSuffix()); const bool hadOld = std::filesystem::exists(folder, code); diff --git a/dUgcServer/UgcThrottle.cpp b/dUgcServer/UgcThrottle.cpp new file mode 100644 index 000000000..58c95acbd --- /dev/null +++ b/dUgcServer/UgcThrottle.cpp @@ -0,0 +1,110 @@ +#include "UgcThrottle.h" + +#include +#include +#include +#include +#include +#include + +namespace { + // Up to this much CPU time may be used ahead of the budget (so short jobs aren't slowed at all) + constexpr double BURST_SECONDS = 0.25; + // Checkpoints closer together than this only read the clock + constexpr double MIN_ACCOUNT_SECONDS = 0.005; + + std::atomic g_Budget{ 0.0 }; + std::mutex g_Mutex; + double g_Balance = BURST_SECONDS; // CPU seconds that may still be used + std::chrono::steady_clock::time_point g_Refilled = std::chrono::steady_clock::now(); + std::atomic g_SleptMs{ 0 }; + std::atomic g_LastSleep{ 0 }; + + thread_local double t_LastCpu = -1.0; + + int64_t UnixMs() { + return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + } + + void Refill(double budget) { + const auto now = std::chrono::steady_clock::now(); + const double elapsed = std::chrono::duration(now - g_Refilled).count(); + g_Refilled = now; + g_Balance = std::min(g_Balance + elapsed * budget, BURST_SECONDS * std::max(budget, 1.0)); + } +} + +namespace UgcThrottle { + void SetBudget(double cpus) { + std::lock_guard lock(g_Mutex); + g_Budget = std::max(cpus, 0.0); + g_Balance = std::min(g_Balance, BURST_SECONDS); + g_Refilled = std::chrono::steady_clock::now(); + } + + double GetBudget() { + return g_Budget; + } + + double ThreadCpuSeconds() { +#if defined(CLOCK_THREAD_CPUTIME_ID) + timespec ts{}; + if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts) == 0) return static_cast(ts.tv_sec) + ts.tv_nsec / 1e9; +#endif + return std::chrono::duration(std::chrono::steady_clock::now().time_since_epoch()).count(); + } + + void Begin() { + t_LastCpu = ThreadCpuSeconds(); + } + + void Checkpoint() { + const double budget = g_Budget; + if (budget <= 0.0) return; + const double cpu = ThreadCpuSeconds(); + if (t_LastCpu < 0.0) t_LastCpu = cpu; + const double used = cpu - t_LastCpu; + if (used < MIN_ACCOUNT_SECONDS) return; + t_LastCpu = cpu; + + double wait = 0.0; + { + std::lock_guard lock(g_Mutex); + Refill(budget); + g_Balance -= used; + // Overdrawn: wait until the budget has paid it back. Other threads waiting meanwhile each owe their own + // share, so the waits add up to what the budget allows. + if (g_Balance < 0.0) wait = -g_Balance / budget; + } + if (wait <= 0.0) return; + wait = std::min(wait, 5.0); + g_SleptMs += static_cast(wait * 1000.0); + g_LastSleep = UnixMs(); + std::this_thread::sleep_for(std::chrono::duration(wait)); + // Time asleep costs no CPU; don't count this call's own bookkeeping twice + t_LastCpu = ThreadCpuSeconds(); + } + + Stats GetStats() { + return { g_SleptMs.load(), g_LastSleep.load() }; + } + + bool ParseHours(const std::string& text, int& from, int& to) { + const auto dash = text.find('-'); + if (dash == std::string::npos) return false; + try { + const int a = std::stoi(text.substr(0, dash)), b = std::stoi(text.substr(dash + 1)); + if (a < 0 || a > 23 || b < 0 || b > 23) return false; + from = a; + to = b; + return true; + } catch (...) { + return false; + } + } + + bool InHours(int hour, int from, int to) { + if (from < 0 || to < 0) return false; + return from <= to ? hour >= from && hour <= to : hour >= from || hour <= to; + } +} diff --git a/dUgcServer/UgcThrottle.h b/dUgcServer/UgcThrottle.h new file mode 100644 index 000000000..634519149 --- /dev/null +++ b/dUgcServer/UgcThrottle.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include + +/** + * Keeps the UGC workers' CPU use under a budget. The long loops (renders, ambient occlusion, icons) call Checkpoint + * every few milliseconds of work; it adds the calling thread's CPU time since its last call to a shared account that + * the budget fills at `budget` CPUs per second, and sleeps while the account is overdrawn. So the workers together + * average at most the budget however many there are, long jobs included, and without a budget it costs nothing. + */ +namespace UgcThrottle { + // CPUs the workers may use together on average (1.5: one and a half cores); 0 or less: no limit + void SetBudget(double cpus); + double GetBudget(); + + // Account the calling thread's CPU time and sleep when over the budget + void Checkpoint(); + + // Start accounting on this thread from now (a worker starting a job), so time spent idle isn't counted + void Begin(); + + struct Stats { + uint64_t sleptMs{}; // since start + int64_t lastSleepUnixMs{}; // when it last slept, 0 never + }; + Stats GetStats(); + + // This thread's CPU time in seconds + double ThreadCpuSeconds(); + + // Parses "18-23" (from 18:00 to 23:59; may wrap past midnight, "22-6") into from and to; false when it isn't that + bool ParseHours(const std::string& text, int& from, int& to); + // Whether `hour` (0-23) is inside the range; never when from or to is negative + bool InHours(int hour, int from, int to); +} diff --git a/resources/ugcconfig.ini b/resources/ugcconfig.ini index 286d82935..9c7ac289f 100644 --- a/resources/ugcconfig.ini +++ b/resources/ugcconfig.ini @@ -22,27 +22,76 @@ ugc_max_storage_mb=2048 # Worker threads making files (0: half the CPU cores) worker_threads=0 +# How much of the machine making files may use (changes apply while running): +# max_cpu_percent: the workers together average at most this share of all CPU cores (0: no limit); they pause +# between bits of work to stay under it, long renders included. +# worker_nice: the workers' scheduling priority on Linux, 0 (normal) to 19 (only when nothing else wants the CPU). +# max_memory_mb: jobs whose estimated memory would go past this together wait for running ones to finish; a job +# bigger than it alone runs when nothing else does (0: no limit). +# max_model_bricks: models with more bricks fail with that reason instead of being made (0: no limit). +# pause_hours: local hours in which no new jobs start, e.g. 18-23 or 22-6 (empty: never). +max_cpu_percent=0 +worker_nice=0 +max_memory_mb=0 +max_model_bricks=0 +pause_hours= + # How often to look for models to make (milliseconds), how many to take at once, and how often a model may fail # before it's given up on (the dashboard can try again) poll_interval_ms=2000 poll_batch=32 max_attempts=3 -# Brick detail used for meshes and icons: 0 (most detailed) to 2 -brick_lod=0 +# Models are made like LU Toolbox's Process Model and Bake Lighting make them; these default to its settings. +# color_palette: lu_toolbox (its LU palette, unknown colors black) or brickdb (the client's Materials.xml). +# color_variation: each brick's brightness is shifted by up to this percent (the same every time; 0: none). +# transparent_opacity: transparent bricks' opacity in percent. +color_palette=lu_toolbox +color_variation=5 +transparent_opacity=58.82 -# Remove faces that can't be seen from anywhere, and darken hidden corners (ambient occlusion baked into the vertex -# colors): 0 or 1. ao_strength: 0 (none) to 1. optimize_resolution: detail of the renders that decide both. +# Levels of detail made, from the client's brickprimitives (0 most detailed, 2 least), and the distances they're +# drawn at (LU Toolbox picks each level's range from these by which levels are made; lod_cull: drawn up to). +lods=0,2 +lod_distance_0=0 +lod_distance_1=50 +lod_distance_2=100 +lod_distance_3=280 +lod_cull=10000 + +# The shapes' shader number: S_Opaque_Model (transparent shapes are S01_Alpha_Model) +shader_opaque=01 + +# Remove faces that can't be seen from anywhere (optimize_resolution: detail of the renders that decide it; +# hsr_ground_plane: 1 also removes what can only be seen from below the model) remove_hidden_faces=1 -bake_ao=1 -ao_strength=0.6 +hsr_ground_plane=0 optimize_resolution=1024 -# Icons: size in pixels, the camera's angle around (yaw, from the model's front) and above (pitch) the model in degrees, -# and the empty border (1 fills the icon). Modular builds (cars, rockets) have their own angles. +# Bake ambient occlusion into the vertex colors: rays per vertex, how far they look, strength 0 (none) to 1, and how +# much glowing colors add +bake_ao=1 +ao_samples=64 +ao_distance=5 +ao_strength=1 +glow_strength=6 + +# Icons, set up like LU Toolbox's icon renderer: size in pixels, the camera's angle around (yaw, from the model's +# front) and above (pitch) in degrees, its field of view, the empty border (1 fills the icon), the sun's angles and +# strength, the world light, shadows and ambient occlusion, its color corrections and color variation (percent). +# Modular builds (cars, rockets) have their own camera angles. icon_size=128 -icon_yaw=35 -icon_pitch=25 +icon_yaw=53.36 +icon_pitch=19.54 +icon_fov=39.6 icon_margin=1.03 -modular_icon_yaw=35 -modular_icon_pitch=20 +icon_sun_yaw=21 +icon_sun_pitch=50.3 +icon_sun_strength=2.5 +icon_ambient=0.192 +icon_shadows=1 +icon_ao=1 +icon_correct_colors=1 +icon_color_variation=0 +modular_icon_yaw=53.36 +modular_icon_pitch=19.54 diff --git a/tests/dUgcTests/UgcTests.cpp b/tests/dUgcTests/UgcTests.cpp index 08e23cb1d..8fe716a18 100644 --- a/tests/dUgcTests/UgcTests.cpp +++ b/tests/dUgcTests/UgcTests.cpp @@ -11,9 +11,12 @@ #include "UgcBricks.h" #include "UgcFormats.h" #include "UgcModel.h" +#include "UgcJobs.h" #include "UgcModular.h" +#include "UgcPalette.h" #include "UgcRender.h" #include "UgcStorage.h" +#include "UgcThrottle.h" #include "ZCompression.h" class Logger; @@ -137,7 +140,10 @@ TEST(UgcModel, BuildsOpaqueAndTransparentMeshes) { UgcBricks::BrickLibrary library(MakeRes(), 0); library.SetMaterials({ { 21, { 222, 0, 13, 255 } }, { 40, { 238, 238, 238, 150 } } }); std::string error; - const auto model = UgcModel::Build(UgcModel::ParseLxfml(LXFML5, error), library); + UgcModel::BuildOptions options; + options.palette = UgcModel::ePalette::BRICKDB; + options.colorVariation = 0.0f; + const auto model = UgcModel::Build(UgcModel::ParseLxfml(LXFML5, error), library, options); EXPECT_EQ(model.bricks, 2u); EXPECT_EQ(model.missingDesigns, std::vector{ 9999 }); EXPECT_EQ(model.opaque.TriangleCount(), 12u); @@ -176,7 +182,7 @@ TEST(UgcRender, RemovesWhatIsInsideAndDrawsIcons) { )", error); auto model = UgcModel::Build(parts, library); ASSERT_EQ(model.opaque.TriangleCount(), 24u); - const auto result = UgcRender::Optimize(model, UgcRender::OptimizeOptions{ 256, true, true, 0.6f }); + const auto result = UgcRender::Optimize(model, UgcRender::OptimizeOptions{ 256, true }); EXPECT_EQ(result.trianglesRemoved, 12u); EXPECT_EQ(model.opaque.TriangleCount(), 12u); EXPECT_EQ(model.opaque.positions.size(), 8u); @@ -280,12 +286,277 @@ TEST(UgcStorage, WritesListsAndEvicts) { ASSERT_TRUE(storage.Write(UgcStorage::Kind::MODULAR, 2002, { { "icon.png", std::string(100, 'b') } }, error)) << error; ASSERT_TRUE(storage.Write(UgcStorage::Kind::MODEL, 1001, { { "icon.png", std::string(50, 'c') } }, error)) << error; // replaced EXPECT_TRUE(storage.File(UgcStorage::Kind::MODEL, 1001, "icon.png")); + // The version before is kept to compare with + const auto previous = storage.File(UgcStorage::Kind::MODEL, 1001, "previous.icon.png"); + ASSERT_TRUE(previous); + EXPECT_EQ(std::filesystem::file_size(*previous), 100u); EXPECT_FALSE(storage.File(UgcStorage::Kind::MODEL, 1001, "../../etc/passwd")); EXPECT_EQ(storage.List().size(), 2u); std::filesystem::last_write_time(storage.Folder(UgcStorage::Kind::MODULAR, 2002), std::filesystem::file_time_type::clock::now() - std::chrono::hours(1)); - const auto removed = storage.Evict(60); + const auto removed = storage.Evict(160); ASSERT_EQ(removed.size(), 1u); EXPECT_EQ(removed[0].id, 2002); EXPECT_TRUE(storage.File(UgcStorage::Kind::MODEL, 1001, "icon.png")); std::filesystem::remove_all(storage.GetRoot()); } + +TEST(UgcPalette, ColorVariationMatchesLuToolbox) { + const glm::vec3 red = *UgcPalette::Linear(21); + // random 0.5 is the middle of the range: no change + const auto same = UgcPalette::ApplyVariation(red, 7.0f, 0.5f); + EXPECT_NEAR(same.r, red.r, 1e-5f); + EXPECT_NEAR(same.b, red.b, 1e-5f); + // The top of the range: value^(1/2.224) + variation/200, back to the power of 2.224; hue and saturation kept + const auto brighter = UgcPalette::ApplyVariation(red, 7.0f, 1.0f); + const float expected = std::pow(std::pow(red.r, 1.0f / 2.224f) + 0.035f, 2.224f); + EXPECT_NEAR(brighter.r, expected, 1e-5f); + EXPECT_NEAR(brighter.b / brighter.r, red.b / red.r, 1e-5f); + const auto darker = UgcPalette::ApplyVariation(red, 7.0f, 0.0f); + EXPECT_LT(darker.r, red.r); + // Clamped to 0..1, and black turns grey rather than staying black + EXPECT_LE(UgcPalette::ApplyVariation(glm::vec3(1.0f), 100.0f, 1.0f).r, 1.0f); + EXPECT_GT(UgcPalette::ApplyVariation(glm::vec3(0.0f), 10.0f, 1.0f).g, 0.0f); + EXPECT_FLOAT_EQ(UgcPalette::ApplyVariation(red, 0.0f, 1.0f).r, red.r); + + // Per color amounts, aliases, transparency, glow and the icon's corrections + EXPECT_FLOAT_EQ(UgcPalette::VariationScale(26), 0.4f); + EXPECT_FLOAT_EQ(UgcPalette::VariationScale(5), 1.0f); + EXPECT_EQ(*UgcPalette::Linear(0), *UgcPalette::Linear(26)); + EXPECT_EQ(*UgcPalette::Linear(293), *UgcPalette::Linear(43)); + EXPECT_TRUE(UgcPalette::IsTransparent(40)); + EXPECT_FALSE(UgcPalette::IsTransparent(21)); + EXPECT_TRUE(UgcPalette::Glow(9013).has_value()); + EXPECT_FALSE(UgcPalette::Glow(21).has_value()); + EXPECT_TRUE(UgcPalette::IsMetallic(309)); + EXPECT_FALSE(UgcPalette::Linear(123456).has_value()); + EXPECT_NEAR(UgcPalette::LinearToSrgb(*UgcPalette::Linear(1, true)).r, 0.7f, 1e-5f); + EXPECT_NEAR(UgcPalette::LinearToSrgb(red).r * 255.0f, 222.0f, 0.5f); // LDD's bright red + EXPECT_NEAR(UgcPalette::SrgbToLinear(UgcPalette::LinearToSrgb(0.3f)), 0.3f, 1e-5f); +} + +TEST(UgcPalette, BrickRandomIsStableAndSpread) { + EXPECT_EQ(UgcPalette::BrickRandom(7, 3, 21), UgcPalette::BrickRandom(7, 3, 21)); + EXPECT_NE(UgcPalette::BrickRandom(7, 3, 21), UgcPalette::BrickRandom(7, 4, 21)); + EXPECT_NE(UgcPalette::BrickRandom(7, 3, 21), UgcPalette::BrickRandom(8, 3, 21)); + EXPECT_NE(UgcPalette::BrickRandom(7, 3, 21), UgcPalette::BrickRandom(7, 3, 23)); + double sum = 0.0; + float low = 1.0f, high = 0.0f; + for (uint32_t brick = 0; brick < 10000; brick++) { + const float value = UgcPalette::BrickRandom(1, brick, 1); + ASSERT_GE(value, 0.0f); + ASSERT_LT(value, 1.0f); + sum += value; + low = std::min(low, value); + high = std::max(high, value); + } + EXPECT_NEAR(sum / 10000.0, 0.5, 0.02); // uniform, like random.uniform + EXPECT_LT(low, 0.01f); + EXPECT_GT(high, 0.99f); +} + +TEST(UgcModel, ColorsLikeLuToolbox) { + const auto res = MakeRes(); + std::filesystem::create_directories(res / "brickprimitives" / "lod1"); + std::ofstream(res / "brickprimitives" / "lod1" / "3001.g", std::ios::binary) << BoxGeometry(glm::vec3(0.0f), glm::vec3(1.0f)); + UgcBricks::BrickLibrary library(res, 0); + std::string error; + // Three red bricks, a transparent one, a red and transparent one, an unknown color + const auto parts = UgcModel::ParseLxfml(R"( + + + + + + + )", error); + ASSERT_EQ(parts.size(), 6u); + + UgcModel::BuildOptions plain; + plain.colorVariation = 0.0f; + const auto flat = UgcModel::Build(parts, library, plain); + EXPECT_EQ(flat.transparent.TriangleCount(), 12u); // only the all-transparent brick + EXPECT_EQ(flat.opaque.TriangleCount(), 60u); + EXPECT_NEAR(flat.opaque.colors[0].r * 255.0f, 222.0f, 0.5f); + EXPECT_FLOAT_EQ(flat.opaque.colors[0].a, 1.0f); + EXPECT_NEAR(flat.transparent.colors[0].a, 0.5882f, 1e-4f); + const auto black = UgcPalette::LinearToSrgb(*UgcPalette::Linear(26)); + EXPECT_NEAR(flat.opaque.colors[4 * 8].r, black.r, 1e-5f); // the unknown color is black + EXPECT_TRUE(flat.opaque.glow.empty()); + + UgcModel::BuildOptions varied; + varied.seed = 42; + const auto a = UgcModel::Build(parts, library, varied); + const auto again = UgcModel::Build(parts, library, varied); + EXPECT_EQ(a.opaque.colors, again.opaque.colors); // the same every time + // Each brick has one shift for all its vertices, different between bricks of the same color + EXPECT_EQ(a.opaque.colors[0], a.opaque.colors[7]); + EXPECT_NE(a.opaque.colors[0], a.opaque.colors[8]); + EXPECT_NE(a.opaque.colors[8], a.opaque.colors[16]); + // Within 5% x 1.4 (red's own amount) of the plain color in LU Toolbox's gamma + for (size_t brick = 0; brick < 3; brick++) { + const float value = UgcPalette::SrgbToLinear(a.opaque.colors[brick * 8].r); + const float base = UgcPalette::Linear(21)->r; + EXPECT_LE(std::abs(std::pow(value, 1.0f / 2.224f) - std::pow(base, 1.0f / 2.224f)), 0.035f + 1e-4f); + } + // The same brick gets the same color in another LOD + varied.lod = 1; + const auto lod1 = UgcModel::Build(parts, library, varied); + EXPECT_EQ(lod1.opaque.colors[8], a.opaque.colors[8]); + // Another model (seed) gets other shifts + varied.lod = 0; + varied.seed = 43; + EXPECT_NE(UgcModel::Build(parts, library, varied).opaque.colors[0], a.opaque.colors[0]); + + // The icon: its corrections, and no variation + UgcModel::BuildOptions icon; + icon.icon = true; + icon.colorVariation = 0.0f; + const auto white = UgcModel::Build(UgcModel::ParseLxfml(R"( + )", error), library, icon); + EXPECT_NEAR(white.opaque.colors[0].r, 0.7f, 1e-5f); +} + +TEST(UgcModel, LodRangesLikeLuToolbox) { + const UgcModel::LodDistances d; + using Ranges = std::vector>; + EXPECT_EQ(UgcModel::LodRanges({ 0, 2 }, d), (Ranges{ { 0.0f, 100.0f }, { 100.0f, 10000.0f } })); + EXPECT_EQ(UgcModel::LodRanges({ 0 }, d), (Ranges{ { 0.0f, 10000.0f } })); + EXPECT_EQ(UgcModel::LodRanges({ 0, 1, 2 }, d), (Ranges{ { 0.0f, 50.0f }, { 50.0f, 100.0f }, { 100.0f, 10000.0f } })); + EXPECT_EQ(UgcModel::LodRanges({ 0, 1 }, d), (Ranges{ { 0.0f, 50.0f }, { 50.0f, 10000.0f } })); + EXPECT_EQ(UgcModel::LodRanges({ 0, 2, 3 }, d), (Ranges{ { 0.0f, 100.0f }, { 100.0f, 280.0f }, { 280.0f, 10000.0f } })); +} + +TEST(UgcModel, DividesAlongTheLongestSide) { + // Two separate strips of triangles far apart on x: divided between them, each kept whole + UgcModel::Mesh mesh; + for (int cluster = 0; cluster < 2; cluster++) { + for (uint32_t i = 0; i < 40; i++) { + mesh.positions.push_back(glm::vec3(cluster * 100.0f + static_cast(i % 2), static_cast(i / 2), 0.0f)); + mesh.normals.push_back(glm::vec3(0, 0, 1)); + mesh.colors.push_back(glm::vec4(1.0f)); + } + const uint32_t base = cluster * 40; + for (uint32_t i = 0; i + 2 < 40; i++) mesh.indices.insert(mesh.indices.end(), { base + i, base + i + 1, base + i + 2 }); + } + const auto pieces = UgcModel::Divide(mesh, 50, 1000); + ASSERT_EQ(pieces.size(), 2u); + for (const auto& piece : pieces) { + EXPECT_EQ(piece.positions.size(), 40u); + EXPECT_EQ(piece.TriangleCount(), 38u); + } + EXPECT_EQ(UgcModel::Divide(mesh, 100, 1000).size(), 1u); +} + +TEST(UgcFormats, LodNifReadsBack) { + UgcModel::Mesh near, far; + near.positions = { { 0, 0, 0 }, { 1, 0, 0 }, { 0, 1, 0 }, { 1, 1, 0 } }; + near.normals.assign(4, { 0, 0, 1 }); + near.colors.assign(4, { 1, 0, 0, 1 }); + near.indices = { 0, 1, 2, 1, 3, 2 }; + far = near; + far.indices = { 0, 1, 2 }; + const auto nif = UgcFormats::WriteLodNif("SceneNode_Model", { { "S01_Opaque_Model", false, { { 0.0f, 100.0f, "LOD_0", { &near } }, { 100.0f, 10000.0f, "LOD_2", { &far } } } } }); + std::string error; + const auto lod0 = NifFile::Parse(nif, 0, error); + ASSERT_TRUE(lod0) << error; + EXPECT_TRUE(lod0->skipped.empty()); + ASSERT_EQ(lod0->meshes.size(), 1u); + EXPECT_EQ(lod0->meshes[0].indices.size(), 6u); + EXPECT_TRUE(lod0->nodes.contains("S01_Opaque_Model")); + EXPECT_TRUE(lod0->nodes.contains("LOD_0")); + const auto lod1 = NifFile::Parse(nif, 1, error); + ASSERT_TRUE(lod1) << error; + ASSERT_EQ(lod1->meshes.size(), 1u); + EXPECT_EQ(lod1->meshes[0].indices.size(), 3u); +} + +TEST(UgcRender, AmbientOcclusionUnderARoof) { + // A floor vertex under a low roof is dark, one out in the open is lit; nothing is hit past the distance + UgcModel::Mesh points; + points.positions = { { 0, 0, 0 }, { 50, 0, 0 } }; + points.normals = { { 0, 1, 0 }, { 0, 1, 0 } }; + UgcModel::Mesh roof; + roof.positions = { { -10, 1, -10 }, { 10, 1, -10 }, { -10, 1, 10 }, { 10, 1, 10 } }; + roof.normals.assign(4, { 0, -1, 0 }); + roof.indices = { 0, 1, 2, 1, 3, 2 }; + const auto ao = UgcRender::AmbientOcclusion(points, roof, 5.0f, 64); + ASSERT_EQ(ao.size(), 2u); + EXPECT_LT(ao[0], 0.2f); + EXPECT_FLOAT_EQ(ao[1], 1.0f); + EXPECT_FLOAT_EQ(UgcRender::AmbientOcclusion(points, roof, 0.5f, 64)[0], 1.0f); + + // Baking darkens the colors of occluded vertices only, and glow lights them up again + UgcModel::Model model; + model.opaque = roof; + model.opaque.colors.assign(4, glm::vec4(0.8f, 0.8f, 0.8f, 1.0f)); + UgcModel::Mesh floor = roof; + for (auto& p : floor.positions) p.y = 0.0f; + floor.normals.assign(4, { 0, 1, 0 }); + floor.colors.assign(4, glm::vec4(0.8f, 0.8f, 0.8f, 1.0f)); + model.opaque.Append(floor); + UgcRender::BakeAo(model, UgcRender::AoOptions{}); + EXPECT_LT(model.opaque.colors[5].r, 0.8f); + EXPECT_EQ(model.opaque.colors[5].a, 1.0f); +} + +TEST(UgcThrottle, KeepsUnderTheBudget) { + int from = -1, to = -1; + EXPECT_TRUE(UgcThrottle::ParseHours("22-6", from, to)); + EXPECT_TRUE(UgcThrottle::InHours(23, from, to)); + EXPECT_TRUE(UgcThrottle::InHours(3, from, to)); + EXPECT_FALSE(UgcThrottle::InHours(12, from, to)); + EXPECT_FALSE(UgcThrottle::ParseHours("", from, to)); + EXPECT_FALSE(UgcThrottle::ParseHours("25-3", from, to)); + EXPECT_FALSE(UgcThrottle::InHours(3, -1, -1)); + + // 0.6 s of CPU work at a quarter of a CPU takes at least (0.6 - the burst) / 0.25 s + UgcThrottle::SetBudget(0.25); + UgcThrottle::Begin(); + const auto start = std::chrono::steady_clock::now(); + const double cpuStart = UgcThrottle::ThreadCpuSeconds(); + volatile double sink = 0.0; + while (UgcThrottle::ThreadCpuSeconds() - cpuStart < 0.6) { + for (int i = 0; i < 10000; i++) sink = sink + std::sqrt(static_cast(i)); + UgcThrottle::Checkpoint(); + } + const double wall = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + UgcThrottle::SetBudget(0.0); + EXPECT_GE(wall, 1.2); + EXPECT_GT(UgcThrottle::GetStats().sleptMs, 0u); +} + +TEST(UgcJobs, MakesLodsStatsAndIcons) { + UgcBricks::BrickLibrary library(MakeRes(), 0); + UgcJobs::Settings settings; + settings.optimize.resolution = 128; + settings.ao.samples = 8; + settings.icon.size = 32; + settings.icon.supersample = 1; + settings.icon.ao.samples = 4; + const auto outcome = UgcJobs::ProcessModel(LXFML5, library, settings, 99); + ASSERT_TRUE(outcome.ok) << outcome.error; + for (const auto* name : { "model.nif", "model.nif.gz", "model.nif.checksum", "model.noao.nif", "icon.png", "icon.dds.gz", "stats.json", "model.lxfml.gz" }) { + EXPECT_TRUE(outcome.files.contains(name)) << name; + } + EXPECT_NE(outcome.stats.find("\"lods\""), std::string::npos); + EXPECT_NE(outcome.stats.find("\"opaqueAfter\""), std::string::npos); + std::string error; + const auto nif = NifFile::Parse(outcome.files.at("model.nif"), 0, error); + ASSERT_TRUE(nif) << error; + EXPECT_TRUE(nif->nodes.contains("S01_Opaque_Model")); + EXPECT_TRUE(nif->nodes.contains("S01_Alpha_Model")); + EXPECT_TRUE(nif->nodes.contains("LOD_0")); + const auto far = NifFile::Parse(outcome.files.at("model.nif"), 1, error); + ASSERT_TRUE(far) << error; + EXPECT_TRUE(far->nodes.contains("LOD_2")); + // The same colors when made again + EXPECT_EQ(UgcJobs::ProcessModel(LXFML5, library, settings, 99).files.at("model.nif"), outcome.files.at("model.nif")); + + settings.maxBricks = 2; + const auto tooBig = UgcJobs::ProcessModel(LXFML5, library, settings, 99); + EXPECT_FALSE(tooBig.ok); + EXPECT_NE(tooBig.error.find("max_model_bricks"), std::string::npos); + + EXPECT_EQ(UgcJobs::CountParts(LXFML5), 3u); + EXPECT_GT(UgcJobs::EstimateMemory(1000, settings), UgcJobs::EstimateMemory(10, settings)); +}