Mirrored_Repos/DarkflameServer
1
0
Fork 0
mirror of https://github.com/DarkflameUniverse/DarkflameServer.git synced 2026-10-02 02:43:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ugc): make models like LU Toolbox does, within CPU and memory budgets

Browse Source 
Parity with LU Toolbox's Process Model, Bake Lighting and icon renderer, with
its defaults as the settings' defaults:

- Colors from its LU palette (UgcPalette: LU colors, LDD colors mapped to the
  nearest LU one, unknown ones black), transparent bricks at 58.82% opacity, a
  brick transparent only when all of its materials are.
- Color variation: each brick's material gets its HSV value shifted in a 2.224
  gamma by up to 5% (times the color's own amount), from a random number of
  the model, brick and material, so reprocessing gives the same colors in
  every LOD. Icons get none, and the icon renderer's color corrections.
- LODs 0 and 2 with its distance logic, written as NiLODNode/NiRangeLODData
  like its exports and the game's own brick models, shapes divided at 65535
  vertices along the longest side like divide_mesh.
- Ambient occlusion like its AO-only bake: 64 rays per vertex, distance 5,
  after hidden surface removal, transparent bricks neither baked nor
  occluding, glow colors added.
- Icons from its icon scene: 50 mm lens at 53.4/19.5 degrees, sun of 2.5 at
  21/50.3 degrees with soft shadows, grey world light with occlusion; LOD 0's
  hidden surface removal and occlusion are reused for them.
- Optional ground plane for hidden surface removal; stats.json per model and
  the previous version's previews kept for comparing.

Budgets, applied live on config reload: max_cpu_percent (workers account
their thread CPU time and sleep to stay under it, long renders included),
worker_nice, max_memory_mb (jobs are estimated from their brick count and
wait until they fit), max_model_bricks and pause_hours. /status and the
traffic report show CPU, memory and throttling.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-09-27 09:19:59 -05:00
parent 0de9decc7c
commit 7c5d69c1ed
21 changed files with 1875 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
dUgcServer/CMakeLists.txt
View File

@@ -5,12 +5,14 @@ set(DUGC_SOURCES
"UgcJobs.cpp"
"UgcModel.cpp"
"UgcModular.cpp"
"UgcPalette.cpp"
"UgcRender.cpp"
"UgcStorage.cpp"
"UgcThrottle.cpp"
)
add_library(dUgc STATIC ${DUGC_SOURCES})
target_include_directories(dUgc PUBLIC "." "${PROJECT_SOURCE_DIR}/thirdparty/MD5")
target_include_directories(dUgc PUBLIC "." "${PROJECT_SOURCE_DIR}/thirdparty/MD5" "${PROJECT_SOURCE_DIR}/thirdparty/nlohmann")
target_link_libraries(dUgc PUBLIC dCommon glm::glm tinyxml2 MD5)
add_executable(UgcServer "UgcServer.cpp" "UgcProcessor.cpp" "UgcCdClient.cpp")
@@ -36,5 +38,5 @@ target_link_libraries(UgcServer ${COMMON_LIBRARIES} dWeb dServer dUgc)
# Rendering is far too slow unoptimized (minutes a model instead of seconds), so it is optimized in every build type
if(NOT MSVC)
set_source_files_properties("UgcRender.cpp" "UgcModel.cpp" PROPERTIES COMPILE_OPTIONS "-O2")
set_source_files_properties("UgcRender.cpp" "UgcModel.cpp" "UgcPalette.cpp" PROPERTIES COMPILE_OPTIONS "-O2")
endif()

16
dUgcServer/UgcBricks.cpp
View File

@@ -185,14 +185,16 @@ namespace UgcBricks {
return it != m_Materials.end() ? it->second : Material{};
}
std::shared_ptr<const std::vector<Geometry>> BrickLibrary::GetDesign(uint32_t design) {
std::shared_ptr<const std::vector<Geometry>> BrickLibrary::GetDesign(uint32_t design, std::optional<uint32_t> lodLevel) {
const uint32_t lod = std::min<uint32_t>(lodLevel.value_or(m_Lod), 2);
const uint64_t key = (static_cast<uint64_t>(lod) << 32) | design;
{
std::lock_guard lock(m_Mutex);
if (const auto it = m_Designs.find(design); it != m_Designs.end()) return it->second;
if (const auto it = m_Designs.find(key); it != m_Designs.end()) return it->second;
}
// Loaded outside the lock; two threads loading the same design at once is harmless
auto parts = std::make_shared<std::vector<Geometry>>();
const auto folder = "brickprimitives/lod" + std::to_string(m_Lod) + "/";
const auto folder = "brickprimitives/lod" + std::to_string(lod) + "/";
for (uint32_t index = 0; index < MAX_GEOMETRY_PARTS; index++) {
const auto name = std::to_string(design) + ".g" + (index == 0 ? "" : std::to_string(index));
const auto path = ResolvePath(m_Res, folder + name);
@@ -202,7 +204,13 @@ namespace UgcBricks {
if (!geometry) break;
parts->push_back(std::move(*geometry));
}
if (parts->empty() && lod > 0) return GetDesign(design, lod - 1);
std::lock_guard lock(m_Mutex);
return m_Designs.emplace(design, std::move(parts)).first->second;
return m_Designs.emplace(key, std::move(parts)).first->second;
}
size_t BrickLibrary::CachedDesigns() const {
std::lock_guard lock(m_Mutex);
return m_Designs.size();
}
}

11
dUgcServer/UgcBricks.h
View File

@@ -64,7 +64,12 @@ namespace UgcBricks {
Material GetMaterial(uint32_t id) const;
// Every geometry file of a design, in order (.g, .g1, ...); empty when the design has none. Loaded once.
std::shared_ptr<const std::vector<Geometry>> GetDesign(uint32_t design);
// `lod`: the brickprimitives level, the library's own when omitted; a design without that level uses the
// next more detailed one.
std::shared_ptr<const std::vector<Geometry>> GetDesign(uint32_t design, std::optional<uint32_t> lod = std::nullopt);
// Designs loaded (for the memory report)
size_t CachedDesigns() const;
const std::filesystem::path& GetResPath() const { return m_Res; }
@@ -72,7 +77,7 @@ namespace UgcBricks {
std::filesystem::path m_Res;
uint32_t m_Lod;
std::map<uint32_t, Material> m_Materials;
std::mutex m_Mutex;
std::map<uint32_t, std::shared_ptr<const std::vector<Geometry>>> m_Designs;
mutable std::mutex m_Mutex;
std::map<uint64_t, std::shared_ptr<const std::vector<Geometry>>> m_Designs; // lod << 32 | design
};
}

151
dUgcServer/UgcFormats.cpp
View File

@@ -187,63 +187,120 @@ namespace {
out.U16(0); // match groups
return std::move(out.Data());
}
// An NiNode's data: no properties, `children`, no effects
std::string NodeData(int32_t name, const std::vector<int32_t>& children) {
Writer node;
WriteAv(node, name, {});
node.U32(static_cast<uint32_t>(children.size()));
for (const auto child : children) node.I32(child);
node.U32(0); // effects
return std::move(node.Data());
}
// The properties every shape shares, and the shapes
class SharedProperties {
public:
explicit SharedProperties(NifBuilder& nif) : m_Nif(nif) {
Writer material;
WriteNet(material, -1);
for (int i = 0; i < 3; i++) material.Float(1.0f); // ambient
for (int i = 0; i < 3; i++) material.Float(1.0f); // diffuse
for (int i = 0; i < 3; i++) material.Float(0.0f); // specular
for (int i = 0; i < 3; i++) material.Float(0.0f); // emissive
material.Float(10.0f); // glossiness
material.Float(1.0f); // alpha
m_Material = nif.Add("NiMaterialProperty", std::move(material.Data()));
Writer vertexColor;
WriteNet(vertexColor, -1);
vertexColor.U16((2 << 4) | (1 << 3)); // vertex colors are ambient and diffuse; lit
m_VertexColor = nif.Add("NiVertexColorProperty", std::move(vertexColor.Data()));
}
// An NiTriShape of `mesh` (-1 when it is empty or too big for the format)
int32_t Shape(const std::string& name, const UgcModel::Mesh* mesh, bool transparent) {
if (!mesh || mesh->Empty() || mesh->positions.size() > 65535 || mesh->TriangleCount() > 65535) return -1;
std::vector<int32_t> properties{ m_Material, m_VertexColor };
if (transparent) {
if (m_Alpha < 0) {
Writer alpha;
WriteNet(alpha, -1);
alpha.U16(1 | (6 << 1) | (7 << 5)); // blend source alpha over one minus source alpha
alpha.U8(0);
m_Alpha = m_Nif.Add("NiAlphaProperty", std::move(alpha.Data()));
}
properties.push_back(m_Alpha);
}
const auto shapeBlock = m_Nif.Reserve("NiTriShape");
const auto dataBlock = m_Nif.Add("NiTriShapeData", TriShapeData(*mesh));
Writer tri;
WriteAv(tri, m_Nif.String(name), properties);
tri.I32(dataBlock);
tri.I32(-1); // skin instance
tri.U32(0); // materials
tri.I32(-1); // active material
tri.U8(0); // material needs update
m_Nif.Fill(shapeBlock, std::move(tri.Data()));
return shapeBlock;
}
private:
NifBuilder& m_Nif;
int32_t m_Material{ -1 };
int32_t m_VertexColor{ -1 };
int32_t m_Alpha{ -1 };
};
}
namespace UgcFormats {
std::string WriteNif(const std::string& rootName, const std::vector<NifShape>& shapes) {
NifBuilder nif;
const int32_t root = nif.Reserve("NiNode");
// Shared properties
Writer material;
WriteNet(material, -1);
for (int i = 0; i < 3; i++) material.Float(1.0f); // ambient
for (int i = 0; i < 3; i++) material.Float(1.0f); // diffuse
for (int i = 0; i < 3; i++) material.Float(0.0f); // specular
for (int i = 0; i < 3; i++) material.Float(0.0f); // emissive
material.Float(10.0f); // glossiness
material.Float(1.0f); // alpha
const auto materialBlock = nif.Add("NiMaterialProperty", std::move(material.Data()));
Writer vertexColor;
WriteNet(vertexColor, -1);
vertexColor.U16((2 << 4) | (1 << 3)); // vertex colors are ambient and diffuse; lit
const auto vertexColorBlock = nif.Add("NiVertexColorProperty", std::move(vertexColor.Data()));
int32_t alphaBlock = -1;
SharedProperties properties(nif);
std::vector<int32_t> children;
for (const auto& shape : shapes) {
if (!shape.mesh || shape.mesh->Empty() || shape.mesh->positions.size() > 65535 || shape.mesh->TriangleCount() > 65535) continue;
std::vector<int32_t> properties{ materialBlock, vertexColorBlock };
if (shape.transparent) {
if (alphaBlock < 0) {
Writer alpha;
WriteNet(alpha, -1);
alpha.U16(1 | (6 << 1) | (7 << 5)); // blend source alpha over one minus source alpha
alpha.U8(0);
alphaBlock = nif.Add("NiAlphaProperty", std::move(alpha.Data()));
}
properties.push_back(alphaBlock);
}
const auto shapeBlock = nif.Reserve("NiTriShape");
const auto dataBlock = nif.Add("NiTriShapeData", TriShapeData(*shape.mesh));
Writer tri;
WriteAv(tri, nif.String(shape.name), properties);
tri.I32(dataBlock);
tri.I32(-1); // skin instance
tri.U32(0); // materials
tri.I32(-1); // active material
tri.U8(0); // material needs update
nif.Fill(shapeBlock, std::move(tri.Data()));
children.push_back(shapeBlock);
const auto block = properties.Shape(shape.name, shape.mesh, shape.transparent);
if (block >= 0) children.push_back(block);
}
nif.Fill(root, NodeData(nif.String(rootName), children));
return nif.Finish(root);
}
Writer node;
WriteAv(node, nif.String(rootName), {});
node.U32(static_cast<uint32_t>(children.size()));
for (const auto child : children) node.I32(child);
node.U32(0); // effects
nif.Fill(root, std::move(node.Data()));
std::string WriteLodNif(const std::string& rootName, const std::vector<NifLodGroup>& groups) {
NifBuilder nif;
const int32_t root = nif.Reserve("NiNode");
SharedProperties properties(nif);
std::vector<int32_t> groupBlocks;
for (const auto& group : groups) {
if (group.lods.empty()) continue;
const auto lodNode = nif.Reserve("NiLODNode");
std::vector<int32_t> levels;
Writer ranges;
for (int i = 0; i < 3; i++) ranges.Float(0.0f); // LOD center
ranges.U32(static_cast<uint32_t>(group.lods.size()));
for (const auto& lod : group.lods) {
const auto level = nif.Reserve("NiNode");
std::vector<int32_t> shapes;
for (const auto* piece : lod.pieces) {
const auto block = properties.Shape(group.name, piece, group.transparent);
if (block >= 0) shapes.push_back(block);
}
nif.Fill(level, NodeData(nif.String(lod.name), shapes));
levels.push_back(level);
ranges.Float(lod.nearDistance);
ranges.Float(lod.farDistance);
}
const auto rangeData = nif.Add("NiRangeLODData", std::move(ranges.Data()));
auto data = NodeData(nif.String(group.name), levels);
Writer lod;
lod.Raw(data);
lod.U16(3); // switch flags: update only the active child, and controllers (as the game's own files)
lod.U32(0); // index
lod.I32(rangeData);
nif.Fill(lodNode, std::move(lod.Data()));
groupBlocks.push_back(lodNode);
}
nif.Fill(root, NodeData(nif.String(rootName), groupBlocks));
return nif.Finish(root);
}

20
dUgcServer/UgcFormats.h
View File

@@ -26,6 +26,26 @@ namespace UgcFormats {
*/
std::string WriteNif(const std::string& rootName, const std::vector<NifShape>& shapes);
struct NifLod {
float nearDistance{};
float farDistance{};
std::string name; // the level's node, e.g. LOD_0
std::vector<const UgcModel::Mesh*> pieces; // its shapes (UgcModel::Divide's pieces)
};
struct NifLodGroup {
std::string name; // S01_Opaque_Model: the NiLODNode and its shapes
bool transparent{};
std::vector<NifLod> lods;
};
/**
* The layout LU Toolbox exports (setup_lod_data) and the game's own brick models (res/BrickModels/ndmade) have:
* the root node, an NiLODNode per group with NiRangeLODData holding each level's distances, a node per level and
* the level's shapes under it, named like the group. Properties as WriteNif.
*/
std::string WriteLodNif(const std::string& rootName, const std::vector<NifLodGroup>& groups);
// PNG (8-bit RGBA)
std::string EncodePng(const UgcRender::Image& image);

171
dUgcServer/UgcJobs.cpp
View File

@@ -1,7 +1,10 @@
#include "UgcJobs.h"
#include <chrono>
#include <sstream>
#include "json.hpp"
#include "Sd0.h"
#include "UgcFormats.h"
#include "UgcModel.h"
@@ -27,8 +30,8 @@ namespace UgcJobs {
namespace {
// The icon files of a model, and false when nothing was drawn
bool AddIcon(UgcStorage::Files& files, const UgcModel::Model& model, const UgcRender::IconOptions& options) {
const auto icon = UgcRender::RenderIcon(model, options);
bool AddIcon(UgcStorage::Files& files, const UgcModel::Model& model, const UgcRender::IconOptions& options, const std::vector<float>* ao = nullptr) {
const auto icon = UgcRender::RenderIcon(model, options, ao);
bool drawn = false;
for (size_t i = 3; i < icon.rgba.size(); i += 4) drawn = drawn || icon.rgba[i] != 0;
files["icon.png"] = UgcFormats::EncodePng(icon);
@@ -37,8 +40,34 @@ namespace UgcJobs {
}
}
Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings) {
size_t CountParts(std::string_view lxfml) {
size_t count = 0;
for (size_t at = lxfml.find("<Part"); at != std::string_view::npos; at = lxfml.find("<Part", at + 5)) count++;
return count;
}
uint64_t EstimateMemory(size_t parts, const Settings& settings) {
// Measured: the renders' buffers, and per brick its mesh in each LOD (positions, normals, colors, indices,
// the occlusion tree and copies made along the way), about 40 KB at LOD 0
const uint64_t resolution = static_cast<uint64_t>(std::clamp(settings.optimize.resolution, 64, 4096));
const uint64_t icon = static_cast<uint64_t>(settings.icon.size) * settings.icon.supersample;
const uint64_t fixed = resolution * resolution * 8 + icon * icon * 20 + 1024 * 1024 * 4 + 16 * 1024 * 1024;
return fixed + static_cast<uint64_t>(parts) * 40 * 1024 * (1 + settings.lods.size());
}
namespace {
double Since(std::chrono::steady_clock::time_point start) {
return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count();
}
std::string ShapeName(const std::string& shader, bool transparent) {
return ("S" + (transparent ? std::string("01") : shader) + (transparent ? "_Alpha_" : "_Opaque_") + "Model").substr(0, 60);
}
}
Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings, uint64_t seed) {
Outcome outcome;
const auto started = std::chrono::steady_clock::now();
const auto lxfml = LxfmlFromBlob(blob);
if (lxfml.empty()) {
outcome.error = "the stored LXFML can't be read";
@@ -50,40 +79,124 @@ namespace UgcJobs {
outcome.error = error;
return outcome;
}
auto model = UgcModel::Build(parts, library);
if (!model.missingDesigns.empty()) {
outcome.note = "no geometry for design(s)";
for (const auto design : model.missingDesigns) outcome.note += " " + std::to_string(design);
}
if (model.Empty()) {
outcome.error = "none of the model's bricks have geometry";
if (!outcome.note.empty()) outcome.error += " (" + outcome.note + ")";
if (settings.maxBricks > 0 && parts.size() > settings.maxBricks) {
outcome.error = "the model has " + std::to_string(parts.size()) + " bricks, more than max_model_bricks (" + std::to_string(settings.maxBricks) + ")";
return outcome;
}
const auto optimized = UgcRender::Optimize(model, settings.optimize);
outcome.aoBaked = settings.optimize.bakeAo;
if (optimized.trianglesRemoved > 0) {
if (!outcome.note.empty()) outcome.note += "; ";
outcome.note += "removed " + std::to_string(optimized.trianglesRemoved) + " of " + std::to_string(optimized.trianglesBefore) + " triangles";
}
auto lods = settings.lods;
std::erase_if(lods, [](uint32_t lod) { return lod > 3; });
std::sort(lods.begin(), lods.end());
lods.erase(std::unique(lods.begin(), lods.end()), lods.end());
if (lods.empty()) lods.push_back(0);
const auto ranges = UgcModel::LodRanges(lods, settings.lodDistances);
// Named like LU Toolbox names its shapes: S<shader>_<Opaque|Alpha>_<name>
std::vector<std::pair<UgcModel::Mesh, bool>> pieces;
for (const bool transparent : { false, true }) {
for (auto& piece : UgcModel::Split(transparent ? model.transparent : model.opaque)) pieces.emplace_back(std::move(piece), transparent);
nlohmann::json stats;
stats["version"] = 1;
stats["bricks"] = parts.size();
auto& lodStats = stats["lods"] = nlohmann::json::array();
double buildMs = 0, hsrMs = 0, aoMs = 0;
// Every LOD made like LU Toolbox makes each LOD collection: colored, hidden faces removed, lighting baked, divided
std::vector<UgcModel::Model> models;
std::vector<std::vector<UgcModel::Mesh>> opaquePieces, transparentPieces;
UgcModel::Model preview; // LOD 0 before the lighting bake, for the dashboard
// The icon: LU Toolbox's icon renderer imports LOD 0 again, with its own color corrections and no variation.
// The same bricks in the same order, so LOD 0's hidden surface removal and occlusion apply to it too (neither
// changes what the icon's camera sees).
UgcModel::Model iconModel;
std::vector<float> iconAo;
double iconMs = 0;
for (size_t i = 0; i < lods.size(); i++) {
auto options = settings.build;
options.seed = seed;
options.lod = lods[i];
auto step = std::chrono::steady_clock::now();
auto model = UgcModel::Build(parts, library, options);
buildMs += Since(step);
if (i == 0) {
if (!model.missingDesigns.empty()) {
outcome.note = "no geometry for design(s)";
for (const auto design : model.missingDesigns) outcome.note += " " + std::to_string(design);
stats["missingDesigns"] = model.missingDesigns;
}
if (model.Empty()) {
outcome.error = "none of the model's bricks have geometry";
if (!outcome.note.empty()) outcome.error += " (" + outcome.note + ")";
return outcome;
}
}
nlohmann::json entry{ { "lod", lods[i] }, { "near", ranges[i].first }, { "far", ranges[i].second },
{ "opaqueBefore", model.opaque.TriangleCount() }, { "transparent", model.transparent.TriangleCount() } };
step = std::chrono::steady_clock::now();
const auto optimized = UgcRender::Optimize(model, settings.optimize);
hsrMs += Since(step);
if (i == 0 && optimized.trianglesRemoved > 0) {
if (!outcome.note.empty()) outcome.note += "; ";
outcome.note += "removed " + std::to_string(optimized.trianglesRemoved) + " of " + std::to_string(optimized.trianglesBefore) + " triangles";
}
if (i == 0) {
preview = model;
step = std::chrono::steady_clock::now();
auto iconOptions = settings.build;
iconOptions.seed = seed;
iconOptions.lod = 0;
iconOptions.icon = settings.iconCorrectColors;
iconOptions.colorVariation = settings.iconColorVariation;
iconModel = UgcModel::Build(parts, library, iconOptions);
if (!optimized.kept.empty() && iconModel.opaque.TriangleCount() == optimized.kept.size()) UgcModel::KeepTriangles(iconModel.opaque, optimized.kept);
iconMs += Since(step);
}
step = std::chrono::steady_clock::now();
auto ao = UgcRender::BakeAo(model, settings.ao);
aoMs += Since(step);
if (i == 0) iconAo = std::move(ao);
entry["opaqueAfter"] = model.opaque.TriangleCount();
entry["vertices"] = model.opaque.positions.size() + model.transparent.positions.size();
opaquePieces.push_back(UgcModel::Divide(model.opaque));
transparentPieces.push_back(UgcModel::Divide(model.transparent));
entry["shapes"] = opaquePieces.back().size() + transparentPieces.back().size();
lodStats.push_back(entry);
}
std::vector<UgcFormats::NifShape> shapes;
size_t opaqueIndex = 0, transparentIndex = 0;
for (const auto& [piece, transparent] : pieces) {
const auto index = transparent ? transparentIndex++ : opaqueIndex++;
shapes.push_back({ std::string("S01_") + (transparent ? "Alpha" : "Opaque") + "_Model" + (index > 0 ? std::to_string(index) : ""), &piece, transparent });
}
const auto nif = UgcFormats::WriteNif("SceneNode_Model", shapes);
outcome.aoBaked = settings.ao.enabled;
// An NiLODNode for the opaque bricks and one for the transparent ones, as LU Toolbox names them
const auto groups = [&](size_t levels, const std::vector<std::vector<UgcModel::Mesh>>& opaque, const std::vector<std::vector<UgcModel::Mesh>>& transparent) {
std::vector<UgcFormats::NifLodGroup> out;
for (const bool isTransparent : { false, true }) {
UgcFormats::NifLodGroup group{ ShapeName(settings.shaderOpaque, isTransparent), isTransparent, {} };
bool any = false;
for (size_t i = 0; i < levels; i++) {
UgcFormats::NifLod lod{ ranges[i].first, ranges[i].second, "LOD_" + std::to_string(lods[i]), {} };
for (const auto& piece : (isTransparent ? transparent : opaque)[i]) lod.pieces.push_back(&piece);
any = any || !lod.pieces.empty();
group.lods.push_back(std::move(lod));
}
if (any) out.push_back(std::move(group));
}
return out;
};
const auto nif = UgcFormats::WriteLodNif("SceneNode_Model", groups(lods.size(), opaquePieces, transparentPieces));
outcome.files["model.nif"] = nif;
AddDownload(outcome.files, "model.nif", nif);
AddDownload(outcome.files, "model.lxfml", lxfml);
AddIcon(outcome.files, model, settings.icon);
{
const std::vector<std::vector<UgcModel::Mesh>> opaque{ UgcModel::Divide(preview.opaque) }, transparent{ UgcModel::Divide(preview.transparent) };
outcome.files["model.noao.nif"] = UgcFormats::WriteLodNif("SceneNode_Model", groups(1, opaque, transparent));
}
const auto iconStart = std::chrono::steady_clock::now();
AddIcon(outcome.files, iconModel, settings.icon, iconAo.empty() ? nullptr : &iconAo);
iconMs += Since(iconStart);
stats["ms"] = { { "build", std::lround(buildMs) }, { "hiddenSurfaces", std::lround(hsrMs) }, { "ambientOcclusion", std::lround(aoMs) },
{ "icon", std::lround(iconMs) }, { "total", std::lround(Since(started)) } };
stats["settings"] = { { "palette", settings.build.palette == UgcModel::ePalette::LU_TOOLBOX ? "lu_toolbox" : "brickdb" },
{ "colorVariation", settings.build.colorVariation }, { "transparentOpacity", settings.build.transparentOpacity },
{ "removeHiddenFaces", settings.optimize.removeHidden }, { "groundPlane", settings.optimize.groundPlane },
{ "ao", settings.ao.enabled }, { "aoDistance", settings.ao.distance }, { "aoSamples", settings.ao.samples }, { "aoStrength", settings.ao.strength } };
outcome.stats = stats.dump();
outcome.files["stats.json"] = outcome.stats;
outcome.ok = true;
return outcome;
}

26
dUgcServer/UgcJobs.h
View File

@@ -15,10 +15,22 @@
* the finished files. No database, network or CDClient: the main thread gathers the input and stores the result.
*/
namespace UgcJobs {
/**
* How models are made. The defaults are LU Toolbox's (Process Model, Bake Lighting and the icon renderer), see the
* parity table in docs/UgcServer.md.
*/
struct Settings {
UgcRender::OptimizeOptions optimize;
UgcModel::BuildOptions build; // palette, color variation, transparent opacity
std::vector<uint32_t> lods{ 0, 2 }; // brickprimitives levels made (LU Toolbox imports LOD 0 and 2; the client has no 3)
UgcModel::LodDistances lodDistances;
std::string shaderOpaque{ "01" }; // S<shader>_Opaque_...; transparent shapes are always S01
UgcRender::OptimizeOptions optimize; // hidden surface removal
UgcRender::AoOptions ao; // Bake Lighting (AO Only)
UgcRender::IconOptions icon;
bool iconCorrectColors{ true }; // the icon renderer's Correct Colors
float iconColorVariation{ 0.0f }; // percent; its Apply Color Variation is off
UgcRender::IconOptions modularIcon;
uint32_t maxBricks{}; // a model with more fails; 0: no limit
};
struct Outcome {
@@ -27,6 +39,7 @@ namespace UgcJobs {
std::string note; // what was odd but not fatal (missing bricks, ...)
UgcStorage::Files files; // name -> bytes, when ok
bool aoBaked{};
std::string stats; // stats.json: bricks, triangles before and after per LOD, timings
};
// The client's download of `data` (`name` + ".gz" and ".checksum") added to `files`
@@ -35,8 +48,15 @@ namespace UgcJobs {
// The LXFML of a ugc row's lxfml column (an sd0 stream, or plain LXFML); empty when it can't be read
std::string LxfmlFromBlob(const std::string& blob);
// A player model: the optimized .nif, its icon and its LXFML for download
Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings);
// A player model: the optimized .nif, its icon and its LXFML for download, and stats.json. `seed` picks the color
// variation's random numbers (the model's id, so making it again gives the same colors).
Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings, uint64_t seed = 0);
// How many bricks (parts) an LXFML has, counted cheaply (for the memory estimate before a job starts)
size_t CountParts(std::string_view lxfml);
// About how much memory making a model of `parts` bricks takes, in bytes
uint64_t EstimateMemory(size_t parts, const Settings& settings);
struct ModuleInput {
uint32_t lot{};

169
dUgcServer/UgcModel.cpp
View File

@@ -10,6 +10,7 @@
#include <glm/gtc/matrix_transform.hpp>
#include "NifFile.h"
#include "UgcPalette.h"
#include "tinyxml2.h"
namespace {
@@ -146,6 +147,11 @@ namespace UgcModel {
positions.insert(positions.end(), other.positions.begin(), other.positions.end());
normals.insert(normals.end(), other.normals.begin(), other.normals.end());
colors.insert(colors.end(), other.colors.begin(), other.colors.end());
if (!glow.empty() || !other.glow.empty()) {
glow.resize(base, glm::vec3(0.0f));
if (other.glow.empty()) glow.resize(positions.size(), glm::vec3(0.0f));
else glow.insert(glow.end(), other.glow.begin(), other.glow.end());
}
indices.reserve(indices.size() + other.indices.size());
for (const auto index : other.indices) indices.push_back(base + index);
}
@@ -172,24 +178,59 @@ namespace UgcModel {
return any;
}
Model Build(const std::vector<Part>& parts, UgcBricks::BrickLibrary& library) {
Model Build(const std::vector<Part>& parts, UgcBricks::BrickLibrary& library, const BuildOptions& options) {
Model model;
std::set<uint32_t> missing;
for (const auto& part : parts) {
const auto design = library.GetDesign(part.designId);
const bool luToolbox = options.palette == ePalette::LU_TOOLBOX;
bool anyGlow = false;
for (uint32_t brick = 0; brick < parts.size(); brick++) {
const auto& part = parts[brick];
const auto design = library.GetDesign(part.designId, options.lod);
if (!design || design->empty()) {
missing.insert(part.designId);
continue;
}
model.bricks++;
const auto materialOf = [&part](size_t index) {
auto id = index < part.materials.size() ? part.materials[index] : (part.materials.empty() ? 0 : part.materials[0]);
// Unknown colors are black in LU Toolbox (its name included, so black's variation too)
if (id == 0 || !UgcPalette::Linear(id)) id = UgcPalette::FALLBACK_ID;
return id;
};
// A brick is transparent only when all of its materials are (LU Toolbox's IS_TRANSPARENT)
bool transparent = true;
for (size_t index = 0; index < design->size(); index++) {
const auto id = index < part.materials.size() ? part.materials[index] : (part.materials.empty() ? 0 : part.materials[0]);
transparent = transparent && (luToolbox ? UgcPalette::IsTransparent(materialOf(index)) : library.GetMaterial(id).Transparent());
}
auto& mesh = transparent ? model.transparent : model.opaque;
const glm::mat3 normalMatrix = glm::transpose(glm::inverse(glm::mat3(part.transform)));
for (size_t index = 0; index < design->size(); index++) {
const auto& geometry = (*design)[index];
const auto materialId = index < part.materials.size() ? part.materials[index] : (part.materials.empty() ? 0 : part.materials[0]);
const auto material = library.GetMaterial(materialId);
auto& mesh = material.Transparent() ? model.transparent : model.opaque;
glm::vec3 linear{};
float alpha = 1.0f;
glm::vec3 glow(0.0f);
uint32_t colorId{};
if (luToolbox) {
colorId = materialOf(index);
linear = *UgcPalette::Linear(colorId, options.icon);
if (transparent) alpha = std::clamp(options.transparentOpacity / 100.0f, 0.0f, 1.0f);
if (!transparent) {
if (const auto g = UgcPalette::Glow(colorId)) glow = *g;
}
} else {
colorId = index < part.materials.size() ? part.materials[index] : (part.materials.empty() ? 0 : part.materials[0]);
const auto material = library.GetMaterial(colorId);
linear = UgcPalette::SrgbToLinear(glm::vec3(material.r, material.g, material.b) / 255.0f);
if (transparent) alpha = material.a / 255.0f;
}
if (options.colorVariation > 0.0f) {
const float variation = options.colorVariation * (luToolbox ? UgcPalette::VariationScale(colorId) : 1.0f);
linear = UgcPalette::ApplyVariation(linear, variation, UgcPalette::BrickRandom(options.seed, brick, colorId));
}
const glm::vec4 color(UgcPalette::LinearToSrgb(linear), alpha);
anyGlow = anyGlow || glow != glm::vec3(0.0f);
const auto base = static_cast<uint32_t>(mesh.positions.size());
const auto color = ToColor(material);
const size_t vertexCount = geometry.positions.size() / 3;
for (size_t v = 0; v < vertexCount; v++) {
const glm::vec3 position(geometry.positions[v * 3], geometry.positions[v * 3 + 1], geometry.positions[v * 3 + 2]);
@@ -200,14 +241,49 @@ namespace UgcModel {
mesh.positions.push_back(glm::vec3(part.transform * glm::vec4(position, 1.0f)));
mesh.normals.push_back(normal);
mesh.colors.push_back(color);
if (&mesh == &model.opaque) model.opaque.glow.push_back(glow);
}
for (const auto i : geometry.indices) mesh.indices.push_back(base + i);
}
}
if (!anyGlow) model.opaque.glow.clear();
else model.opaque.glow.resize(model.opaque.positions.size(), glm::vec3(0.0f));
model.missingDesigns.assign(missing.begin(), missing.end());
return model;
}
std::vector<std::pair<float, float>> LodRanges(const std::vector<uint32_t>& used, const LodDistances& d) {
// LU Toolbox's setup_lod_data ("DYNAMIC LOD HELL"), by the set of levels there are
const std::set<uint32_t> set(used.begin(), used.end());
const auto is = [&set](std::initializer_list<uint32_t> levels) { return set == std::set<uint32_t>(levels); };
std::vector<std::pair<float, float>> ranges;
for (const auto level : used) {
std::pair<float, float> range{ 0.0f, 0.0f };
if (set.size() == 1) {
range = { d.lod0, d.cull };
} else if (level == 0) {
range.first = d.lod0;
if (is({ 0, 2 }) || is({ 0, 2, 3 })) range.second = d.lod2;
else if (is({ 0, 3 })) range.second = d.lod3;
else range.second = d.lod1;
} else if (level == 1) {
if (is({ 0, 1 })) range = { d.lod1, d.cull };
else if (is({ 1, 2 }) || is({ 1, 2, 3 })) range = { d.lod0, d.lod2 };
else if (is({ 0, 1, 3 })) range = { d.lod1, d.lod3 };
else if (is({ 1, 3 })) range = { d.lod0, d.lod3 };
else if (is({ 0, 1, 2 }) || is({ 0, 1, 2, 3 })) range = { d.lod1, d.lod2 };
} else if (level == 2) {
if (is({ 0, 2 }) || is({ 1, 2 }) || is({ 0, 1, 2 })) range = { d.lod2, d.cull };
else if (is({ 2, 3 })) range = { d.lod0, d.lod3 };
else if (is({ 0, 2, 3 }) || is({ 1, 2, 3 }) || is({ 0, 1, 2, 3 })) range = { d.lod2, d.lod3 };
} else if (level == 3) {
range = { d.lod3, d.cull };
}
ranges.push_back(range);
}
return ranges;
}
Model FromNif(const NifFile::Model& nif) {
Model model;
for (const auto& source : nif.meshes) {
@@ -241,6 +317,26 @@ namespace UgcModel {
return model;
}
void KeepTriangles(Mesh& mesh, const std::vector<bool>& keep) {
Mesh kept;
std::vector<uint32_t> remap(mesh.positions.size(), UINT32_MAX);
for (size_t t = 0; t < mesh.TriangleCount(); t++) {
if (t >= keep.size() || !keep[t]) continue;
for (int k = 0; k < 3; k++) {
const auto source = mesh.indices[t * 3 + k];
if (remap[source] == UINT32_MAX) {
remap[source] = static_cast<uint32_t>(kept.positions.size());
kept.positions.push_back(mesh.positions[source]);
if (source < mesh.normals.size()) kept.normals.push_back(mesh.normals[source]);
if (source < mesh.colors.size()) kept.colors.push_back(mesh.colors[source]);
if (source < mesh.glow.size()) kept.glow.push_back(mesh.glow[source]);
}
kept.indices.push_back(remap[source]);
}
}
mesh = std::move(kept);
}
std::vector<Mesh> Split(const Mesh& mesh, size_t maxVertices, size_t maxTriangles) {
std::vector<Mesh> pieces;
if (mesh.positions.size() <= maxVertices && mesh.TriangleCount() <= maxTriangles) {
@@ -265,6 +361,7 @@ namespace UgcModel {
current.positions.push_back(mesh.positions[source]);
if (source < mesh.normals.size()) current.normals.push_back(mesh.normals[source]);
if (source < mesh.colors.size()) current.colors.push_back(mesh.colors[source]);
if (source < mesh.glow.size()) current.glow.push_back(mesh.glow[source]);
}
current.indices.push_back(it->second);
}
@@ -272,4 +369,62 @@ namespace UgcModel {
flush();
return pieces;
}
std::vector<Mesh> Divide(const Mesh& mesh, size_t maxVertices, size_t maxTriangles) {
if (mesh.Empty()) return {};
if (mesh.positions.size() <= maxVertices && mesh.TriangleCount() <= maxTriangles) return { mesh };
// Connected pieces (vertices joined by triangles), so a brick's faces stay together
std::vector<uint32_t> parent(mesh.positions.size());
for (uint32_t i = 0; i < parent.size(); i++) parent[i] = i;
const std::function<uint32_t(uint32_t)> find = [&](uint32_t x) {
while (parent[x] != x) x = parent[x] = parent[parent[x]];
return x;
};
for (size_t i = 0; i + 2 < mesh.indices.size(); i += 3) {
const auto a = find(mesh.indices[i]);
parent[find(mesh.indices[i + 1])] = a;
parent[find(mesh.indices[i + 2])] = a;
}
// divide_mesh: vertices below the mean along the longest side of the bounds, and everything linked to them
glm::vec3 min = mesh.positions[0], max = mesh.positions[0], mean(0.0f);
for (const auto& p : mesh.positions) {
min = glm::min(min, p);
max = glm::max(max, p);
mean += p;
}
mean /= static_cast<float>(mesh.positions.size());
const auto size = max - min;
const int axis = size.x >= size.y && size.x >= size.z ? 0 : size.y >= size.z ? 1 : 2;
std::vector<bool> below(mesh.positions.size(), false);
for (uint32_t v = 0; v < mesh.positions.size(); v++) {
if (mesh.positions[v][axis] < mean[axis]) below[find(v)] = true;
}
Mesh halves[2];
std::vector<uint32_t> remap(mesh.positions.size(), UINT32_MAX);
for (size_t i = 0; i + 2 < mesh.indices.size(); i += 3) {
auto& half = halves[below[find(mesh.indices[i])] ? 1 : 0];
for (int k = 0; k < 3; k++) {
const auto source = mesh.indices[i + k];
if (remap[source] == UINT32_MAX) {
remap[source] = static_cast<uint32_t>(half.positions.size());
half.positions.push_back(mesh.positions[source]);
if (source < mesh.normals.size()) half.normals.push_back(mesh.normals[source]);
if (source < mesh.colors.size()) half.colors.push_back(mesh.colors[source]);
if (source < mesh.glow.size()) half.glow.push_back(mesh.glow[source]);
}
half.indices.push_back(remap[source]);
}
}
// LU Toolbox gives up below a 10% share; this splits the old way then
const float share = static_cast<float>(halves[1].positions.size()) / static_cast<float>(mesh.positions.size());
if (std::min(share, 1.0f - share) < 0.1f) return Split(mesh, maxVertices, maxTriangles);
std::vector<Mesh> pieces;
for (const auto& half : halves) {
for (auto& piece : Divide(half, maxVertices, maxTriangles)) pieces.push_back(std::move(piece));
}
return pieces;
}
}

48
dUgcServer/UgcModel.h
View File

@@ -33,6 +33,7 @@ namespace UgcModel {
std::vector<glm::vec3> positions;
std::vector<glm::vec3> normals;
std::vector<glm::vec4> colors; // sRGB, 0..1, alpha is opacity
std::vector<glm::vec3> glow; // linear glow color per vertex (LU Toolbox's "Glow" layer); empty when nothing glows
std::vector<uint32_t> indices;
size_t TriangleCount() const { return indices.size() / 3; }
@@ -52,12 +53,55 @@ namespace UgcModel {
bool Bounds(glm::vec3& min, glm::vec3& max) const;
};
// The mesh of a model's parts
Model Build(const std::vector<Part>& parts, UgcBricks::BrickLibrary& library);
enum class ePalette {
LU_TOOLBOX, // LU Toolbox's colors (UgcPalette), what its importer colors models with
BRICKDB, // the brick database's Materials.xml
};
struct BuildOptions {
ePalette palette{ ePalette::LU_TOOLBOX };
float colorVariation{ 5.0f }; // percent, 0: none (LU Toolbox: Apply Color Variation, 5%)
uint64_t seed{}; // of the variation's random numbers
float transparentOpacity{ 58.82f }; // percent, transparent bricks' vertex alpha (LU Toolbox palette only)
bool icon{}; // the icon renderer's color corrections
uint32_t lod{}; // brickprimitives level
};
/**
* The mesh of a model's parts, colored as LU Toolbox's Process Model does: a brick is transparent only when all of
* its materials are, each material of each brick has its brightness shifted by the color variation (the same
* random number for a brick's material in every LOD and every time), vertex colors are sRGB with alpha 1 for
* opaque bricks and the transparent opacity for transparent ones.
*/
Model Build(const std::vector<Part>& parts, UgcBricks::BrickLibrary& library, const BuildOptions& options = {});
/**
* The distance range (near, far) of each LOD LU Toolbox makes, for the brickprimitives levels in `used` (0 to 3),
* from its settings (lod0..lod3, cull): its setup_lod_data, which picks the ranges by which levels are there.
* {0, 0} for a level it has no range for.
*/
struct LodDistances {
float lod0{ 0.0f };
float lod1{ 50.0f };
float lod2{ 100.0f };
float lod3{ 280.0f };
float cull{ 10000.0f };
};
std::vector<std::pair<float, float>> LodRanges(const std::vector<uint32_t>& used, const LodDistances& distances);
/**
* Splits a mesh the way LU Toolbox's divide_mesh does while it has too many vertices (or triangles): at the mean
* of its vertices along its longest side, keeping connected pieces whole. Falls back to Split when that can't
* divide it.
*/
std::vector<Mesh> Divide(const Mesh& mesh, size_t maxVertices = 65535, size_t maxTriangles = 65535);
// A client .nif's meshes as one model (vertex colors times material color; transparent when blended)
Model FromNif(const NifFile::Model& nif);
// Keeps the triangles whose flag is set (and the vertices they use)
void KeepTriangles(Mesh& mesh, const std::vector<bool>& keep);
// Splits a mesh into pieces the .nif format can hold (at most `maxVertices` vertices and `maxTriangles` triangles)
std::vector<Mesh> Split(const Mesh& mesh, size_t maxVertices = 65535, size_t maxTriangles = 65535);
}

219
dUgcServer/UgcPalette.cpp Normal file
View File

@@ -0,0 +1,219 @@
#include "UgcPalette.h"
#include <algorithm>
#include <cmath>
#include <unordered_map>
namespace {
using Table = std::unordered_map<uint32_t, glm::vec3>;
glm::vec3 FromSrgb(float r, float g, float b) {
return UgcPalette::SrgbToLinear(glm::vec3(r, g, b));
}
// Colors are linear, as LU Toolbox keeps them
const Table& Opaque() {
static const Table table = [] {
Table t = {
{ 1, { 0.904661f, 0.904661f, 0.904661f } },
{ 5, { 0.693872f, 0.491021f, 0.194618f } },
{ 18, { 0.672443f, 0.168269f, 0.051269f } },
{ 21, { 0.730461f, 0.0f, 0.004025f } },
{ 23, { 0.0f, 0.095307f, 0.391572f } },
{ 24, { 0.991102f, 0.552011f, 0.0f } },
{ 26, { 0.006f, 0.006f, 0.006f } },
{ 28, { 0.0f, 0.198069f, 0.021219f } },
{ 37, { 0.0f, 0.304987f, 0.017642f } },
{ 38, { 0.391572f, 0.046665f, 0.007499f } },
{ 50, { 0.7399f, 0.7399f, 0.7399f } },
{ 102, { 0.06301f, 0.262251f, 0.564712f } },
{ 106, { 0.799103f, 0.124772f, 0.009134f } },
{ 107, { 0.002732f, 0.462077f, 0.462077f } },
{ 119, { 0.296138f, 0.47932f, 0.003035f } },
{ 120, { 0.672444f, 0.768151f, 0.262251f } },
{ 124, { 0.332452f, 0.0f, 0.147027f } },
{ 135, { 0.111932f, 0.174647f, 0.262251f } },
{ 138, { 0.262251f, 0.177888f, 0.084376f } },
{ 140, { 0.0f, 0.0185f, 0.052861f } },
{ 141, { 0.0f, 0.03434f, 0.008023f } },
{ 151, { 0.114435f, 0.223228f, 0.130137f } },
{ 154, { 0.215861f, 0.002428f, 0.01096f } },
{ 191, { 0.887923f, 0.318547f, 0.0f } },
{ 192, { 0.104617f, 0.011612f, 0.003677f } },
{ 194, { 0.332452f, 0.283149f, 0.283149f } },
{ 199, { 0.072272f, 0.082283f, 0.093059f } },
{ 208, { 0.768151f, 0.768151f, 0.693872f } },
{ 212, { 0.242281f, 0.520996f, 0.83077f } },
{ 221, { 0.730461f, 0.038204f, 0.258183f } },
{ 222, { 0.846873f, 0.341914f, 0.53948f } },
{ 226, { 1.0f, 0.768151f, 0.141263f } },
{ 268, { 0.066626f, 0.011612f, 0.341914f } },
{ 283, { 0.913099f, 0.533276f, 0.250158f } },
{ 294, { 0.991102f, 0.973445f, 0.665388f } },
{ 308, { 0.03434f, 0.015209f, 0.0f } },
{ 329, { 1.0f, 1.0f, 1.0f } },
{ 330, { 0.184475f, 0.184475f, 0.076185f } },
{ 9013, { 1.0f, 0.009721f, 0.020289f } },
{ 9014, { 0.799103f, 0.088655f, 0.0f } },
{ 9015, { 1.0f, 0.630757f, 0.033105f } },
{ 9016, { 0.012983f, 1.0f, 0.090842f } },
{ 9017, { 0.059511f, 0.768152f, 0.913099f } },
{ 9018, { 0.0f, 0.226966f, 1.0f } },
{ 9019, { 0.40724f, 0.012983f, 1.0f } },
{ 9020, { 0.686686f, 0.000303f, 1.0f } },
};
// LDD colors LU doesn't have, as the nearest LU color
const std::pair<uint32_t, uint32_t> aliases[] = {
{ 0, 26 }, { 2, 194 }, { 3, 5 }, { 4, 106 }, { 6, 119 }, { 9, 222 }, { 11, 212 }, { 12, 106 }, { 13, 191 }, { 19, 191 },
{ 22, 221 }, { 25, 192 }, { 27, 199 }, { 29, 151 }, { 36, 283 }, { 39, 194 }, { 45, 212 }, { 100, 283 }, { 101, 106 },
{ 103, 194 }, { 104, 268 }, { 105, 191 }, { 110, 23 }, { 112, 23 }, { 115, 119 }, { 116, 107 }, { 118, 212 }, { 326, 120 },
{ 125, 283 }, { 128, 38 }, { 133, 106 }, { 134, 119 }, { 136, 135 }, { 153, 138 }, { 180, 191 }, { 195, 23 }, { 196, 23 },
{ 198, 124 }, { 216, 154 }, { 217, 138 }, { 218, 124 }, { 219, 268 }, { 223, 222 }, { 232, 212 }, { 233, 37 }, { 295, 222 },
{ 312, 138 }, { 321, 102 }, { 322, 212 }, { 323, 208 }, { 324, 124 }, { 325, 222 },
};
for (const auto& [id, target] : aliases) t[id] = t.at(target);
return t;
}();
return table;
}
const Table& Transparent() {
static const Table table = [] {
Table t = {
{ 20, { 0.930111f, 0.672443f, 0.250158f } },
{ 40, { 0.854993f, 0.854993f, 0.854993f } },
{ 41, FromSrgb(0.674509f, 0.0f, 0.0f) },
{ 42, FromSrgb(0.244106f, 0.720966f, 0.772058f) },
{ 43, FromSrgb(0.031372f, 0.285668f, 0.643137f) },
{ 44, FromSrgb(0.858f, 0.771375f, 0.0f) },
{ 47, FromSrgb(0.986f, 0.336526f, 0.120035f) },
{ 48, FromSrgb(0.0f, 0.391f, 0.0f) },
{ 49, FromSrgb(0.697f, 1.0f, 0.0f) },
{ 111, FromSrgb(0.741177f, 0.670588f, 0.639216f) },
{ 113, FromSrgb(0.754717f, 0.060520f, 0.541647f) },
{ 126, FromSrgb(0.267974f, 0.196078f, 0.627451f) },
{ 143, FromSrgb(0.325985f, 0.551358f, 0.821f) },
{ 182, FromSrgb(0.913726f, 0.524575f, 0.0156863f) },
{ 311, FromSrgb(0.454640f, 0.788235f, 0.0980392f) },
};
const std::pair<uint32_t, uint32_t> aliases[] = { { 157, 44 }, { 230, 113 }, { 231, 182 }, { 234, 44 }, { 284, 126 }, { 285, 111 }, { 293, 43 } };
for (const auto& [id, target] : aliases) t[id] = t.at(target);
return t;
}();
return table;
}
const Table& GlowColors() {
static const Table table = [] {
const auto& opaque = Opaque();
Table t = { { 50, { 0.401978f, 0.401978f, 0.401978f } } };
for (const uint32_t id : { 329u, 294u, 9013u, 9014u, 9015u, 9016u, 9017u, 9018u, 9019u, 9020u }) t[id] = opaque.at(id);
const std::pair<uint32_t, uint32_t> aliases[] = {
{ 9000, 9020 }, { 9002, 9016 }, { 9004, 9018 }, { 9008, 9013 }, { 9009, 9014 }, { 9010, 9016 }, { 9011, 9017 }, { 9012, 9019 },
{ 9021, 329 }, { 9022, 50 }, { 9023, 9013 }, { 9024, 9014 }, { 9025, 9016 }, { 9026, 9018 }, { 9027, 329 },
};
for (const auto& [id, target] : aliases) t[id] = t.at(target);
return t;
}();
return table;
}
const Table& Metallic() {
static const Table table = [] {
Table t;
const auto add = [&t](std::initializer_list<uint32_t> ids, glm::vec3 color) { for (const auto id : ids) t[id] = color; };
add({ 131, 150, 179, 298, 315 }, { 0.262251f, 0.296138f, 0.296138f });
add({ 139, 187, 300 }, { 0.174648f, 0.066626f, 0.029557f });
add({ 148 }, { 0.06301f, 0.051269f, 0.043735f });
add({ 149 }, { 0.006f, 0.006f, 0.006f });
add({ 184 }, { 0.238095f, 0.00907f, 0.00907f });
add({ 186, 200 }, { 0.081104f, 0.252379f, 0.045668f });
add({ 145, 185 }, { 0.104617f, 0.177888f, 0.278894f });
add({ 309, 183 }, { 0.617207f, 0.617207f, 0.617207f });
add({ 297, 147, 189 }, { 0.401978f, 0.212231f, 0.027321f });
add({ 310, 127 }, { 0.737911f, 0.533276f, 0.181164f });
return t;
}();
return table;
}
const std::unordered_map<uint32_t, float>& CustomVariation() {
static const std::unordered_map<uint32_t, float> table = {
{ 1, 1.3f }, { 21, 1.4f }, { 23, 1.25f }, { 24, 1.5f }, { 26, 0.4f }, { 28, 0.8f }, { 37, 0.8f }, { 135, 0.85f }, { 141, 0.7f },
{ 199, 0.7f }, { 192, 0.75f }, { 212, 1.25f }, { 222, 1.05f }, { 226, 1.75f }, { 283, 1.15f }, { 308, 0.85f }, { 323, 1.4f }, { 326, 1.75f },
};
return table;
}
uint64_t SplitMix(uint64_t x) {
x += 0x9E3779B97F4A7C15ull;
x = (x ^ (x >> 30)) * 0xBF58476D1CE4E5B9ull;
x = (x ^ (x >> 27)) * 0x94D049BB133111EBull;
return x ^ (x >> 31);
}
}
namespace UgcPalette {
std::optional<glm::vec3> Linear(uint32_t id, bool icon) {
if (icon) {
// ICON_MATERIALS_OPAQUE
if (id == 1) return FromSrgb(0.7f, 0.7f, 0.7f);
if (id == 26) return FromSrgb(0.01f, 0.01f, 0.01f);
}
// The importer loads opaque, then transparent, metallic and glow colors, the first one of an id winning
for (const auto* table : { &Opaque(), &Transparent(), &Metallic(), &GlowColors() }) {
if (const auto it = table->find(id); it != table->end()) return it->second;
}
return std::nullopt;
}
bool IsTransparent(uint32_t id) {
return Transparent().contains(id);
}
bool IsMetallic(uint32_t id) {
return Metallic().contains(id);
}
std::optional<glm::vec3> Glow(uint32_t id) {
const auto it = GlowColors().find(id);
return it != GlowColors().end() ? std::optional(it->second) : std::nullopt;
}
float VariationScale(uint32_t id) {
const auto it = CustomVariation().find(id);
return it != CustomVariation().end() ? it->second : 1.0f;
}
float SrgbToLinear(float srgb) {
return srgb <= 0.0404482362771082f ? srgb / 12.92f : std::pow((srgb + 0.055f) / 1.055f, 2.4f);
}
float LinearToSrgb(float linear) {
return linear > 0.0031308f ? 1.055f * std::pow(linear, 1.0f / 2.4f) - 0.055f : 12.92f * linear;
}
glm::vec3 SrgbToLinear(const glm::vec3& srgb) {
return { SrgbToLinear(srgb.r), SrgbToLinear(srgb.g), SrgbToLinear(srgb.b) };
}
glm::vec3 LinearToSrgb(const glm::vec3& linear) {
return { LinearToSrgb(linear.r), LinearToSrgb(linear.g), LinearToSrgb(linear.b) };
}
glm::vec3 ApplyVariation(const glm::vec3& color, float variationPercent, float random) {
const float value = std::max({ color.r, color.g, color.b });
float gamma = std::pow(std::max(value, 0.0f), 1.0f / 2.224f);
const float range = variationPercent / 200.0f;
gamma += -range + random * 2.0f * range;
const float newValue = std::pow(std::clamp(gamma, 0.0f, 1.0f), 2.224f);
// Setting an HSV value keeps hue and saturation: the channels scale together (a black color becomes grey)
if (value <= 0.0f) return glm::vec3(newValue);
return color * (newValue / value);
}
float BrickRandom(uint64_t seed, uint32_t brick, uint32_t material) {
const auto bits = SplitMix(SplitMix(seed ^ (static_cast<uint64_t>(brick) << 32)) ^ material);
return static_cast<float>(bits >> 40) / static_cast<float>(1ull << 24);
}
}

51
dUgcServer/UgcPalette.h Normal file
View File

@@ -0,0 +1,51 @@
#pragma once
#include <cstdint>
#include <optional>
#include <glm/glm.hpp>
/**
* LU Toolbox's color palette (its materials/__init__.py), which it colors imported models with instead of the brick
* database's Materials.xml: LU's colors in linear RGB, the LDD colors LU doesn't have mapped onto the nearest LU one,
* which colors are transparent, glow and metallic, how much each color varies from brick to brick, and the icon
* renderer's corrections. Also the brick to brick color variation itself (process_model.py apply_color_variation).
* Pure.
*/
namespace UgcPalette {
// A material id's color in linear RGB; nullopt when LU Toolbox doesn't know it (it uses black, 26, then).
// `icon`: with the icon renderer's corrections (white and black are toned down).
std::optional<glm::vec3> Linear(uint32_t id, bool icon = false);
bool IsTransparent(uint32_t id);
bool IsMetallic(uint32_t id);
// The glow color (linear) of a glowing material, nullopt for the others
std::optional<glm::vec3> Glow(uint32_t id);
// How much of the color variation a color gets (CUSTOM_VARIATION: black varies less, orange more); 1 by default
float VariationScale(uint32_t id);
// The color LU Toolbox falls back to for unknown ids
constexpr uint32_t FALLBACK_ID = 26;
// sRGB <-> linear, exactly as LU Toolbox converts (color_conversions.py)
float SrgbToLinear(float srgb);
float LinearToSrgb(float linear);
glm::vec3 SrgbToLinear(const glm::vec3& srgb);
glm::vec3 LinearToSrgb(const glm::vec3& linear);
/**
* LU Toolbox's color variation of one color: its HSV value, taken to a 1/2.224 gamma, shifted by
* `random` (0..1, mapped to -variation/200 .. +variation/200, variation in percent), clamped to 0..1 and taken back.
* Hue and saturation stay. `color` is linear RGB.
*/
glm::vec3 ApplyVariation(const glm::vec3& color, float variationPercent, float random);
/**
* The random number (0..1) of one brick's material: the same for the same seed, brick and material every time,
* so making a model again gives the same colors, and the same in every LOD (LU Toolbox restarts its random sequence
* for each LOD). `brick` is the brick's index in the LXFML.
*/
float BrickRandom(uint64_t seed, uint32_t brick, uint32_t material);
}

157
dUgcServer/UgcProcessor.cpp
View File

@@ -4,8 +4,19 @@
#include "Database.h"
#include "Logger.h"
#include "UgcCdClient.h"
#include "UgcThrottle.h"
#include "json.hpp"
#include <ctime>
#include <fstream>
#include <thread>
#if defined(__linux__)
#include <malloc.h>
#include <sys/resource.h>
#include <sys/syscall.h>
#include <unistd.h>
#endif
namespace {
constexpr size_t MAX_ERROR_LENGTH = 1000; // process_error holds 1024
constexpr size_t LOG_LENGTH = 50;
@@ -16,6 +27,22 @@ namespace {
return kind == UgcStorage::Kind::MODEL ? "model" : "modular";
}
// The process's CPU time (user and system, every thread), seconds
double ProcessCpuSeconds() {
timespec ts{};
if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts) == 0) return static_cast<double>(ts.tv_sec) + ts.tv_nsec / 1e9;
return 0.0;
}
void ApplyNice(int nice) {
#if defined(__linux__)
// Per thread on Linux: only the calling worker's priority changes
setpriority(PRIO_PROCESS, static_cast<id_t>(syscall(SYS_gettid)), std::clamp(nice, 0, 19));
#else
(void)nice;
#endif
}
int64_t UnixNow() {
return std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();
}
@@ -24,6 +51,41 @@ namespace {
UgcProcessor::UgcProcessor(Config config, UgcStorage& storage, UgcBricks::BrickLibrary& library, UgcJobs::Settings settings)
: m_Config(config), m_Storage(storage), m_Library(library), m_Settings(std::move(settings)) {}
uint64_t UgcProcessor::ResidentBytes() {
#if defined(__linux__)
std::ifstream statm("/proc/self/statm");
uint64_t size = 0, resident = 0;
if (statm >> size >> resident) return resident * static_cast<uint64_t>(sysconf(_SC_PAGESIZE));
#endif
return 0;
}
bool UgcProcessor::Throttled() const {
const auto last = UgcThrottle::GetStats().lastSleepUnixMs;
return last > 0 && UnixNow() * 1000 - last < 5000;
}
void UgcProcessor::Configure(UgcJobs::Settings settings, Limits limits) {
{
std::lock_guard lock(m_Mutex);
m_Settings = std::move(settings);
m_Limits = limits;
}
UgcThrottle::SetBudget(limits.maxCpus);
m_Wake.notify_all();
}
void UgcProcessor::SampleUsage() {
const auto now = std::chrono::steady_clock::now();
const double cpu = ProcessCpuSeconds();
if (m_CpuSampled.time_since_epoch().count() != 0) {
const double elapsed = std::chrono::duration<double>(now - m_CpuSampled).count();
if (elapsed > 0.0) m_CpuPercent = (cpu - m_CpuSeconds) / elapsed * 100.0;
}
m_CpuSampled = now;
m_CpuSeconds = cpu;
}
UgcProcessor::~UgcProcessor() {
Stop();
}
@@ -50,27 +112,54 @@ void UgcProcessor::Stop() {
}
void UgcProcessor::Worker() {
int appliedNice = 0;
while (true) {
Job job;
UgcJobs::Settings settings;
int nice = 0;
{
std::unique_lock lock(m_Mutex);
m_Wake.wait(lock, [this] { return m_Stopping || !m_Jobs.empty(); });
// The first job that fits the memory budget beside the ones running; one that is too big alone runs
// when nothing else does
auto pick = m_Jobs.end();
m_Wake.wait(lock, [this, &pick] {
if (m_Stopping) return true;
pick = m_Jobs.end();
for (auto it = m_Jobs.begin(); it != m_Jobs.end(); ++it) {
if (m_Limits.maxMemoryBytes == 0 || m_Active == 0 || m_MemoryInUse + it->memory <= m_Limits.maxMemoryBytes) {
pick = it;
break;
}
}
if (pick == m_Jobs.end() && !m_Jobs.empty()) m_Waiting++;
return pick != m_Jobs.end();
});
if (m_Stopping) return;
job = std::move(m_Jobs.front());
m_Jobs.pop_front();
job = std::move(*pick);
m_Jobs.erase(pick);
m_Active++;
m_MemoryInUse += job.memory;
settings = m_Settings;
nice = m_Limits.nice;
}
if (nice != appliedNice) {
ApplyNice(nice);
appliedNice = nice;
}
UgcThrottle::Begin();
const auto start = std::chrono::steady_clock::now();
Done done{ job.kind, job.id, job.attempts };
try {
done.outcome = job.kind == Kind::MODEL
? UgcJobs::ProcessModel(job.blob, m_Library, m_Settings)
: UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), m_Settings);
? UgcJobs::ProcessModel(job.blob, m_Library, settings, static_cast<uint64_t>(job.id))
: UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings);
} catch (const std::exception& ex) {
done.outcome.ok = false;
done.outcome.error = std::string("crashed: ") + ex.what();
}
job.blob.clear();
job.blob.shrink_to_fit();
if (done.outcome.ok) {
std::string error;
const auto bytes = m_Storage.Write(job.kind, job.id, done.outcome.files, error);
@@ -83,20 +172,42 @@ void UgcProcessor::Worker() {
}
done.outcome.files.clear();
done.milliseconds = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count();
UgcThrottle::Checkpoint();
std::lock_guard lock(m_Mutex);
m_Active--;
m_Done.push_back(std::move(done));
{
std::lock_guard lock(m_Mutex);
m_Active--;
m_MemoryInUse -= std::min(m_MemoryInUse, job.memory);
m_Done.push_back(std::move(done));
}
#if defined(__linux__) && defined(__GLIBC__)
// Give the big buffers of the job back to the system
malloc_trim(0);
#endif
m_Wake.notify_all();
}
}
void UgcProcessor::Poll() {
size_t queued = 0, active = 0;
Limits limits;
UgcJobs::Settings settings;
{
std::lock_guard lock(m_Mutex);
queued = m_Jobs.size();
active = m_Active;
limits = m_Limits;
settings = m_Settings;
}
const auto now = std::time(nullptr);
std::tm local{};
#if defined(_WIN32)
localtime_s(&local, &now);
#else
localtime_r(&now, &local);
#endif
m_Paused = UgcThrottle::InHours(local.tm_hour, limits.pauseFromHour, limits.pauseToHour);
if (m_Paused) return;
// Enough to keep every worker busy until the next poll
const size_t wanted = std::max<size_t>(m_Threads.size() * 2, m_Config.pollBatch);
if (queued + active >= wanted) return;
@@ -105,11 +216,16 @@ void UgcProcessor::Poll() {
std::vector<Job> jobs;
for (auto& model : Database::Get()->GetUgcModelsToProcess(limit)) {
if (m_InFlight.contains({ Kind::MODEL, model.id })) continue;
jobs.push_back(Job{ Kind::MODEL, model.id, model.attempts, std::move(model.lxfml) });
Job job{ Kind::MODEL, model.id, model.attempts, UgcJobs::LxfmlFromBlob(model.lxfml) };
if (job.blob.empty()) job.blob = std::move(model.lxfml); // the worker reports it can't be read
job.parts = UgcJobs::CountParts(job.blob);
job.memory = UgcJobs::EstimateMemory(job.parts, settings);
jobs.push_back(std::move(job));
}
for (auto& build : Database::Get()->GetModularBuildsToProcess(limit)) {
if (m_InFlight.contains({ Kind::MODULAR, build.id })) continue;
Job job{ Kind::MODULAR, build.id, build.attempts };
job.memory = UgcJobs::EstimateMemory(64, settings);
std::string error;
if (!UgcCdClient::GatherModular(build.modules, job.modular, error)) {
// Nothing a worker could do: record it right away
@@ -171,6 +287,7 @@ void UgcProcessor::Collect() {
void UgcProcessor::Update() {
Collect();
const auto now = std::chrono::steady_clock::now();
if (now - m_CpuSampled >= std::chrono::seconds(2)) SampleUsage();
if (now >= m_NextPoll) {
m_NextPoll = now + std::chrono::milliseconds(m_Config.pollIntervalMs);
Poll();
@@ -220,7 +337,29 @@ nlohmann::json UgcProcessor::Status() const {
status["queued"] = m_Jobs.size();
status["active"] = m_Active;
}
Limits limits;
{
std::lock_guard lock(m_Mutex);
limits = m_Limits;
status["jobMemoryBytes"] = m_MemoryInUse;
status["memoryWaits"] = m_Waiting;
}
status["workers"] = m_Threads.size();
const auto throttle = UgcThrottle::GetStats();
status["usage"] = {
{ "cpuPercent", std::lround(m_CpuPercent) }, // of one core
{ "cores", std::thread::hardware_concurrency() },
{ "residentBytes", ResidentBytes() },
{ "throttled", Throttled() },
{ "throttledMs", throttle.sleptMs },
{ "paused", m_Paused },
};
status["limits"] = {
{ "maxCpus", limits.maxCpus },
{ "maxMemoryBytes", limits.maxMemoryBytes },
{ "nice", limits.nice },
{ "pauseHours", limits.pauseFromHour >= 0 ? std::to_string(limits.pauseFromHour) + "-" + std::to_string(limits.pauseToHour) : "" },
};
status["made"] = m_Made;
status["failed"] = m_Failed;
status["evicted"] = m_Evicted;

34
dUgcServer/UgcProcessor.h
View File

@@ -36,10 +36,24 @@ public:
size_t threads{ 2 };
};
/**
* What the workers may use; changed while running (Configure) when the settings are reloaded.
*/
struct Limits {
double maxCpus{}; // CPUs the workers may use together on average (UgcThrottle); 0: no limit
uint64_t maxMemoryBytes{}; // estimated memory of the jobs running at once; 0: no limit
int nice{}; // the workers' scheduling priority (Linux nice, 0 to 19)
int pauseFromHour{ -1 }; // local hours in which no new jobs start (from, to; -1: never)
int pauseToHour{ -1 };
};
UgcProcessor(Config config, UgcStorage& storage, UgcBricks::BrickLibrary& library, UgcJobs::Settings settings);
~UgcProcessor();
void Start();
// Main thread: new settings and limits (config reload)
void Configure(UgcJobs::Settings settings, Limits limits);
// Waits for the jobs that are running; queued ones are dropped (they stay pending in the database)
void Stop();
@@ -62,6 +76,11 @@ public:
size_t Queued() const { std::lock_guard lock(m_Mutex); return m_Jobs.size(); }
size_t Busy() const { std::lock_guard lock(m_Mutex); return m_Active; }
size_t Threads() const { return m_Config.threads; }
// Main thread: the process's CPU use (percent of one core) and resident memory, and the running jobs' estimate
double CpuPercent() const { return m_CpuPercent; }
static uint64_t ResidentBytes();
uint64_t JobMemory() const { std::lock_guard lock(m_Mutex); return m_MemoryInUse; }
bool Throttled() const;
// Main thread: totals since the start and the files' size (traffic reports, then the dashboard and /metrics)
uint64_t Made() const { return m_Made; }
@@ -75,8 +94,10 @@ private:
Kind kind{};
LWOOBJID id{};
uint32_t attempts{};
std::string blob; // models: the stored LXFML
std::string blob; // models: the LXFML
UgcJobs::ModularInput modular; // modular builds
uint64_t memory{}; // estimated bytes it needs
size_t parts{};
};
struct Done {
@@ -92,17 +113,26 @@ private:
void Collect();
void Record(const Done& done);
void Worker();
void SampleUsage();
Config m_Config;
UgcStorage& m_Storage;
UgcBricks::BrickLibrary& m_Library;
UgcJobs::Settings m_Settings;
UgcJobs::Settings m_Settings; // guarded by m_Mutex (workers copy it per job)
Limits m_Limits; // guarded by m_Mutex
mutable std::mutex m_Mutex;
std::condition_variable m_Wake;
std::deque<Job> m_Jobs;
std::deque<Done> m_Done;
size_t m_Active{};
uint64_t m_MemoryInUse{}; // estimates of the running jobs
uint64_t m_Waiting{}; // times a job waited for memory
bool m_Paused{}; // main thread: in the pause hours
// Main thread: process CPU use, measured between status reads
std::chrono::steady_clock::time_point m_CpuSampled{};
double m_CpuSeconds{};
double m_CpuPercent{};
bool m_Stopping{};
std::vector<std::thread> m_Threads;

375
dUgcServer/UgcRender.cpp
View File

@@ -4,9 +4,13 @@
#include <cmath>
#include <limits>
#include <numeric>
#include <unordered_map>
#include <glm/gtc/matrix_transform.hpp>
#include "UgcPalette.h"
#include "UgcThrottle.h"
namespace {
constexpr float INF = std::numeric_limits<float>::infinity();
@@ -37,8 +41,151 @@ namespace {
}
}
float ToLinear(float c) { return std::pow(std::clamp(c, 0.0f, 1.0f), 2.2f); }
float ToSrgb(float c) { return std::pow(std::clamp(c, 0.0f, 1.0f), 1.0f / 2.2f); }
float ToLinear(float c) { return UgcPalette::SrgbToLinear(std::clamp(c, 0.0f, 1.0f)); }
float ToSrgb(float c) { return UgcPalette::LinearToSrgb(std::clamp(c, 0.0f, 1.0f)); }
// A bounding volume hierarchy over a mesh's triangles, for the occlusion rays
class Bvh {
public:
explicit Bvh(const UgcModel::Mesh& mesh) : m_Mesh(mesh) {
const size_t count = mesh.TriangleCount();
m_Order.resize(count);
std::iota(m_Order.begin(), m_Order.end(), 0u);
m_Centers.resize(count);
for (size_t t = 0; t < count; t++) m_Centers[t] = (Vertex(t, 0) + Vertex(t, 1) + Vertex(t, 2)) / 3.0f;
if (count > 0) Build(0, static_cast<uint32_t>(count));
Flatten();
}
// Whether a ray from `origin` along `direction` (unit) hits a triangle nearer than `maxDistance`
bool Hits(const glm::vec3& origin, const glm::vec3& direction, float maxDistance) const {
if (m_Nodes.empty()) return false;
const glm::vec3 inverse(1.0f / (std::abs(direction.x) > 1e-12f ? direction.x : 1e-12f), 1.0f / (std::abs(direction.y) > 1e-12f ? direction.y : 1e-12f),
1.0f / (std::abs(direction.z) > 1e-12f ? direction.z : 1e-12f));
uint32_t stack[64];
int top = 0;
stack[top++] = 0;
while (top > 0) {
const auto& node = m_Nodes[stack[--top]];
if (!BoxHit(node, origin, inverse, maxDistance)) continue;
if (node.count > 0) {
for (uint32_t i = node.first; i < node.first + node.count; i++) {
if (TriangleHit(m_Triangles[i], origin, direction, maxDistance)) return true;
}
} else if (top < 62) {
stack[top++] = node.first;
stack[top++] = node.first + 1;
}
}
return false;
}
private:
struct Node {
glm::vec3 min{};
glm::vec3 max{};
uint32_t first{}; // leaf: first triangle in m_Order; inner: the first of two children
uint32_t count{}; // triangles, 0 for inner nodes
};
glm::vec3 Vertex(size_t t, int k) const { return m_Mesh.positions[m_Mesh.indices[t * 3 + k]]; }
void Build(uint32_t first, uint32_t count) {
// Iterative, so deep trees don't use the stack
struct Task { uint32_t node, first, count; };
m_Nodes.push_back({});
std::vector<Task> tasks{ { 0, first, count } };
while (!tasks.empty()) {
const auto task = tasks.back();
tasks.pop_back();
Node node;
node.min = glm::vec3(INF);
node.max = glm::vec3(-INF);
glm::vec3 centerMin(INF), centerMax(-INF);
for (uint32_t i = task.first; i < task.first + task.count; i++) {
for (int k = 0; k < 3; k++) {
node.min = glm::min(node.min, Vertex(m_Order[i], k));
node.max = glm::max(node.max, Vertex(m_Order[i], k));
}
centerMin = glm::min(centerMin, m_Centers[m_Order[i]]);
centerMax = glm::max(centerMax, m_Centers[m_Order[i]]);
}
const auto extent = centerMax - centerMin;
const int axis = extent.x >= extent.y && extent.x >= extent.z ? 0 : extent.y >= extent.z ? 1 : 2;
if (task.count <= 4 || extent[axis] <= 0.0f) {
node.first = task.first;
node.count = task.count;
m_Nodes[task.node] = node;
continue;
}
const uint32_t half = task.count / 2;
auto* begin = m_Order.data() + task.first;
std::nth_element(begin, begin + half, begin + task.count, [&](uint32_t a, uint32_t b) { return m_Centers[a][axis] < m_Centers[b][axis]; });
node.first = static_cast<uint32_t>(m_Nodes.size());
node.count = 0;
m_Nodes[task.node] = node;
m_Nodes.push_back({});
m_Nodes.push_back({});
tasks.push_back({ node.first, task.first, half });
tasks.push_back({ node.first + 1, task.first + half, task.count - half });
}
}
static bool BoxHit(const Node& node, const glm::vec3& origin, const glm::vec3& inverse, float maxDistance) {
const auto t0 = (node.min - origin) * inverse;
const auto t1 = (node.max - origin) * inverse;
const auto near = glm::min(t0, t1), far = glm::max(t0, t1);
const float enter = std::max(std::max(near.x, near.y), std::max(near.z, 0.0f));
const float exit = std::min(std::min(far.x, far.y), std::min(far.z, maxDistance));
return enter <= exit;
}
struct Triangle {
glm::vec3 a, e1, e2;
};
// The triangles in leaf order, edges worked out once (the rays read them far more often than the tree is built)
void Flatten() {
m_Triangles.reserve(m_Order.size());
for (const auto t : m_Order) {
const auto a = Vertex(t, 0);
m_Triangles.push_back({ a, Vertex(t, 1) - a, Vertex(t, 2) - a });
}
}
static bool TriangleHit(const Triangle& triangle, const glm::vec3& origin, const glm::vec3& direction, float maxDistance) {
const auto& a = triangle.a;
const auto& e1 = triangle.e1;
const auto& e2 = triangle.e2;
const auto p = glm::cross(direction, e2);
const float det = glm::dot(e1, p);
if (std::abs(det) < 1e-12f) return false;
const float inv = 1.0f / det;
const auto s = origin - a;
const float u = glm::dot(s, p) * inv;
if (u < 0.0f || u > 1.0f) return false;
const auto q = glm::cross(s, e1);
const float v = glm::dot(direction, q) * inv;
if (v < 0.0f || u + v > 1.0f) return false;
const float distance = glm::dot(e2, q) * inv;
return distance > 1e-4f && distance < maxDistance;
}
const UgcModel::Mesh& m_Mesh;
std::vector<Triangle> m_Triangles;
std::vector<uint32_t> m_Order;
std::vector<glm::vec3> m_Centers;
std::vector<Node> m_Nodes;
};
float RadicalInverse(uint32_t bits) {
bits = (bits << 16u) | (bits >> 16u);
bits = ((bits & 0x55555555u) << 1u) | ((bits & 0xAAAAAAAAu) >> 1u);
bits = ((bits & 0x33333333u) << 2u) | ((bits & 0xCCCCCCCCu) >> 2u);
bits = ((bits & 0x0F0F0F0Fu) << 4u) | ((bits & 0xF0F0F0F0u) >> 4u);
bits = ((bits & 0x00FF00FFu) << 8u) | ((bits & 0xFF00FF00u) >> 8u);
return static_cast<float>(bits) * 2.3283064365386963e-10f;
}
// Looking at a sphere (center, radius) from direction `dir` (towards the viewer), square orthographic view
struct OrthoView {
@@ -102,7 +249,7 @@ namespace UgcRender {
OptimizeResult result;
auto& opaque = model.opaque;
result.trianglesBefore = opaque.TriangleCount() + model.transparent.TriangleCount();
if (model.Empty() || (!options.removeHidden && !options.bakeAo)) return result;
if (opaque.Empty() || !options.removeHidden) return result;
glm::vec3 center{};
float radius{};
@@ -124,10 +271,11 @@ namespace UgcRender {
}
// Without normals nothing is culled
if (opaque.normals.size() != opaque.positions.size()) std::fill(faceNormals.begin(), faceNormals.end(), glm::vec3(0.0f));
std::vector<float> aoOpaque(opaque.positions.size()), weightOpaque(opaque.positions.size());
std::vector<float> aoTransparent(model.transparent.positions.size()), weightTransparent(model.transparent.positions.size());
for (const auto& direction : SphereDirections()) {
// A ground plane under the model hides everything from below
if (options.groundPlane && direction.y < -0.05f) continue;
UgcThrottle::Checkpoint();
const OrthoView view(center, radius, direction, resolution);
const float bias = view.PixelSize();
std::fill(depth.begin(), depth.end(), INF);
@@ -138,6 +286,7 @@ namespace UgcRender {
for (size_t t = 0; t < triangles; t++) facing[t] = glm::dot(faceNormals[t], direction) > -0.1f;
for (size_t t = 0; t < triangles; t++) {
if (!facing[t]) continue;
if ((t & 0x3FFF) == 0) UgcThrottle::Checkpoint();
const auto id = static_cast<uint32_t>(t + 1);
Rasterize(resolution, resolution, screen[opaque.indices[t * 3]], screen[opaque.indices[t * 3 + 1]], screen[opaque.indices[t * 3 + 2]],
[&](int x, int y, float z, float, float, float) {
@@ -152,93 +301,96 @@ namespace UgcRender {
if (id != 0) visible[id - 1] = true;
}
// Whether a point is in front of what was drawn around its pixel
const auto exposed = [&](const glm::vec3& point, float allowance, bool neighbours) {
const auto p = view.Project(point);
const int px = static_cast<int>(std::floor(p.x)), py = static_cast<int>(std::floor(p.y));
const int reach = neighbours ? 1 : 0;
for (int dy = -reach; dy <= reach; dy++) {
for (int dx = -reach; dx <= reach; dx++) {
const int x = px + dx, y = py + dy;
if (x < 0 || y < 0 || x >= resolution || y >= resolution) return true;
if (p.z <= depth[static_cast<size_t>(y) * resolution + x] + allowance) return true;
}
}
return false;
};
// Triangles too small or thin to cover a pixel centre: kept when their centre isn't behind what was drawn.
// Bigger ones that show would have covered one.
if (options.removeHidden) {
const float smallArea = 2.0f; // pixels
for (size_t t = 0; t < triangles; t++) {
if (visible[t] || !facing[t]) continue;
const auto& a = screen[opaque.indices[t * 3]];
const auto& b = screen[opaque.indices[t * 3 + 1]];
const auto& c = screen[opaque.indices[t * 3 + 2]];
if (std::abs(Edge(a, b, c.x, c.y)) * 0.5f > smallArea) continue;
const auto centre = (a + b + c) / 3.0f;
const int x = static_cast<int>(centre.x), y = static_cast<int>(centre.y);
if (x < 0 || y < 0 || x >= resolution || y >= resolution || centre.z <= depth[static_cast<size_t>(y) * resolution + x] + bias) visible[t] = true;
}
}
if (options.bakeAo) {
const auto accumulate = [&](const UgcModel::Mesh& mesh, std::vector<float>& ao, std::vector<float>& weight) {
for (size_t v = 0; v < mesh.positions.size(); v++) {
const float w = v < mesh.normals.size() ? glm::dot(mesh.normals[v], direction) : 1.0f;
if (w <= 0.0f) continue;
weight[v] += w;
const auto normal = v < mesh.normals.size() ? mesh.normals[v] : glm::vec3(0.0f);
if (exposed(mesh.positions[v] + normal * bias, bias, false)) ao[v] += w;
}
};
accumulate(opaque, aoOpaque, weightOpaque);
accumulate(model.transparent, aoTransparent, weightTransparent);
}
}
if (options.bakeAo) {
const float strength = std::clamp(options.aoStrength, 0.0f, 1.0f);
const auto apply = [strength](UgcModel::Mesh& mesh, const std::vector<float>& ao, const std::vector<float>& weight) {
for (size_t v = 0; v < mesh.colors.size() && v < ao.size(); v++) {
if (weight[v] <= 0.0f) continue;
const float factor = 1.0f - strength * (1.0f - ao[v] / weight[v]);
auto& color = mesh.colors[v];
color.r = ToSrgb(ToLinear(color.r) * factor);
color.g = ToSrgb(ToLinear(color.g) * factor);
color.b = ToSrgb(ToLinear(color.b) * factor);
}
};
apply(opaque, aoOpaque, weightOpaque);
apply(model.transparent, aoTransparent, weightTransparent);
}
if (options.removeHidden) {
UgcModel::Mesh kept;
std::vector<uint32_t> remap(opaque.positions.size(), UINT32_MAX);
const float smallArea = 2.0f; // pixels
for (size_t t = 0; t < triangles; t++) {
if (!visible[t]) {
result.trianglesRemoved++;
continue;
}
for (int k = 0; k < 3; k++) {
const auto source = opaque.indices[t * 3 + k];
if (remap[source] == UINT32_MAX) {
remap[source] = static_cast<uint32_t>(kept.positions.size());
kept.positions.push_back(opaque.positions[source]);
if (source < opaque.normals.size()) kept.normals.push_back(opaque.normals[source]);
if (source < opaque.colors.size()) kept.colors.push_back(opaque.colors[source]);
}
kept.indices.push_back(remap[source]);
}
if (visible[t] || !facing[t]) continue;
const auto& a = screen[opaque.indices[t * 3]];
const auto& b = screen[opaque.indices[t * 3 + 1]];
const auto& c = screen[opaque.indices[t * 3 + 2]];
if (std::abs(Edge(a, b, c.x, c.y)) * 0.5f > smallArea) continue;
const auto centre = (a + b + c) / 3.0f;
const int x = static_cast<int>(centre.x), y = static_cast<int>(centre.y);
if (x < 0 || y < 0 || x >= resolution || y >= resolution || centre.z <= depth[static_cast<size_t>(y) * resolution + x] + bias) visible[t] = true;
}
opaque = std::move(kept);
}
for (size_t t = 0; t < triangles; t++) result.trianglesRemoved += visible[t] ? 0 : 1;
result.kept = visible;
UgcModel::KeepTriangles(opaque, visible);
return result;
}
Image RenderIcon(const UgcModel::Model& source, const IconOptions& options) {
std::vector<float> AmbientOcclusion(const UgcModel::Mesh& mesh, const UgcModel::Mesh& occluders, float distance, int samples) {
std::vector<float> ao(mesh.positions.size(), 1.0f);
if (occluders.Empty() || samples <= 0 || distance <= 0.0f || mesh.normals.size() != mesh.positions.size()) return ao;
const Bvh bvh(occluders);
const auto count = static_cast<uint32_t>(samples);
// Vertices at the same place facing the same way (bricks' shared corners) are worked out once
struct Key {
int32_t p[3], n[3];
bool operator==(const Key& o) const { return std::equal(p, p + 3, o.p) && std::equal(n, n + 3, o.n); }
};
struct KeyHash {
size_t operator()(const Key& k) const {
size_t h = 1469598103934665603ull;
for (int i = 0; i < 3; i++) h = (h ^ static_cast<uint32_t>(k.p[i])) * 1099511628211ull ^ static_cast<uint32_t>(k.n[i]) * 0x9E3779B97F4A7C15ull;
return h;
}
};
std::unordered_map<Key, float, KeyHash> known;
known.reserve(mesh.positions.size());
for (size_t v = 0; v < mesh.positions.size(); v++) {
if ((v & 0xFF) == 0) UgcThrottle::Checkpoint();
const auto& normal = mesh.normals[v];
const Key key{ { static_cast<int32_t>(std::lround(mesh.positions[v].x * 1000.0f)), static_cast<int32_t>(std::lround(mesh.positions[v].y * 1000.0f)),
static_cast<int32_t>(std::lround(mesh.positions[v].z * 1000.0f)) }, { static_cast<int32_t>(std::lround(normal.x * 100.0f)),
static_cast<int32_t>(std::lround(normal.y * 100.0f)), static_cast<int32_t>(std::lround(normal.z * 100.0f)) } };
if (const auto it = known.find(key); it != known.end()) {
ao[v] = it->second;
continue;
}
if (glm::dot(normal, normal) < 0.5f) continue;
// A frame around the normal
const glm::vec3 helper = std::abs(normal.x) < 0.9f ? glm::vec3(1, 0, 0) : glm::vec3(0, 1, 0);
const auto tangent = glm::normalize(glm::cross(helper, normal));
const auto bitangent = glm::cross(normal, tangent);
// Hammersley points, turned by an amount of the vertex's own (fixed) so neighbours don't band
const float turn = static_cast<float>((v * 0x9E3779B9u) >> 8 & 0xFFFFFF) / 16777216.0f;
const auto origin = mesh.positions[v] + normal * 1e-3f;
uint32_t open = 0;
for (uint32_t i = 0; i < count; i++) {
const float u = (i + 0.5f) / static_cast<float>(count);
const float phi = 2.0f * 3.14159265f * std::fmod(RadicalInverse(i) + turn, 1.0f);
const float r = std::sqrt(u), z = std::sqrt(std::max(0.0f, 1.0f - u));
const auto direction = tangent * (r * std::cos(phi)) + bitangent * (r * std::sin(phi)) + normal * z;
if (!bvh.Hits(origin, direction, distance)) open++;
}
ao[v] = static_cast<float>(open) / static_cast<float>(count);
known.emplace(key, ao[v]);
}
return ao;
}
std::vector<float> BakeAo(UgcModel::Model& model, const AoOptions& options) {
auto& opaque = model.opaque;
if (!options.enabled || opaque.Empty()) return {};
auto ao = AmbientOcclusion(opaque, opaque, options.distance, options.samples);
const float strength = std::clamp(options.strength, 0.0f, 1.0f);
for (size_t v = 0; v < opaque.colors.size() && v < ao.size(); v++) {
glm::vec3 lit(1.0f - strength * (1.0f - ao[v]));
if (v < opaque.glow.size()) lit += opaque.glow[v] * options.glowStrength;
lit = glm::clamp(lit, 0.0f, 1.0f);
auto& color = opaque.colors[v];
color.r = ToSrgb(ToLinear(color.r) * lit.r);
color.g = ToSrgb(ToLinear(color.g) * lit.g);
color.b = ToSrgb(ToLinear(color.b) * lit.b);
}
return ao;
}
Image RenderIcon(const UgcModel::Model& source, const IconOptions& options, const std::vector<float>* opaqueAo) {
const int size = std::clamp(options.size, 8, 1024);
const int supersample = std::clamp(options.supersample, 1, 8);
const int n = size * supersample;
@@ -284,8 +436,47 @@ namespace UgcRender {
// Linear, premultiplied
std::vector<glm::vec4> color(static_cast<size_t>(n) * n, glm::vec4(0.0f));
std::vector<float> depth(static_cast<size_t>(n) * n, INF);
const glm::vec3 light = glm::normalize(dir + glm::vec3(-0.35f, 1.1f, 0.25f));
const auto shade = [&](const UgcModel::Mesh& mesh, uint32_t i0, uint32_t i1, uint32_t i2, float w0, float w1, float w2) {
const float sunYaw = glm::radians(options.sunYawDegrees), sunPitch = glm::radians(options.sunPitchDegrees);
const glm::vec3 light = glm::normalize(glm::vec3(std::sin(sunYaw) * std::cos(sunPitch), std::sin(sunPitch), std::cos(sunYaw) * std::cos(sunPitch)));
// Ambient occlusion darkens the world light (opaque bricks only, as they are what occludes)
std::vector<float> ao;
if (options.ao.enabled) {
ao = opaqueAo && opaqueAo->size() == model.opaque.positions.size() ? *opaqueAo : AmbientOcclusion(model.opaque, model.opaque, options.ao.distance, options.ao.samples);
}
// The sun's shadows: a depth map seen from the sun, looked up with a few taps for the sun's soft edge
const int shadowSize = 1024;
const OrthoView sunView(center, radius * 1.05f, light, shadowSize);
std::vector<float> shadowDepth;
if (options.shadows) {
shadowDepth.assign(static_cast<size_t>(shadowSize) * shadowSize, INF);
const auto& mesh = model.opaque;
std::vector<glm::vec3> screen(mesh.positions.size());
for (size_t v = 0; v < mesh.positions.size(); v++) screen[v] = sunView.Project(mesh.positions[v]);
for (size_t i = 0; i + 2 < mesh.indices.size(); i += 3) {
if ((i & 0xFFFF) == 0) UgcThrottle::Checkpoint();
Rasterize(shadowSize, shadowSize, screen[mesh.indices[i]], screen[mesh.indices[i + 1]], screen[mesh.indices[i + 2]], [&](int x, int y, float z, float, float, float) {
auto& stored = shadowDepth[static_cast<size_t>(y) * shadowSize + x];
stored = std::min(stored, z);
});
}
}
const float shadowBias = sunView.PixelSize() * 2.0f;
const auto sunlit = [&](const glm::vec3& position) {
if (shadowDepth.empty()) return 1.0f;
const auto p = sunView.Project(position);
float lit = 0.0f;
for (int dy = -1; dy <= 1; dy++) {
for (int dx = -1; dx <= 1; dx++) {
const int x = static_cast<int>(p.x) + dx, y = static_cast<int>(p.y) + dy;
if (x < 0 || y < 0 || x >= shadowSize || y >= shadowSize || p.z <= shadowDepth[static_cast<size_t>(y) * shadowSize + x] + shadowBias) lit += 1.0f;
}
}
return lit / 9.0f;
};
const auto shade = [&](const UgcModel::Mesh& mesh, bool isOpaque, uint32_t i0, uint32_t i1, uint32_t i2, float w0, float w1, float w2) {
glm::vec3 normal(0.0f, 1.0f, 0.0f);
if (mesh.normals.size() == mesh.positions.size()) {
normal = mesh.normals[i0] * w0 + mesh.normals[i1] * w1 + mesh.normals[i2] * w2;
@@ -295,7 +486,13 @@ namespace UgcRender {
}
glm::vec4 base(0.63f, 0.63f, 0.63f, 1.0f);
if (mesh.colors.size() == mesh.positions.size()) base = mesh.colors[i0] * w0 + mesh.colors[i1] * w1 + mesh.colors[i2] * w2;
const float lighting = 0.55f + 0.6f * std::max(0.0f, glm::dot(normal, light));
const float occlusion = isOpaque && ao.size() == mesh.positions.size() ? ao[i0] * w0 + ao[i1] * w1 + ao[i2] * w2 : 1.0f;
const float strength = std::clamp(options.ao.strength, 0.0f, 1.0f);
const auto position = mesh.positions[i0] * w0 + mesh.positions[i1] * w1 + mesh.positions[i2] * w2;
const float direct = std::max(0.0f, glm::dot(normal, light));
// Diffuse: the world's light (radiance `ambient`) and the sun's (irradiance `sunStrength`, over pi)
const float lighting = options.ambient * (1.0f - strength * (1.0f - occlusion)) +
options.sunStrength / 3.14159265f * direct * (direct > 0.0f ? sunlit(position + normal * shadowBias) : 0.0f);
return glm::vec4(ToLinear(base.r) * lighting, ToLinear(base.g) * lighting, ToLinear(base.b) * lighting, std::clamp(base.a, 0.0f, 1.0f));
};
@@ -310,7 +507,7 @@ namespace UgcRender {
const size_t index = static_cast<size_t>(y) * n + x;
if (z >= depth[index]) return;
depth[index] = z;
const auto shaded = shade(mesh, i0, i1, i2, w0, w1, w2);
const auto shaded = shade(mesh, true, i0, i1, i2, w0, w1, w2);
color[index] = glm::vec4(glm::vec3(shaded), 1.0f);
});
}
@@ -330,7 +527,7 @@ namespace UgcRender {
Rasterize(n, n, screen[i0], screen[i1], screen[i2], [&](int x, int y, float z, float w0, float w1, float w2) {
const size_t index = static_cast<size_t>(y) * n + x;
if (z >= depth[index]) return;
const auto shaded = shade(mesh, i0, i1, i2, w0, w1, w2);
const auto shaded = shade(mesh, false, i0, i1, i2, w0, w1, w2);
const float alpha = shaded.a;
auto& target = color[index];
target = glm::vec4(glm::vec3(shaded) * alpha + glm::vec3(target) * (1.0f - alpha), alpha + target.a * (1.0f - alpha));

53
dUgcServer/UgcRender.h
View File

@@ -19,38 +19,71 @@ namespace UgcRender {
std::vector<uint8_t> rgba;
};
struct AoOptions {
bool enabled{ true };
float distance{ 5.0f }; // LU Toolbox's AO distance (Bake Lighting, AO Only)
int samples{ 64 }; // rays per vertex (AO Samples)
float strength{ 1.0f }; // 0 leaves the colors, 1 is the full bake
float glowStrength{ 6.0f }; // what glowing colors add to the light (Glow Strength 3 x Glow Multiplier 2)
};
/**
* The icon, set up like LU Toolbox's icon renderer (the BrickBuild scene of its UGC render add-on): a 50 mm lens
* (39.6 degree field of view) looking from 53.4 degrees around and 19.5 above, framed at 1.03, a sun of strength
* 2.5 from 21 around and 50.3 above with soft shadows, and a grey (0.192) world light, darkened by ambient occlusion.
*/
struct IconOptions {
int size{ 128 };
int supersample{ 4 };
float yawDegrees{ 35.0f }; // camera around the model, from +Z towards +X
float pitchDegrees{ 25.0f }; // camera above the model
float fovDegrees{ 30.0f };
float yawDegrees{ 53.36f }; // camera around the model, from +Z towards +X
float pitchDegrees{ 19.54f }; // camera above the model
float fovDegrees{ 39.6f };
float margin{ 1.03f }; // 1 fills the icon, more leaves a border
glm::mat4 modelRotation{ 1.0f }; // applied to the model before the camera looks at it
float sunYawDegrees{ 21.0f };
float sunPitchDegrees{ 50.3f };
float sunStrength{ 2.5f };
float ambient{ 0.192f };
bool shadows{ true };
AoOptions ao{ true, 5.0f, 32, 1.0f, 0.0f };
};
// The model drawn from the icon's camera, framed to fit, on a transparent background
Image RenderIcon(const UgcModel::Model& model, const IconOptions& options);
// The model drawn from the icon's camera, framed to fit, on a transparent background. `opaqueAo`: the opaque mesh's
// ambient occlusion (AmbientOcclusion) when it is known already, else it is worked out when options.ao wants it.
Image RenderIcon(const UgcModel::Model& model, const IconOptions& options, const std::vector<float>* opaqueAo = nullptr);
struct OptimizeOptions {
int resolution{ 1024 }; // of each direction's render
bool removeHidden{ true };
bool bakeAo{ true };
float aoStrength{ 0.6f }; // 0 leaves the colors, 1 makes fully hidden corners black
bool groundPlane{ false }; // LU Toolbox's Use Ground Plane: nothing is seen from below the model
};
struct OptimizeResult {
size_t trianglesBefore{};
size_t trianglesRemoved{};
std::vector<bool> kept; // per opaque triangle before: whether it stayed (for UgcModel::KeepTriangles)
};
/**
* Renders the opaque mesh from 42 directions around it: triangles that show in none of them are removed (with a
* conservative test, so small visible ones stay), and each vertex's share of the directions it can be seen from
* darkens its color (ambient occlusion). Transparent bricks hide nothing but are darkened too.
* Hidden surface removal: renders the opaque mesh from 42 directions around it and removes the triangles that show
* in none of them (with a conservative test, so small visible ones stay). Transparent bricks hide nothing and
* aren't touched, as in LU Toolbox.
*/
OptimizeResult Optimize(UgcModel::Model& model, const OptimizeOptions& options);
/**
* Ambient occlusion of each vertex of `mesh`: the share of `samples` rays (cosine weighted around the vertex
* normal, the same pattern every time) that leave without hitting a triangle of `occluders` within `distance`.
* 1 is open, 0 fully hidden.
*/
std::vector<float> AmbientOcclusion(const UgcModel::Mesh& mesh, const UgcModel::Mesh& occluders, float distance, int samples);
/**
* LU Toolbox's Bake Lighting with AO Only (its defaults): the opaque mesh's occlusion (transparent bricks are hidden
* while baking and not baked) plus the glow colors, multiplied into the vertex colors.
*/
std::vector<float> BakeAo(UgcModel::Model& model, const AoOptions& options); // the occlusion used, per opaque vertex
// The 42 directions Optimize renders from (an icosahedron's corners and edge centres), unit length
std::vector<glm::vec3> SphereDirections();
}

91
dUgcServer/UgcServer.cpp
View File

@@ -8,6 +8,8 @@
#include <fstream>
#include <iostream>
#include <random>
#include <sstream>
#include <set>
#include <thread>
#include "BinaryPathFinder.h"
@@ -33,6 +35,7 @@
#include "UgcModel.h"
#include "UgcProcessor.h"
#include "UgcStorage.h"
#include "UgcThrottle.h"
namespace Game {
Logger* logger = nullptr;
@@ -52,22 +55,70 @@ namespace {
return GeneralUtils::TryParse<T>(Game::config->GetValue(key)).value_or(fallback);
}
std::vector<uint32_t> ParseLods(const std::string& text) {
std::vector<uint32_t> lods;
std::stringstream stream(text);
std::string item;
while (std::getline(stream, item, ',')) {
std::erase_if(item, [](unsigned char c) { return std::isspace(c); });
if (const auto lod = GeneralUtils::TryParse<uint32_t>(item); lod && *lod <= 3) lods.push_back(*lod);
}
return lods;
}
// ugcconfig.ini (and the dashboard's settings); the defaults are LU Toolbox's
UgcJobs::Settings ReadSettings() {
UgcJobs::Settings settings;
settings.build.palette = Game::config->GetValue("color_palette") == "brickdb" ? UgcModel::ePalette::BRICKDB : UgcModel::ePalette::LU_TOOLBOX;
settings.build.colorVariation = std::clamp(Setting<float>("color_variation", 5.0f), 0.0f, 100.0f);
settings.build.transparentOpacity = std::clamp(Setting<float>("transparent_opacity", 58.82f), 0.0f, 100.0f);
const auto lods = ParseLods(Game::config->GetValue("lods").empty() ? "0,2" : Game::config->GetValue("lods"));
if (!lods.empty()) settings.lods = lods;
settings.lodDistances.lod0 = Setting<float>("lod_distance_0", 0.0f);
settings.lodDistances.lod1 = Setting<float>("lod_distance_1", 50.0f);
settings.lodDistances.lod2 = Setting<float>("lod_distance_2", 100.0f);
settings.lodDistances.lod3 = Setting<float>("lod_distance_3", 280.0f);
settings.lodDistances.cull = Setting<float>("lod_cull", 10000.0f);
if (!Game::config->GetValue("shader_opaque").empty()) settings.shaderOpaque = Game::config->GetValue("shader_opaque");
settings.optimize.removeHidden = Setting<int32_t>("remove_hidden_faces", 1) != 0;
settings.optimize.bakeAo = Setting<int32_t>("bake_ao", 1) != 0;
settings.optimize.aoStrength = Setting<float>("ao_strength", 0.6f);
settings.optimize.groundPlane = Setting<int32_t>("hsr_ground_plane", 0) != 0;
settings.optimize.resolution = Setting<int32_t>("optimize_resolution", 1024);
settings.ao.enabled = Setting<int32_t>("bake_ao", 1) != 0;
settings.ao.distance = Setting<float>("ao_distance", 5.0f);
settings.ao.samples = std::clamp(Setting<int32_t>("ao_samples", 64), 1, 1024);
settings.ao.strength = Setting<float>("ao_strength", 1.0f);
settings.ao.glowStrength = Setting<float>("glow_strength", 6.0f);
settings.icon.size = Setting<int32_t>("icon_size", 128);
settings.icon.yawDegrees = Setting<float>("icon_yaw", 35.0f);
settings.icon.pitchDegrees = Setting<float>("icon_pitch", 25.0f);
settings.icon.yawDegrees = Setting<float>("icon_yaw", 53.36f);
settings.icon.pitchDegrees = Setting<float>("icon_pitch", 19.54f);
settings.icon.fovDegrees = Setting<float>("icon_fov", 39.6f);
settings.icon.margin = Setting<float>("icon_margin", 1.03f);
settings.icon.sunYawDegrees = Setting<float>("icon_sun_yaw", 21.0f);
settings.icon.sunPitchDegrees = Setting<float>("icon_sun_pitch", 50.3f);
settings.icon.sunStrength = Setting<float>("icon_sun_strength", 2.5f);
settings.icon.ambient = Setting<float>("icon_ambient", 0.192f);
settings.icon.shadows = Setting<int32_t>("icon_shadows", 1) != 0;
settings.icon.ao.enabled = Setting<int32_t>("icon_ao", 1) != 0;
settings.icon.ao.distance = settings.ao.distance;
settings.iconCorrectColors = Setting<int32_t>("icon_correct_colors", 1) != 0;
settings.iconColorVariation = std::clamp(Setting<float>("icon_color_variation", 0.0f), 0.0f, 100.0f);
settings.modularIcon = settings.icon;
settings.modularIcon.yawDegrees = Setting<float>("modular_icon_yaw", 35.0f);
settings.modularIcon.pitchDegrees = Setting<float>("modular_icon_pitch", 20.0f);
settings.modularIcon.yawDegrees = Setting<float>("modular_icon_yaw", 53.36f);
settings.modularIcon.pitchDegrees = Setting<float>("modular_icon_pitch", 19.54f);
settings.maxBricks = Setting<uint32_t>("max_model_bricks", 0);
return settings;
}
UgcProcessor::Limits ReadLimits() {
UgcProcessor::Limits limits;
const auto cores = std::max<unsigned>(std::thread::hardware_concurrency(), 1);
limits.maxCpus = std::clamp(Setting<double>("max_cpu_percent", 0.0), 0.0, 100.0) / 100.0 * cores;
limits.maxMemoryBytes = Setting<uint64_t>("max_memory_mb", 0) * 1024 * 1024;
limits.nice = std::clamp(Setting<int32_t>("worker_nice", 0), 0, 19);
UgcThrottle::ParseHours(Game::config->GetValue("pause_hours"), limits.pauseFromHour, limits.pauseToHour);
return limits;
}
std::vector<std::string> Segments(const std::string& path) {
std::vector<std::string> segments;
size_t start = 0;
@@ -187,9 +238,17 @@ namespace {
const auto id = GeneralUtils::TryParse<uint64_t>(segments[2]);
const auto kind = segments[1] == "model" ? std::optional(UgcStorage::Kind::MODEL) : segments[1] == "modular" ? std::optional(UgcStorage::Kind::MODULAR) : std::nullopt;
const auto& name = segments[3];
if (!id || !kind || (name != "icon.png" && name != "model.nif")) return;
static const std::set<std::string> PREVIEWS = { "icon.png", "model.nif", "model.noao.nif", "stats.json",
"previous.icon.png", "previous.model.nif", "previous.model.noao.nif", "previous.stats.json" };
if (!id || !kind || !PREVIEWS.contains(name)) return;
reply.headers.clear();
ServeFile(reply, *kind, static_cast<LWOOBJID>(*id), name, name == "icon.png" ? eContentType::IMAGE_PNG : eContentType::APPLICATION_OCTET_STREAM, true);
const auto type = name.ends_with(".png") ? eContentType::IMAGE_PNG : name.ends_with(".json") ? eContentType::APPLICATION_JSON : eContentType::APPLICATION_OCTET_STREAM;
ServeFile(reply, *kind, static_cast<LWOOBJID>(*id), name, type, true);
// Made again since: the newest files differ, so they mustn't be cached as long as the client's
if (reply.status == eHTTPStatusCode::OK) {
std::erase_if(reply.headers, [](const std::string& header) { return header.starts_with("Cache-Control"); });
reply.headers.push_back("Cache-Control: no-cache");
}
} });
Game::web.RegisterHTTPRoute({ .path = "/status", .method = eHTTPMethod::GET, .middleware = {},
@@ -212,7 +271,7 @@ namespace {
int MakeFromCommandLine(const std::string& mode, const std::string& input, const std::filesystem::path& output) {
const auto res = ResPath();
const auto settings = ReadSettings();
UgcBricks::BrickLibrary library(res, Setting<uint32_t>("brick_lod", 0));
UgcBricks::BrickLibrary library(res, 0);
if (!library.LoadMaterials()) std::cerr << "Couldn't read Materials.xml from " << (res / "brickdb.zip") << "; bricks will be grey\n";
UgcJobs::Outcome outcome;
const auto start = std::chrono::steady_clock::now();
@@ -309,7 +368,7 @@ int main(int argc, char** argv) {
ServiceType::UGC, Game::config, &Game::lastSignal, masterPassword);
Game::server = g_Server;
UgcBricks::BrickLibrary library(res, Setting<uint32_t>("brick_lod", 0));
UgcBricks::BrickLibrary library(res, 0);
if (!library.LoadMaterials()) LOG("Couldn't read Materials.xml from %s; bricks will be grey", (res / "brickdb.zip").string().c_str());
auto outputDir = std::filesystem::path(Game::config->GetValue("ugc_output_dir").empty() ? "ugc" : Game::config->GetValue("ugc_output_dir"));
@@ -325,6 +384,7 @@ int main(int argc, char** argv) {
const auto threads = Setting<uint32_t>("worker_threads", 0);
processorConfig.threads = threads > 0 ? threads : std::max<size_t>(std::thread::hardware_concurrency() / 2, 1);
UgcProcessor processor(processorConfig, storage, library, ReadSettings());
processor.Configure(ReadSettings(), ReadLimits());
g_Processor = &processor;
// Sent with the traffic reports to the dashboard (Diagnostics)
TrafficStats::Local().SetGauge("workers_busy", [&processor] { return static_cast<double>(processor.Busy()); });
@@ -339,6 +399,10 @@ int main(int argc, char** argv) {
{ "ugc_max_storage_bytes", [&processor] { return static_cast<double>(processor.MaxStorageBytes()); } },
};
for (const auto& [name, gauge] : ugcGauges) TrafficStats::Local().SetGauge(name, gauge);
TrafficStats::Local().SetGauge("cpu_percent", [&processor] { return processor.CpuPercent(); });
TrafficStats::Local().SetGauge("memory_mb", [] { return static_cast<double>(UgcProcessor::ResidentBytes()) / (1024.0 * 1024.0); });
TrafficStats::Local().SetGauge("job_memory_mb", [&processor] { return static_cast<double>(processor.JobMemory()) / (1024.0 * 1024.0); });
TrafficStats::Local().SetGauge("throttled", [&processor] { return processor.Throttled() ? 1.0 : 0.0; });
const auto listenIp = Game::config->GetValue("listen_ip").empty() ? std::string("0.0.0.0") : Game::config->GetValue("listen_ip");
const auto port = Setting<uint32_t>("port", 2008);
@@ -351,6 +415,7 @@ int main(int argc, char** argv) {
LOG("UGC Server started on %s:%u, files in %s", listenIp.c_str(), port, outputDir.string().c_str());
auto lastTick = std::chrono::steady_clock::now();
auto lastConfigure = lastTick;
while (!Game::ShouldShutdown()) {
// The poll's wait for network traffic is also this loop's sleep
Game::web.ReceiveRequests(10);
@@ -364,6 +429,11 @@ int main(int argc, char** argv) {
packet = g_Server->ReceiveFromMaster();
}
processor.Update();
// Settings the dashboard changed arrive as a config reload; pick them up
if (now - lastConfigure >= std::chrono::seconds(5)) {
lastConfigure = now;
processor.Configure(ReadSettings(), ReadLimits());
}
}
LOG("Stopping the UGC server");
@@ -372,6 +442,7 @@ int main(int argc, char** argv) {
TrafficStats::Local().SetGauge("workers_queued", nullptr);
TrafficStats::Local().SetGauge("workers_threads", nullptr);
for (const auto& [name, gauge] : ugcGauges) TrafficStats::Local().SetGauge(name, nullptr);
for (const auto* gauge : { "cpu_percent", "memory_mb", "job_memory_mb", "throttled" }) TrafficStats::Local().SetGauge(gauge, nullptr);
Game::web.Shutdown();
g_Processor = nullptr;
Database::Destroy("UgcServer");

13
dUgcServer/UgcStorage.cpp
View File

@@ -10,8 +10,13 @@ namespace {
"model.nif", "model.nif.gz", "model.nif.checksum",
"model.lxfml.gz", "model.lxfml.checksum",
"icon.dds.gz", "icon.dds.checksum", "icon.png",
"model.noao.nif", "stats.json",
"previous.icon.png", "previous.model.nif", "previous.model.noao.nif", "previous.stats.json",
};
// What of the files made before is kept when an item is made again, for comparing (as previous.<name>)
constexpr std::array KEPT_FILES = { "icon.png", "model.nif", "model.noao.nif", "stats.json" };
const char* KindFolder(UgcStorage::Kind kind) {
return kind == UgcStorage::Kind::MODEL ? "models" : "modular";
}
@@ -61,6 +66,14 @@ std::optional<uint64_t> UgcStorage::Write(Kind kind, LWOOBJID id, const Files& f
}
bytes += data.size();
}
// Keep the last version's previews to compare with
for (const auto* name : KEPT_FILES) {
if (files.contains(name)) {
std::filesystem::copy_file(folder / name, temporary / (std::string("previous.") + name), std::filesystem::copy_options::overwrite_existing, code);
if (!code) bytes += std::filesystem::file_size(temporary / (std::string("previous.") + name), code);
code.clear();
}
}
// Swap the old folder out and the new one in; the old one is deleted after
const auto old = folder.parent_path() / (".old-" + folder.filename().string() + "-" + RandomSuffix());
const bool hadOld = std::filesystem::exists(folder, code);

110
dUgcServer/UgcThrottle.cpp Normal file
View File

@@ -0,0 +1,110 @@
#include "UgcThrottle.h"
#include <algorithm>
#include <atomic>
#include <chrono>
#include <ctime>
#include <mutex>
#include <thread>
namespace {
// Up to this much CPU time may be used ahead of the budget (so short jobs aren't slowed at all)
constexpr double BURST_SECONDS = 0.25;
// Checkpoints closer together than this only read the clock
constexpr double MIN_ACCOUNT_SECONDS = 0.005;
std::atomic<double> g_Budget{ 0.0 };
std::mutex g_Mutex;
double g_Balance = BURST_SECONDS; // CPU seconds that may still be used
std::chrono::steady_clock::time_point g_Refilled = std::chrono::steady_clock::now();
std::atomic<uint64_t> g_SleptMs{ 0 };
std::atomic<int64_t> g_LastSleep{ 0 };
thread_local double t_LastCpu = -1.0;
int64_t UnixMs() {
return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
}
void Refill(double budget) {
const auto now = std::chrono::steady_clock::now();
const double elapsed = std::chrono::duration<double>(now - g_Refilled).count();
g_Refilled = now;
g_Balance = std::min(g_Balance + elapsed * budget, BURST_SECONDS * std::max(budget, 1.0));
}
}
namespace UgcThrottle {
void SetBudget(double cpus) {
std::lock_guard lock(g_Mutex);
g_Budget = std::max(cpus, 0.0);
g_Balance = std::min(g_Balance, BURST_SECONDS);
g_Refilled = std::chrono::steady_clock::now();
}
double GetBudget() {
return g_Budget;
}
double ThreadCpuSeconds() {
#if defined(CLOCK_THREAD_CPUTIME_ID)
timespec ts{};
if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts) == 0) return static_cast<double>(ts.tv_sec) + ts.tv_nsec / 1e9;
#endif
return std::chrono::duration<double>(std::chrono::steady_clock::now().time_since_epoch()).count();
}
void Begin() {
t_LastCpu = ThreadCpuSeconds();
}
void Checkpoint() {
const double budget = g_Budget;
if (budget <= 0.0) return;
const double cpu = ThreadCpuSeconds();
if (t_LastCpu < 0.0) t_LastCpu = cpu;
const double used = cpu - t_LastCpu;
if (used < MIN_ACCOUNT_SECONDS) return;
t_LastCpu = cpu;
double wait = 0.0;
{
std::lock_guard lock(g_Mutex);
Refill(budget);
g_Balance -= used;
// Overdrawn: wait until the budget has paid it back. Other threads waiting meanwhile each owe their own
// share, so the waits add up to what the budget allows.
if (g_Balance < 0.0) wait = -g_Balance / budget;
}
if (wait <= 0.0) return;
wait = std::min(wait, 5.0);
g_SleptMs += static_cast<uint64_t>(wait * 1000.0);
g_LastSleep = UnixMs();
std::this_thread::sleep_for(std::chrono::duration<double>(wait));
// Time asleep costs no CPU; don't count this call's own bookkeeping twice
t_LastCpu = ThreadCpuSeconds();
}
Stats GetStats() {
return { g_SleptMs.load(), g_LastSleep.load() };
}
bool ParseHours(const std::string& text, int& from, int& to) {
const auto dash = text.find('-');
if (dash == std::string::npos) return false;
try {
const int a = std::stoi(text.substr(0, dash)), b = std::stoi(text.substr(dash + 1));
if (a < 0 || a > 23 || b < 0 || b > 23) return false;
from = a;
to = b;
return true;
} catch (...) {
return false;
}
}
bool InHours(int hour, int from, int to) {
if (from < 0 || to < 0) return false;
return from <= to ? hour >= from && hour <= to : hour >= from || hour <= to;
}
}

36
dUgcServer/UgcThrottle.h Normal file
View File

@@ -0,0 +1,36 @@
#pragma once
#include <cstdint>
#include <string>
/**
* Keeps the UGC workers' CPU use under a budget. The long loops (renders, ambient occlusion, icons) call Checkpoint
* every few milliseconds of work; it adds the calling thread's CPU time since its last call to a shared account that
* the budget fills at `budget` CPUs per second, and sleeps while the account is overdrawn. So the workers together
* average at most the budget however many there are, long jobs included, and without a budget it costs nothing.
*/
namespace UgcThrottle {
// CPUs the workers may use together on average (1.5: one and a half cores); 0 or less: no limit
void SetBudget(double cpus);
double GetBudget();
// Account the calling thread's CPU time and sleep when over the budget
void Checkpoint();
// Start accounting on this thread from now (a worker starting a job), so time spent idle isn't counted
void Begin();
struct Stats {
uint64_t sleptMs{}; // since start
int64_t lastSleepUnixMs{}; // when it last slept, 0 never
};
Stats GetStats();
// This thread's CPU time in seconds
double ThreadCpuSeconds();
// Parses "18-23" (from 18:00 to 23:59; may wrap past midnight, "22-6") into from and to; false when it isn't that
bool ParseHours(const std::string& text, int& from, int& to);
// Whether `hour` (0-23) is inside the range; never when from or to is negative
bool InHours(int hour, int from, int to);
}

73
resources/ugcconfig.ini
View File

@@ -22,27 +22,76 @@ ugc_max_storage_mb=2048
# Worker threads making files (0: half the CPU cores)
worker_threads=0
# How much of the machine making files may use (changes apply while running):
# max_cpu_percent: the workers together average at most this share of all CPU cores (0: no limit); they pause
# between bits of work to stay under it, long renders included.
# worker_nice: the workers' scheduling priority on Linux, 0 (normal) to 19 (only when nothing else wants the CPU).
# max_memory_mb: jobs whose estimated memory would go past this together wait for running ones to finish; a job
# bigger than it alone runs when nothing else does (0: no limit).
# max_model_bricks: models with more bricks fail with that reason instead of being made (0: no limit).
# pause_hours: local hours in which no new jobs start, e.g. 18-23 or 22-6 (empty: never).
max_cpu_percent=0
worker_nice=0
max_memory_mb=0
max_model_bricks=0
pause_hours=
# How often to look for models to make (milliseconds), how many to take at once, and how often a model may fail
# before it's given up on (the dashboard can try again)
poll_interval_ms=2000
poll_batch=32
max_attempts=3
# Brick detail used for meshes and icons: 0 (most detailed) to 2
brick_lod=0
# Models are made like LU Toolbox's Process Model and Bake Lighting make them; these default to its settings.
# color_palette: lu_toolbox (its LU palette, unknown colors black) or brickdb (the client's Materials.xml).
# color_variation: each brick's brightness is shifted by up to this percent (the same every time; 0: none).
# transparent_opacity: transparent bricks' opacity in percent.
color_palette=lu_toolbox
color_variation=5
transparent_opacity=58.82
# Remove faces that can't be seen from anywhere, and darken hidden corners (ambient occlusion baked into the vertex
# colors): 0 or 1. ao_strength: 0 (none) to 1. optimize_resolution: detail of the renders that decide both.
# Levels of detail made, from the client's brickprimitives (0 most detailed, 2 least), and the distances they're
# drawn at (LU Toolbox picks each level's range from these by which levels are made; lod_cull: drawn up to).
lods=0,2
lod_distance_0=0
lod_distance_1=50
lod_distance_2=100
lod_distance_3=280
lod_cull=10000
# The shapes' shader number: S<shader>_Opaque_Model (transparent shapes are S01_Alpha_Model)
shader_opaque=01
# Remove faces that can't be seen from anywhere (optimize_resolution: detail of the renders that decide it;
# hsr_ground_plane: 1 also removes what can only be seen from below the model)
remove_hidden_faces=1
bake_ao=1
ao_strength=0.6
hsr_ground_plane=0
optimize_resolution=1024
# Icons: size in pixels, the camera's angle around (yaw, from the model's front) and above (pitch) the model in degrees,
# and the empty border (1 fills the icon). Modular builds (cars, rockets) have their own angles.
# Bake ambient occlusion into the vertex colors: rays per vertex, how far they look, strength 0 (none) to 1, and how
# much glowing colors add
bake_ao=1
ao_samples=64
ao_distance=5
ao_strength=1
glow_strength=6
# Icons, set up like LU Toolbox's icon renderer: size in pixels, the camera's angle around (yaw, from the model's
# front) and above (pitch) in degrees, its field of view, the empty border (1 fills the icon), the sun's angles and
# strength, the world light, shadows and ambient occlusion, its color corrections and color variation (percent).
# Modular builds (cars, rockets) have their own camera angles.
icon_size=128
icon_yaw=35
icon_pitch=25
icon_yaw=53.36
icon_pitch=19.54
icon_fov=39.6
icon_margin=1.03
modular_icon_yaw=35
modular_icon_pitch=20
icon_sun_yaw=21
icon_sun_pitch=50.3
icon_sun_strength=2.5
icon_ambient=0.192
icon_shadows=1
icon_ao=1
icon_correct_colors=1
icon_color_variation=0
modular_icon_yaw=53.36
modular_icon_pitch=19.54

277
tests/dUgcTests/UgcTests.cpp
View File

@@ -11,9 +11,12 @@
#include "UgcBricks.h"
#include "UgcFormats.h"
#include "UgcModel.h"
#include "UgcJobs.h"
#include "UgcModular.h"
#include "UgcPalette.h"
#include "UgcRender.h"
#include "UgcStorage.h"
#include "UgcThrottle.h"
#include "ZCompression.h"
class Logger;
@@ -137,7 +140,10 @@ TEST(UgcModel, BuildsOpaqueAndTransparentMeshes) {
UgcBricks::BrickLibrary library(MakeRes(), 0);
library.SetMaterials({ { 21, { 222, 0, 13, 255 } }, { 40, { 238, 238, 238, 150 } } });
std::string error;
const auto model = UgcModel::Build(UgcModel::ParseLxfml(LXFML5, error), library);
UgcModel::BuildOptions options;
options.palette = UgcModel::ePalette::BRICKDB;
options.colorVariation = 0.0f;
const auto model = UgcModel::Build(UgcModel::ParseLxfml(LXFML5, error), library, options);
EXPECT_EQ(model.bricks, 2u);
EXPECT_EQ(model.missingDesigns, std::vector<uint32_t>{ 9999 });
EXPECT_EQ(model.opaque.TriangleCount(), 12u);
@@ -176,7 +182,7 @@ TEST(UgcRender, RemovesWhatIsInsideAndDrawsIcons) {
</Bricks></LXFML>)", error);
auto model = UgcModel::Build(parts, library);
ASSERT_EQ(model.opaque.TriangleCount(), 24u);
const auto result = UgcRender::Optimize(model, UgcRender::OptimizeOptions{ 256, true, true, 0.6f });
const auto result = UgcRender::Optimize(model, UgcRender::OptimizeOptions{ 256, true });
EXPECT_EQ(result.trianglesRemoved, 12u);
EXPECT_EQ(model.opaque.TriangleCount(), 12u);
EXPECT_EQ(model.opaque.positions.size(), 8u);
@@ -280,12 +286,277 @@ TEST(UgcStorage, WritesListsAndEvicts) {
ASSERT_TRUE(storage.Write(UgcStorage::Kind::MODULAR, 2002, { { "icon.png", std::string(100, 'b') } }, error)) << error;
ASSERT_TRUE(storage.Write(UgcStorage::Kind::MODEL, 1001, { { "icon.png", std::string(50, 'c') } }, error)) << error; // replaced
EXPECT_TRUE(storage.File(UgcStorage::Kind::MODEL, 1001, "icon.png"));
// The version before is kept to compare with
const auto previous = storage.File(UgcStorage::Kind::MODEL, 1001, "previous.icon.png");
ASSERT_TRUE(previous);
EXPECT_EQ(std::filesystem::file_size(*previous), 100u);
EXPECT_FALSE(storage.File(UgcStorage::Kind::MODEL, 1001, "../../etc/passwd"));
EXPECT_EQ(storage.List().size(), 2u);
std::filesystem::last_write_time(storage.Folder(UgcStorage::Kind::MODULAR, 2002), std::filesystem::file_time_type::clock::now() - std::chrono::hours(1));
const auto removed = storage.Evict(60);
const auto removed = storage.Evict(160);
ASSERT_EQ(removed.size(), 1u);
EXPECT_EQ(removed[0].id, 2002);
EXPECT_TRUE(storage.File(UgcStorage::Kind::MODEL, 1001, "icon.png"));
std::filesystem::remove_all(storage.GetRoot());
}
TEST(UgcPalette, ColorVariationMatchesLuToolbox) {
const glm::vec3 red = *UgcPalette::Linear(21);
// random 0.5 is the middle of the range: no change
const auto same = UgcPalette::ApplyVariation(red, 7.0f, 0.5f);
EXPECT_NEAR(same.r, red.r, 1e-5f);
EXPECT_NEAR(same.b, red.b, 1e-5f);
// The top of the range: value^(1/2.224) + variation/200, back to the power of 2.224; hue and saturation kept
const auto brighter = UgcPalette::ApplyVariation(red, 7.0f, 1.0f);
const float expected = std::pow(std::pow(red.r, 1.0f / 2.224f) + 0.035f, 2.224f);
EXPECT_NEAR(brighter.r, expected, 1e-5f);
EXPECT_NEAR(brighter.b / brighter.r, red.b / red.r, 1e-5f);
const auto darker = UgcPalette::ApplyVariation(red, 7.0f, 0.0f);
EXPECT_LT(darker.r, red.r);
// Clamped to 0..1, and black turns grey rather than staying black
EXPECT_LE(UgcPalette::ApplyVariation(glm::vec3(1.0f), 100.0f, 1.0f).r, 1.0f);
EXPECT_GT(UgcPalette::ApplyVariation(glm::vec3(0.0f), 10.0f, 1.0f).g, 0.0f);
EXPECT_FLOAT_EQ(UgcPalette::ApplyVariation(red, 0.0f, 1.0f).r, red.r);
// Per color amounts, aliases, transparency, glow and the icon's corrections
EXPECT_FLOAT_EQ(UgcPalette::VariationScale(26), 0.4f);
EXPECT_FLOAT_EQ(UgcPalette::VariationScale(5), 1.0f);
EXPECT_EQ(*UgcPalette::Linear(0), *UgcPalette::Linear(26));
EXPECT_EQ(*UgcPalette::Linear(293), *UgcPalette::Linear(43));
EXPECT_TRUE(UgcPalette::IsTransparent(40));
EXPECT_FALSE(UgcPalette::IsTransparent(21));
EXPECT_TRUE(UgcPalette::Glow(9013).has_value());
EXPECT_FALSE(UgcPalette::Glow(21).has_value());
EXPECT_TRUE(UgcPalette::IsMetallic(309));
EXPECT_FALSE(UgcPalette::Linear(123456).has_value());
EXPECT_NEAR(UgcPalette::LinearToSrgb(*UgcPalette::Linear(1, true)).r, 0.7f, 1e-5f);
EXPECT_NEAR(UgcPalette::LinearToSrgb(red).r * 255.0f, 222.0f, 0.5f); // LDD's bright red
EXPECT_NEAR(UgcPalette::SrgbToLinear(UgcPalette::LinearToSrgb(0.3f)), 0.3f, 1e-5f);
}
TEST(UgcPalette, BrickRandomIsStableAndSpread) {
EXPECT_EQ(UgcPalette::BrickRandom(7, 3, 21), UgcPalette::BrickRandom(7, 3, 21));
EXPECT_NE(UgcPalette::BrickRandom(7, 3, 21), UgcPalette::BrickRandom(7, 4, 21));
EXPECT_NE(UgcPalette::BrickRandom(7, 3, 21), UgcPalette::BrickRandom(8, 3, 21));
EXPECT_NE(UgcPalette::BrickRandom(7, 3, 21), UgcPalette::BrickRandom(7, 3, 23));
double sum = 0.0;
float low = 1.0f, high = 0.0f;
for (uint32_t brick = 0; brick < 10000; brick++) {
const float value = UgcPalette::BrickRandom(1, brick, 1);
ASSERT_GE(value, 0.0f);
ASSERT_LT(value, 1.0f);
sum += value;
low = std::min(low, value);
high = std::max(high, value);
}
EXPECT_NEAR(sum / 10000.0, 0.5, 0.02); // uniform, like random.uniform
EXPECT_LT(low, 0.01f);
EXPECT_GT(high, 0.99f);
}
TEST(UgcModel, ColorsLikeLuToolbox) {
const auto res = MakeRes();
std::filesystem::create_directories(res / "brickprimitives" / "lod1");
std::ofstream(res / "brickprimitives" / "lod1" / "3001.g", std::ios::binary) << BoxGeometry(glm::vec3(0.0f), glm::vec3(1.0f));
UgcBricks::BrickLibrary library(res, 0);
std::string error;
// Three red bricks, a transparent one, a red and transparent one, an unknown color
const auto parts = UgcModel::ParseLxfml(R"(<LXFML versionMajor="5"><Bricks>
<Brick><Part designID="3001" materials="21"><Bone transformation="1,0,0,0,1,0,0,0,1,0,0,0"/></Part></Brick>
<Brick><Part designID="3001" materials="21"><Bone transformation="1,0,0,0,1,0,0,0,1,2,0,0"/></Part></Brick>
<Brick><Part designID="3001" materials="21"><Bone transformation="1,0,0,0,1,0,0,0,1,4,0,0"/></Part></Brick>
<Brick><Part designID="3001" materials="40"><Bone transformation="1,0,0,0,1,0,0,0,1,6,0,0"/></Part></Brick>
<Brick><Part designID="3001" materials="21,40"><Bone transformation="1,0,0,0,1,0,0,0,1,8,0,0"/></Part></Brick>
<Brick><Part designID="3001" materials="987654"><Bone transformation="1,0,0,0,1,0,0,0,1,10,0,0"/></Part></Brick>
</Bricks></LXFML>)", error);
ASSERT_EQ(parts.size(), 6u);
UgcModel::BuildOptions plain;
plain.colorVariation = 0.0f;
const auto flat = UgcModel::Build(parts, library, plain);
EXPECT_EQ(flat.transparent.TriangleCount(), 12u); // only the all-transparent brick
EXPECT_EQ(flat.opaque.TriangleCount(), 60u);
EXPECT_NEAR(flat.opaque.colors[0].r * 255.0f, 222.0f, 0.5f);
EXPECT_FLOAT_EQ(flat.opaque.colors[0].a, 1.0f);
EXPECT_NEAR(flat.transparent.colors[0].a, 0.5882f, 1e-4f);
const auto black = UgcPalette::LinearToSrgb(*UgcPalette::Linear(26));
EXPECT_NEAR(flat.opaque.colors[4 * 8].r, black.r, 1e-5f); // the unknown color is black
EXPECT_TRUE(flat.opaque.glow.empty());
UgcModel::BuildOptions varied;
varied.seed = 42;
const auto a = UgcModel::Build(parts, library, varied);
const auto again = UgcModel::Build(parts, library, varied);
EXPECT_EQ(a.opaque.colors, again.opaque.colors); // the same every time
// Each brick has one shift for all its vertices, different between bricks of the same color
EXPECT_EQ(a.opaque.colors[0], a.opaque.colors[7]);
EXPECT_NE(a.opaque.colors[0], a.opaque.colors[8]);
EXPECT_NE(a.opaque.colors[8], a.opaque.colors[16]);
// Within 5% x 1.4 (red's own amount) of the plain color in LU Toolbox's gamma
for (size_t brick = 0; brick < 3; brick++) {
const float value = UgcPalette::SrgbToLinear(a.opaque.colors[brick * 8].r);
const float base = UgcPalette::Linear(21)->r;
EXPECT_LE(std::abs(std::pow(value, 1.0f / 2.224f) - std::pow(base, 1.0f / 2.224f)), 0.035f + 1e-4f);
}
// The same brick gets the same color in another LOD
varied.lod = 1;
const auto lod1 = UgcModel::Build(parts, library, varied);
EXPECT_EQ(lod1.opaque.colors[8], a.opaque.colors[8]);
// Another model (seed) gets other shifts
varied.lod = 0;
varied.seed = 43;
EXPECT_NE(UgcModel::Build(parts, library, varied).opaque.colors[0], a.opaque.colors[0]);
// The icon: its corrections, and no variation
UgcModel::BuildOptions icon;
icon.icon = true;
icon.colorVariation = 0.0f;
const auto white = UgcModel::Build(UgcModel::ParseLxfml(R"(<LXFML versionMajor="5"><Bricks><Brick><Part designID="3001" materials="1">
<Bone transformation="1,0,0,0,1,0,0,0,1,0,0,0"/></Part></Brick></Bricks></LXFML>)", error), library, icon);
EXPECT_NEAR(white.opaque.colors[0].r, 0.7f, 1e-5f);
}
TEST(UgcModel, LodRangesLikeLuToolbox) {
const UgcModel::LodDistances d;
using Ranges = std::vector<std::pair<float, float>>;
EXPECT_EQ(UgcModel::LodRanges({ 0, 2 }, d), (Ranges{ { 0.0f, 100.0f }, { 100.0f, 10000.0f } }));
EXPECT_EQ(UgcModel::LodRanges({ 0 }, d), (Ranges{ { 0.0f, 10000.0f } }));
EXPECT_EQ(UgcModel::LodRanges({ 0, 1, 2 }, d), (Ranges{ { 0.0f, 50.0f }, { 50.0f, 100.0f }, { 100.0f, 10000.0f } }));
EXPECT_EQ(UgcModel::LodRanges({ 0, 1 }, d), (Ranges{ { 0.0f, 50.0f }, { 50.0f, 10000.0f } }));
EXPECT_EQ(UgcModel::LodRanges({ 0, 2, 3 }, d), (Ranges{ { 0.0f, 100.0f }, { 100.0f, 280.0f }, { 280.0f, 10000.0f } }));
}
TEST(UgcModel, DividesAlongTheLongestSide) {
// Two separate strips of triangles far apart on x: divided between them, each kept whole
UgcModel::Mesh mesh;
for (int cluster = 0; cluster < 2; cluster++) {
for (uint32_t i = 0; i < 40; i++) {
mesh.positions.push_back(glm::vec3(cluster * 100.0f + static_cast<float>(i % 2), static_cast<float>(i / 2), 0.0f));
mesh.normals.push_back(glm::vec3(0, 0, 1));
mesh.colors.push_back(glm::vec4(1.0f));
}
const uint32_t base = cluster * 40;
for (uint32_t i = 0; i + 2 < 40; i++) mesh.indices.insert(mesh.indices.end(), { base + i, base + i + 1, base + i + 2 });
}
const auto pieces = UgcModel::Divide(mesh, 50, 1000);
ASSERT_EQ(pieces.size(), 2u);
for (const auto& piece : pieces) {
EXPECT_EQ(piece.positions.size(), 40u);
EXPECT_EQ(piece.TriangleCount(), 38u);
}
EXPECT_EQ(UgcModel::Divide(mesh, 100, 1000).size(), 1u);
}
TEST(UgcFormats, LodNifReadsBack) {
UgcModel::Mesh near, far;
near.positions = { { 0, 0, 0 }, { 1, 0, 0 }, { 0, 1, 0 }, { 1, 1, 0 } };
near.normals.assign(4, { 0, 0, 1 });
near.colors.assign(4, { 1, 0, 0, 1 });
near.indices = { 0, 1, 2, 1, 3, 2 };
far = near;
far.indices = { 0, 1, 2 };
const auto nif = UgcFormats::WriteLodNif("SceneNode_Model", { { "S01_Opaque_Model", false, { { 0.0f, 100.0f, "LOD_0", { &near } }, { 100.0f, 10000.0f, "LOD_2", { &far } } } } });
std::string error;
const auto lod0 = NifFile::Parse(nif, 0, error);
ASSERT_TRUE(lod0) << error;
EXPECT_TRUE(lod0->skipped.empty());
ASSERT_EQ(lod0->meshes.size(), 1u);
EXPECT_EQ(lod0->meshes[0].indices.size(), 6u);
EXPECT_TRUE(lod0->nodes.contains("S01_Opaque_Model"));
EXPECT_TRUE(lod0->nodes.contains("LOD_0"));
const auto lod1 = NifFile::Parse(nif, 1, error);
ASSERT_TRUE(lod1) << error;
ASSERT_EQ(lod1->meshes.size(), 1u);
EXPECT_EQ(lod1->meshes[0].indices.size(), 3u);
}
TEST(UgcRender, AmbientOcclusionUnderARoof) {
// A floor vertex under a low roof is dark, one out in the open is lit; nothing is hit past the distance
UgcModel::Mesh points;
points.positions = { { 0, 0, 0 }, { 50, 0, 0 } };
points.normals = { { 0, 1, 0 }, { 0, 1, 0 } };
UgcModel::Mesh roof;
roof.positions = { { -10, 1, -10 }, { 10, 1, -10 }, { -10, 1, 10 }, { 10, 1, 10 } };
roof.normals.assign(4, { 0, -1, 0 });
roof.indices = { 0, 1, 2, 1, 3, 2 };
const auto ao = UgcRender::AmbientOcclusion(points, roof, 5.0f, 64);
ASSERT_EQ(ao.size(), 2u);
EXPECT_LT(ao[0], 0.2f);
EXPECT_FLOAT_EQ(ao[1], 1.0f);
EXPECT_FLOAT_EQ(UgcRender::AmbientOcclusion(points, roof, 0.5f, 64)[0], 1.0f);
// Baking darkens the colors of occluded vertices only, and glow lights them up again
UgcModel::Model model;
model.opaque = roof;
model.opaque.colors.assign(4, glm::vec4(0.8f, 0.8f, 0.8f, 1.0f));
UgcModel::Mesh floor = roof;
for (auto& p : floor.positions) p.y = 0.0f;
floor.normals.assign(4, { 0, 1, 0 });
floor.colors.assign(4, glm::vec4(0.8f, 0.8f, 0.8f, 1.0f));
model.opaque.Append(floor);
UgcRender::BakeAo(model, UgcRender::AoOptions{});
EXPECT_LT(model.opaque.colors[5].r, 0.8f);
EXPECT_EQ(model.opaque.colors[5].a, 1.0f);
}
TEST(UgcThrottle, KeepsUnderTheBudget) {
int from = -1, to = -1;
EXPECT_TRUE(UgcThrottle::ParseHours("22-6", from, to));
EXPECT_TRUE(UgcThrottle::InHours(23, from, to));
EXPECT_TRUE(UgcThrottle::InHours(3, from, to));
EXPECT_FALSE(UgcThrottle::InHours(12, from, to));
EXPECT_FALSE(UgcThrottle::ParseHours("", from, to));
EXPECT_FALSE(UgcThrottle::ParseHours("25-3", from, to));
EXPECT_FALSE(UgcThrottle::InHours(3, -1, -1));
// 0.6 s of CPU work at a quarter of a CPU takes at least (0.6 - the burst) / 0.25 s
UgcThrottle::SetBudget(0.25);
UgcThrottle::Begin();
const auto start = std::chrono::steady_clock::now();
const double cpuStart = UgcThrottle::ThreadCpuSeconds();
volatile double sink = 0.0;
while (UgcThrottle::ThreadCpuSeconds() - cpuStart < 0.6) {
for (int i = 0; i < 10000; i++) sink = sink + std::sqrt(static_cast<double>(i));
UgcThrottle::Checkpoint();
}
const double wall = std::chrono::duration<double>(std::chrono::steady_clock::now() - start).count();
UgcThrottle::SetBudget(0.0);
EXPECT_GE(wall, 1.2);
EXPECT_GT(UgcThrottle::GetStats().sleptMs, 0u);
}
TEST(UgcJobs, MakesLodsStatsAndIcons) {
UgcBricks::BrickLibrary library(MakeRes(), 0);
UgcJobs::Settings settings;
settings.optimize.resolution = 128;
settings.ao.samples = 8;
settings.icon.size = 32;
settings.icon.supersample = 1;
settings.icon.ao.samples = 4;
const auto outcome = UgcJobs::ProcessModel(LXFML5, library, settings, 99);
ASSERT_TRUE(outcome.ok) << outcome.error;
for (const auto* name : { "model.nif", "model.nif.gz", "model.nif.checksum", "model.noao.nif", "icon.png", "icon.dds.gz", "stats.json", "model.lxfml.gz" }) {
EXPECT_TRUE(outcome.files.contains(name)) << name;
}
EXPECT_NE(outcome.stats.find("\"lods\""), std::string::npos);
EXPECT_NE(outcome.stats.find("\"opaqueAfter\""), std::string::npos);
std::string error;
const auto nif = NifFile::Parse(outcome.files.at("model.nif"), 0, error);
ASSERT_TRUE(nif) << error;
EXPECT_TRUE(nif->nodes.contains("S01_Opaque_Model"));
EXPECT_TRUE(nif->nodes.contains("S01_Alpha_Model"));
EXPECT_TRUE(nif->nodes.contains("LOD_0"));
const auto far = NifFile::Parse(outcome.files.at("model.nif"), 1, error);
ASSERT_TRUE(far) << error;
EXPECT_TRUE(far->nodes.contains("LOD_2"));
// The same colors when made again
EXPECT_EQ(UgcJobs::ProcessModel(LXFML5, library, settings, 99).files.at("model.nif"), outcome.files.at("model.nif"));
settings.maxBricks = 2;
const auto tooBig = UgcJobs::ProcessModel(LXFML5, library, settings, 99);
EXPECT_FALSE(tooBig.ok);
EXPECT_NE(tooBig.error.find("max_model_bricks"), std::string::npos);
EXPECT_EQ(UgcJobs::CountParts(LXFML5), 3u);
EXPECT_GT(UgcJobs::EstimateMemory(1000, settings), UgcJobs::EstimateMemory(10, settings));
}