feat(ugc): make models like LU Toolbox does, within CPU and memory budgets

Parity with LU Toolbox's Process Model, Bake Lighting and icon renderer, with
its defaults as the settings' defaults:

- Colors from its LU palette (UgcPalette: LU colors, LDD colors mapped to the
  nearest LU one, unknown ones black), transparent bricks at 58.82% opacity, a
  brick transparent only when all of its materials are.
- Color variation: each brick's material gets its HSV value shifted in a 2.224
  gamma by up to 5% (times the color's own amount), from a random number of
  the model, brick and material, so reprocessing gives the same colors in
  every LOD. Icons get none, and the icon renderer's color corrections.
- LODs 0 and 2 with its distance logic, written as NiLODNode/NiRangeLODData
  like its exports and the game's own brick models, shapes divided at 65535
  vertices along the longest side like divide_mesh.
- Ambient occlusion like its AO-only bake: 64 rays per vertex, distance 5,
  after hidden surface removal, transparent bricks neither baked nor
  occluding, glow colors added.
- Icons from its icon scene: 50 mm lens at 53.4/19.5 degrees, sun of 2.5 at
  21/50.3 degrees with soft shadows, grey world light with occlusion; LOD 0's
  hidden surface removal and occlusion are reused for them.
- Optional ground plane for hidden surface removal; stats.json per model and
  the previous version's previews kept for comparing.

Budgets, applied live on config reload: max_cpu_percent (workers account
their thread CPU time and sleep to stay under it, long renders included),
worker_nice, max_memory_mb (jobs are estimated from their brick count and
wait until they fit), max_model_bricks and pause_hours. /status and the
traffic report show CPU, memory and throttling.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-09-27 09:19:59 -05:00
parent c8ad83b4f0
commit c27735cbb3
21 changed files with 1875 additions and 232 deletions

View File

@@ -11,9 +11,12 @@
#include "UgcBricks.h"
#include "UgcFormats.h"
#include "UgcModel.h"
#include "UgcJobs.h"
#include "UgcModular.h"
#include "UgcPalette.h"
#include "UgcRender.h"
#include "UgcStorage.h"
#include "UgcThrottle.h"
#include "ZCompression.h"
class Logger;
@@ -137,7 +140,10 @@ TEST(UgcModel, BuildsOpaqueAndTransparentMeshes) {
UgcBricks::BrickLibrary library(MakeRes(), 0);
library.SetMaterials({ { 21, { 222, 0, 13, 255 } }, { 40, { 238, 238, 238, 150 } } });
std::string error;
const auto model = UgcModel::Build(UgcModel::ParseLxfml(LXFML5, error), library);
UgcModel::BuildOptions options;
options.palette = UgcModel::ePalette::BRICKDB;
options.colorVariation = 0.0f;
const auto model = UgcModel::Build(UgcModel::ParseLxfml(LXFML5, error), library, options);
EXPECT_EQ(model.bricks, 2u);
EXPECT_EQ(model.missingDesigns, std::vector<uint32_t>{ 9999 });
EXPECT_EQ(model.opaque.TriangleCount(), 12u);
@@ -176,7 +182,7 @@ TEST(UgcRender, RemovesWhatIsInsideAndDrawsIcons) {
</Bricks></LXFML>)", error);
auto model = UgcModel::Build(parts, library);
ASSERT_EQ(model.opaque.TriangleCount(), 24u);
const auto result = UgcRender::Optimize(model, UgcRender::OptimizeOptions{ 256, true, true, 0.6f });
const auto result = UgcRender::Optimize(model, UgcRender::OptimizeOptions{ 256, true });
EXPECT_EQ(result.trianglesRemoved, 12u);
EXPECT_EQ(model.opaque.TriangleCount(), 12u);
EXPECT_EQ(model.opaque.positions.size(), 8u);
@@ -280,12 +286,277 @@ TEST(UgcStorage, WritesListsAndEvicts) {
ASSERT_TRUE(storage.Write(UgcStorage::Kind::MODULAR, 2002, { { "icon.png", std::string(100, 'b') } }, error)) << error;
ASSERT_TRUE(storage.Write(UgcStorage::Kind::MODEL, 1001, { { "icon.png", std::string(50, 'c') } }, error)) << error; // replaced
EXPECT_TRUE(storage.File(UgcStorage::Kind::MODEL, 1001, "icon.png"));
// The version before is kept to compare with
const auto previous = storage.File(UgcStorage::Kind::MODEL, 1001, "previous.icon.png");
ASSERT_TRUE(previous);
EXPECT_EQ(std::filesystem::file_size(*previous), 100u);
EXPECT_FALSE(storage.File(UgcStorage::Kind::MODEL, 1001, "../../etc/passwd"));
EXPECT_EQ(storage.List().size(), 2u);
std::filesystem::last_write_time(storage.Folder(UgcStorage::Kind::MODULAR, 2002), std::filesystem::file_time_type::clock::now() - std::chrono::hours(1));
const auto removed = storage.Evict(60);
const auto removed = storage.Evict(160);
ASSERT_EQ(removed.size(), 1u);
EXPECT_EQ(removed[0].id, 2002);
EXPECT_TRUE(storage.File(UgcStorage::Kind::MODEL, 1001, "icon.png"));
std::filesystem::remove_all(storage.GetRoot());
}
TEST(UgcPalette, ColorVariationMatchesLuToolbox) {
const glm::vec3 red = *UgcPalette::Linear(21);
// random 0.5 is the middle of the range: no change
const auto same = UgcPalette::ApplyVariation(red, 7.0f, 0.5f);
EXPECT_NEAR(same.r, red.r, 1e-5f);
EXPECT_NEAR(same.b, red.b, 1e-5f);
// The top of the range: value^(1/2.224) + variation/200, back to the power of 2.224; hue and saturation kept
const auto brighter = UgcPalette::ApplyVariation(red, 7.0f, 1.0f);
const float expected = std::pow(std::pow(red.r, 1.0f / 2.224f) + 0.035f, 2.224f);
EXPECT_NEAR(brighter.r, expected, 1e-5f);
EXPECT_NEAR(brighter.b / brighter.r, red.b / red.r, 1e-5f);
const auto darker = UgcPalette::ApplyVariation(red, 7.0f, 0.0f);
EXPECT_LT(darker.r, red.r);
// Clamped to 0..1, and black turns grey rather than staying black
EXPECT_LE(UgcPalette::ApplyVariation(glm::vec3(1.0f), 100.0f, 1.0f).r, 1.0f);
EXPECT_GT(UgcPalette::ApplyVariation(glm::vec3(0.0f), 10.0f, 1.0f).g, 0.0f);
EXPECT_FLOAT_EQ(UgcPalette::ApplyVariation(red, 0.0f, 1.0f).r, red.r);
// Per color amounts, aliases, transparency, glow and the icon's corrections
EXPECT_FLOAT_EQ(UgcPalette::VariationScale(26), 0.4f);
EXPECT_FLOAT_EQ(UgcPalette::VariationScale(5), 1.0f);
EXPECT_EQ(*UgcPalette::Linear(0), *UgcPalette::Linear(26));
EXPECT_EQ(*UgcPalette::Linear(293), *UgcPalette::Linear(43));
EXPECT_TRUE(UgcPalette::IsTransparent(40));
EXPECT_FALSE(UgcPalette::IsTransparent(21));
EXPECT_TRUE(UgcPalette::Glow(9013).has_value());
EXPECT_FALSE(UgcPalette::Glow(21).has_value());
EXPECT_TRUE(UgcPalette::IsMetallic(309));
EXPECT_FALSE(UgcPalette::Linear(123456).has_value());
EXPECT_NEAR(UgcPalette::LinearToSrgb(*UgcPalette::Linear(1, true)).r, 0.7f, 1e-5f);
EXPECT_NEAR(UgcPalette::LinearToSrgb(red).r * 255.0f, 222.0f, 0.5f); // LDD's bright red
EXPECT_NEAR(UgcPalette::SrgbToLinear(UgcPalette::LinearToSrgb(0.3f)), 0.3f, 1e-5f);
}
TEST(UgcPalette, BrickRandomIsStableAndSpread) {
EXPECT_EQ(UgcPalette::BrickRandom(7, 3, 21), UgcPalette::BrickRandom(7, 3, 21));
EXPECT_NE(UgcPalette::BrickRandom(7, 3, 21), UgcPalette::BrickRandom(7, 4, 21));
EXPECT_NE(UgcPalette::BrickRandom(7, 3, 21), UgcPalette::BrickRandom(8, 3, 21));
EXPECT_NE(UgcPalette::BrickRandom(7, 3, 21), UgcPalette::BrickRandom(7, 3, 23));
double sum = 0.0;
float low = 1.0f, high = 0.0f;
for (uint32_t brick = 0; brick < 10000; brick++) {
const float value = UgcPalette::BrickRandom(1, brick, 1);
ASSERT_GE(value, 0.0f);
ASSERT_LT(value, 1.0f);
sum += value;
low = std::min(low, value);
high = std::max(high, value);
}
EXPECT_NEAR(sum / 10000.0, 0.5, 0.02); // uniform, like random.uniform
EXPECT_LT(low, 0.01f);
EXPECT_GT(high, 0.99f);
}
TEST(UgcModel, ColorsLikeLuToolbox) {
const auto res = MakeRes();
std::filesystem::create_directories(res / "brickprimitives" / "lod1");
std::ofstream(res / "brickprimitives" / "lod1" / "3001.g", std::ios::binary) << BoxGeometry(glm::vec3(0.0f), glm::vec3(1.0f));
UgcBricks::BrickLibrary library(res, 0);
std::string error;
// Three red bricks, a transparent one, a red and transparent one, an unknown color
const auto parts = UgcModel::ParseLxfml(R"(<LXFML versionMajor="5"><Bricks>
<Brick><Part designID="3001" materials="21"><Bone transformation="1,0,0,0,1,0,0,0,1,0,0,0"/></Part></Brick>
<Brick><Part designID="3001" materials="21"><Bone transformation="1,0,0,0,1,0,0,0,1,2,0,0"/></Part></Brick>
<Brick><Part designID="3001" materials="21"><Bone transformation="1,0,0,0,1,0,0,0,1,4,0,0"/></Part></Brick>
<Brick><Part designID="3001" materials="40"><Bone transformation="1,0,0,0,1,0,0,0,1,6,0,0"/></Part></Brick>
<Brick><Part designID="3001" materials="21,40"><Bone transformation="1,0,0,0,1,0,0,0,1,8,0,0"/></Part></Brick>
<Brick><Part designID="3001" materials="987654"><Bone transformation="1,0,0,0,1,0,0,0,1,10,0,0"/></Part></Brick>
</Bricks></LXFML>)", error);
ASSERT_EQ(parts.size(), 6u);
UgcModel::BuildOptions plain;
plain.colorVariation = 0.0f;
const auto flat = UgcModel::Build(parts, library, plain);
EXPECT_EQ(flat.transparent.TriangleCount(), 12u); // only the all-transparent brick
EXPECT_EQ(flat.opaque.TriangleCount(), 60u);
EXPECT_NEAR(flat.opaque.colors[0].r * 255.0f, 222.0f, 0.5f);
EXPECT_FLOAT_EQ(flat.opaque.colors[0].a, 1.0f);
EXPECT_NEAR(flat.transparent.colors[0].a, 0.5882f, 1e-4f);
const auto black = UgcPalette::LinearToSrgb(*UgcPalette::Linear(26));
EXPECT_NEAR(flat.opaque.colors[4 * 8].r, black.r, 1e-5f); // the unknown color is black
EXPECT_TRUE(flat.opaque.glow.empty());
UgcModel::BuildOptions varied;
varied.seed = 42;
const auto a = UgcModel::Build(parts, library, varied);
const auto again = UgcModel::Build(parts, library, varied);
EXPECT_EQ(a.opaque.colors, again.opaque.colors); // the same every time
// Each brick has one shift for all its vertices, different between bricks of the same color
EXPECT_EQ(a.opaque.colors[0], a.opaque.colors[7]);
EXPECT_NE(a.opaque.colors[0], a.opaque.colors[8]);
EXPECT_NE(a.opaque.colors[8], a.opaque.colors[16]);
// Within 5% x 1.4 (red's own amount) of the plain color in LU Toolbox's gamma
for (size_t brick = 0; brick < 3; brick++) {
const float value = UgcPalette::SrgbToLinear(a.opaque.colors[brick * 8].r);
const float base = UgcPalette::Linear(21)->r;
EXPECT_LE(std::abs(std::pow(value, 1.0f / 2.224f) - std::pow(base, 1.0f / 2.224f)), 0.035f + 1e-4f);
}
// The same brick gets the same color in another LOD
varied.lod = 1;
const auto lod1 = UgcModel::Build(parts, library, varied);
EXPECT_EQ(lod1.opaque.colors[8], a.opaque.colors[8]);
// Another model (seed) gets other shifts
varied.lod = 0;
varied.seed = 43;
EXPECT_NE(UgcModel::Build(parts, library, varied).opaque.colors[0], a.opaque.colors[0]);
// The icon: its corrections, and no variation
UgcModel::BuildOptions icon;
icon.icon = true;
icon.colorVariation = 0.0f;
const auto white = UgcModel::Build(UgcModel::ParseLxfml(R"(<LXFML versionMajor="5"><Bricks><Brick><Part designID="3001" materials="1">
<Bone transformation="1,0,0,0,1,0,0,0,1,0,0,0"/></Part></Brick></Bricks></LXFML>)", error), library, icon);
EXPECT_NEAR(white.opaque.colors[0].r, 0.7f, 1e-5f);
}
TEST(UgcModel, LodRangesLikeLuToolbox) {
const UgcModel::LodDistances d;
using Ranges = std::vector<std::pair<float, float>>;
EXPECT_EQ(UgcModel::LodRanges({ 0, 2 }, d), (Ranges{ { 0.0f, 100.0f }, { 100.0f, 10000.0f } }));
EXPECT_EQ(UgcModel::LodRanges({ 0 }, d), (Ranges{ { 0.0f, 10000.0f } }));
EXPECT_EQ(UgcModel::LodRanges({ 0, 1, 2 }, d), (Ranges{ { 0.0f, 50.0f }, { 50.0f, 100.0f }, { 100.0f, 10000.0f } }));
EXPECT_EQ(UgcModel::LodRanges({ 0, 1 }, d), (Ranges{ { 0.0f, 50.0f }, { 50.0f, 10000.0f } }));
EXPECT_EQ(UgcModel::LodRanges({ 0, 2, 3 }, d), (Ranges{ { 0.0f, 100.0f }, { 100.0f, 280.0f }, { 280.0f, 10000.0f } }));
}
TEST(UgcModel, DividesAlongTheLongestSide) {
// Two separate strips of triangles far apart on x: divided between them, each kept whole
UgcModel::Mesh mesh;
for (int cluster = 0; cluster < 2; cluster++) {
for (uint32_t i = 0; i < 40; i++) {
mesh.positions.push_back(glm::vec3(cluster * 100.0f + static_cast<float>(i % 2), static_cast<float>(i / 2), 0.0f));
mesh.normals.push_back(glm::vec3(0, 0, 1));
mesh.colors.push_back(glm::vec4(1.0f));
}
const uint32_t base = cluster * 40;
for (uint32_t i = 0; i + 2 < 40; i++) mesh.indices.insert(mesh.indices.end(), { base + i, base + i + 1, base + i + 2 });
}
const auto pieces = UgcModel::Divide(mesh, 50, 1000);
ASSERT_EQ(pieces.size(), 2u);
for (const auto& piece : pieces) {
EXPECT_EQ(piece.positions.size(), 40u);
EXPECT_EQ(piece.TriangleCount(), 38u);
}
EXPECT_EQ(UgcModel::Divide(mesh, 100, 1000).size(), 1u);
}
TEST(UgcFormats, LodNifReadsBack) {
UgcModel::Mesh near, far;
near.positions = { { 0, 0, 0 }, { 1, 0, 0 }, { 0, 1, 0 }, { 1, 1, 0 } };
near.normals.assign(4, { 0, 0, 1 });
near.colors.assign(4, { 1, 0, 0, 1 });
near.indices = { 0, 1, 2, 1, 3, 2 };
far = near;
far.indices = { 0, 1, 2 };
const auto nif = UgcFormats::WriteLodNif("SceneNode_Model", { { "S01_Opaque_Model", false, { { 0.0f, 100.0f, "LOD_0", { &near } }, { 100.0f, 10000.0f, "LOD_2", { &far } } } } });
std::string error;
const auto lod0 = NifFile::Parse(nif, 0, error);
ASSERT_TRUE(lod0) << error;
EXPECT_TRUE(lod0->skipped.empty());
ASSERT_EQ(lod0->meshes.size(), 1u);
EXPECT_EQ(lod0->meshes[0].indices.size(), 6u);
EXPECT_TRUE(lod0->nodes.contains("S01_Opaque_Model"));
EXPECT_TRUE(lod0->nodes.contains("LOD_0"));
const auto lod1 = NifFile::Parse(nif, 1, error);
ASSERT_TRUE(lod1) << error;
ASSERT_EQ(lod1->meshes.size(), 1u);
EXPECT_EQ(lod1->meshes[0].indices.size(), 3u);
}
TEST(UgcRender, AmbientOcclusionUnderARoof) {
// A floor vertex under a low roof is dark, one out in the open is lit; nothing is hit past the distance
UgcModel::Mesh points;
points.positions = { { 0, 0, 0 }, { 50, 0, 0 } };
points.normals = { { 0, 1, 0 }, { 0, 1, 0 } };
UgcModel::Mesh roof;
roof.positions = { { -10, 1, -10 }, { 10, 1, -10 }, { -10, 1, 10 }, { 10, 1, 10 } };
roof.normals.assign(4, { 0, -1, 0 });
roof.indices = { 0, 1, 2, 1, 3, 2 };
const auto ao = UgcRender::AmbientOcclusion(points, roof, 5.0f, 64);
ASSERT_EQ(ao.size(), 2u);
EXPECT_LT(ao[0], 0.2f);
EXPECT_FLOAT_EQ(ao[1], 1.0f);
EXPECT_FLOAT_EQ(UgcRender::AmbientOcclusion(points, roof, 0.5f, 64)[0], 1.0f);
// Baking darkens the colors of occluded vertices only, and glow lights them up again
UgcModel::Model model;
model.opaque = roof;
model.opaque.colors.assign(4, glm::vec4(0.8f, 0.8f, 0.8f, 1.0f));
UgcModel::Mesh floor = roof;
for (auto& p : floor.positions) p.y = 0.0f;
floor.normals.assign(4, { 0, 1, 0 });
floor.colors.assign(4, glm::vec4(0.8f, 0.8f, 0.8f, 1.0f));
model.opaque.Append(floor);
UgcRender::BakeAo(model, UgcRender::AoOptions{});
EXPECT_LT(model.opaque.colors[5].r, 0.8f);
EXPECT_EQ(model.opaque.colors[5].a, 1.0f);
}
TEST(UgcThrottle, KeepsUnderTheBudget) {
int from = -1, to = -1;
EXPECT_TRUE(UgcThrottle::ParseHours("22-6", from, to));
EXPECT_TRUE(UgcThrottle::InHours(23, from, to));
EXPECT_TRUE(UgcThrottle::InHours(3, from, to));
EXPECT_FALSE(UgcThrottle::InHours(12, from, to));
EXPECT_FALSE(UgcThrottle::ParseHours("", from, to));
EXPECT_FALSE(UgcThrottle::ParseHours("25-3", from, to));
EXPECT_FALSE(UgcThrottle::InHours(3, -1, -1));
// 0.6 s of CPU work at a quarter of a CPU takes at least (0.6 - the burst) / 0.25 s
UgcThrottle::SetBudget(0.25);
UgcThrottle::Begin();
const auto start = std::chrono::steady_clock::now();
const double cpuStart = UgcThrottle::ThreadCpuSeconds();
volatile double sink = 0.0;
while (UgcThrottle::ThreadCpuSeconds() - cpuStart < 0.6) {
for (int i = 0; i < 10000; i++) sink = sink + std::sqrt(static_cast<double>(i));
UgcThrottle::Checkpoint();
}
const double wall = std::chrono::duration<double>(std::chrono::steady_clock::now() - start).count();
UgcThrottle::SetBudget(0.0);
EXPECT_GE(wall, 1.2);
EXPECT_GT(UgcThrottle::GetStats().sleptMs, 0u);
}
TEST(UgcJobs, MakesLodsStatsAndIcons) {
UgcBricks::BrickLibrary library(MakeRes(), 0);
UgcJobs::Settings settings;
settings.optimize.resolution = 128;
settings.ao.samples = 8;
settings.icon.size = 32;
settings.icon.supersample = 1;
settings.icon.ao.samples = 4;
const auto outcome = UgcJobs::ProcessModel(LXFML5, library, settings, 99);
ASSERT_TRUE(outcome.ok) << outcome.error;
for (const auto* name : { "model.nif", "model.nif.gz", "model.nif.checksum", "model.noao.nif", "icon.png", "icon.dds.gz", "stats.json", "model.lxfml.gz" }) {
EXPECT_TRUE(outcome.files.contains(name)) << name;
}
EXPECT_NE(outcome.stats.find("\"lods\""), std::string::npos);
EXPECT_NE(outcome.stats.find("\"opaqueAfter\""), std::string::npos);
std::string error;
const auto nif = NifFile::Parse(outcome.files.at("model.nif"), 0, error);
ASSERT_TRUE(nif) << error;
EXPECT_TRUE(nif->nodes.contains("S01_Opaque_Model"));
EXPECT_TRUE(nif->nodes.contains("S01_Alpha_Model"));
EXPECT_TRUE(nif->nodes.contains("LOD_0"));
const auto far = NifFile::Parse(outcome.files.at("model.nif"), 1, error);
ASSERT_TRUE(far) << error;
EXPECT_TRUE(far->nodes.contains("LOD_2"));
// The same colors when made again
EXPECT_EQ(UgcJobs::ProcessModel(LXFML5, library, settings, 99).files.at("model.nif"), outcome.files.at("model.nif"));
settings.maxBricks = 2;
const auto tooBig = UgcJobs::ProcessModel(LXFML5, library, settings, 99);
EXPECT_FALSE(tooBig.ok);
EXPECT_NE(tooBig.error.find("max_model_bricks"), std::string::npos);
EXPECT_EQ(UgcJobs::CountParts(LXFML5), 3u);
EXPECT_GT(UgcJobs::EstimateMemory(1000, settings), UgcJobs::EstimateMemory(10, settings));
}