diff --git a/dCommon/UgcKeys.h b/dCommon/UgcKeys.h index 1bf9dcc64..989b72f19 100644 --- a/dCommon/UgcKeys.h +++ b/dCommon/UgcKeys.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -70,3 +71,59 @@ namespace UgcDebounce { return requested || processAfter <= now; } } + +/** + * How a model is made, where there is more than one way to try (docs/UgcServer.md, "Processing options"): what traces + * the rays (ray_backend), how hidden faces are found (hsr_method) and whether icons are denoised + * (denoise). The UGC settings give the defaults; staff may pick others for one make (the dashboard, /reprocessproperty), + * and each made model records what made it. Written as the chosen names in this order, separated by spaces + * ("embree fast oidn"); a choice left out is the setting's. + */ +namespace UgcProcessOptions { + inline constexpr std::string_view RAYS[] = { "builtin", "embree", "hiprt" }; + inline constexpr std::string_view HSR[] = { "toolbox", "fast" }; + inline constexpr std::string_view DENOISE[] = { "off", "oidn" }; + + struct Choice { + std::string rays; // one of RAYS, or empty for the setting + std::string hsr; // one of HSR, or empty + std::string denoise; // one of DENOISE, or empty + + bool Empty() const { return rays.empty() && hsr.empty() && denoise.empty(); } + }; + + template + inline bool Contains(const std::string_view (&names)[N], std::string_view word) { + return std::find(std::begin(names), std::end(names), word) != std::end(names); + } + + // The words, in any order ("fast embree"); "default" or "-" leave a choice to the setting. False on an unknown + // word or two words for one choice. + inline bool Parse(std::string_view text, Choice& choice) { + choice = {}; + size_t start = 0; + while (start < text.size()) { + while (start < text.size() && (text[start] == ' ' || text[start] == ',')) start++; + size_t end = start; + while (end < text.size() && text[end] != ' ' && text[end] != ',') end++; + const auto word = text.substr(start, end - start); + start = end; + if (word.empty() || word == "default" || word == "-") continue; + std::string* slot = Contains(RAYS, word) ? &choice.rays : Contains(HSR, word) ? &choice.hsr : Contains(DENOISE, word) ? &choice.denoise : nullptr; + if (!slot || !slot->empty()) return false; + *slot = std::string(word); + } + return true; + } + + // The choices made, in order ("embree oidn"); empty when all are the settings' + inline std::string ToString(const Choice& choice) { + std::string text; + for (const auto* part : { &choice.rays, &choice.hsr, &choice.denoise }) { + if (part->empty()) continue; + if (!text.empty()) text += ' '; + text += *part; + } + return text; + } +} diff --git a/dDashboardServer/routes/SettingsCatalog.cpp b/dDashboardServer/routes/SettingsCatalog.cpp index 1f9c731ca..70cab6234 100644 --- a/dDashboardServer/routes/SettingsCatalog.cpp +++ b/dDashboardServer/routes/SettingsCatalog.cpp @@ -501,6 +501,10 @@ namespace { c.Add(Int(UGC, "hsr_bounces", "Path bounces", "Bounces a path may take off the model before it gives up (Cycles' Max Bounces in LU Toolbox's bake).", "8", 0, 64)); c.Add(Float(UGC, "hsr_sample_spacing", "Point spacing", "Distance between the points paths start from, in LDD units (a stud is 0.8; 0.1143 puts 7 x 7 points on a stud-sized square). Smaller finds smaller visible parts of big faces; slower.", "0.1143", 0.01, 10)); c.Add(Int(UGC, "hsr_min_points", "Points per triangle at least", "Small triangles get this many points however close together (LU Toolbox bakes 28 texels for every triangle). Lower is faster but removes more faces that are only just seen.", "28", 1, 4096)); + c.Add(Labels(Choice(UGC, "hsr_method", "How hidden faces are found", "toolbox: LU Toolbox's paths (above). fast: the model rendered from 42 directions around it, removing what shows in none; much faster, but it also removes faces seen only by bounced light (insides seen through openings, recesses). Staff can pick another for one make when making models again.", "toolbox", { "toolbox", "fast" }), { "LU Toolbox (paths)", "Fast (renders from around)" })); + c.Add(Unit(Int(UGC, "hsr_fast_resolution", "Fast method's render size", "The size of each of the fast method's 42 renders; bigger keeps smaller visible faces.", "1024", 64, 4096), "pixels")); + c.Add(Labels(Choice(UGC, "ray_backend", "Ray tracer", "What traces the rays of the hidden faces' paths and of the occlusion: builtin (the UGC server's own), embree (Intel Embree on the CPU) or hiprt (the GPU, when the server was built with it and has one; else embree). They give the same results but for rounding. Staff can pick another for one make when making models again.", "builtin", { "builtin", "embree", "hiprt" }), { "Built in", "Embree (CPU)", "HIPRT (GPU)" })); + c.Add(Labels(Choice(UGC, "denoise", "Denoise icons", "oidn: icons are cleaned up with Intel Open Image Denoise (when the server was built with it; else off).", "off", { "off", "oidn" }), { "Off", "Open Image Denoise" })); c.Add(Bool(UGC, "bake_ao", "Darken hidden corners", "Ambient occlusion baked into the vertex colors.", true)); c.Add(Int(UGC, "ao_samples", "Occlusion rays per vertex", "", "64", 1, 1024)); c.Add(Float(UGC, "ao_distance", "Occlusion distance", "", "5", 0, 1000)); diff --git a/dUgcServer/Processing/UgcJobs.cpp b/dUgcServer/Processing/UgcJobs.cpp index 0f07a0f1f..7309a4f7b 100644 --- a/dUgcServer/Processing/UgcJobs.cpp +++ b/dUgcServer/Processing/UgcJobs.cpp @@ -169,6 +169,18 @@ namespace UgcJobs { return parsed.dump(); } + void ApplyOptions(Settings& settings, const UgcProcessOptions::Choice& choice) { + if (const auto rays = UgcRays::Parse(choice.rays)) settings.hsr.rays = settings.ao.rays = settings.icon.ao.rays = *rays; + if (const auto method = UgcHsr::Parse(choice.hsr)) settings.hsr.method = *method; + if (const auto denoise = UgcRender::ParseDenoise(choice.denoise)) settings.icon.denoise = *denoise; + } + + UgcProcessOptions::Choice MadeWith(const Settings& settings) { + const auto denoise = UgcRender::Available(settings.icon.denoise) ? settings.icon.denoise : UgcRender::eDenoise::OFF; + return { std::string(UgcRays::Name(UgcRays::Resolve(settings.hsr.rays))), std::string(UgcHsr::Name(settings.hsr.method)), + std::string(UgcRender::Name(denoise)) }; + } + Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings, uint64_t seed, const UgcIconParams::Values& iconValues) { Outcome outcome; const auto started = std::chrono::steady_clock::now(); @@ -355,6 +367,12 @@ namespace UgcJobs { { "removeHiddenFaces", settings.hsr.enabled }, { "groundPlane", settings.hsr.groundPlane }, { "hsrSamples", settings.hsr.samples }, { "hsrBounces", settings.hsr.bounces }, { "hsrSampleSpacing", settings.hsr.spacing }, { "hsrMinPoints", settings.hsr.minPoints }, { "ao", settings.ao.enabled }, { "aoDistance", settings.ao.distance }, { "aoSamples", settings.ao.samples }, { "aoStrength", settings.ao.strength } }; + const auto madeWith = MadeWith(settings); + stats["settings"]["rays"] = madeWith.rays; + stats["settings"]["hsrMethod"] = madeWith.hsr; + stats["settings"]["denoise"] = madeWith.denoise; + if (settings.hsr.method == UgcHsr::eMethod::FAST) stats["settings"]["hsrFastResolution"] = settings.hsr.fastResolution; + outcome.options = UgcProcessOptions::ToString(madeWith); outcome.stats = stats.dump(); outcome.files["stats.json"] = outcome.stats; outcome.ok = true; diff --git a/dUgcServer/Processing/UgcJobs.h b/dUgcServer/Processing/UgcJobs.h index f3c66ab9d..5089d1ea4 100644 --- a/dUgcServer/Processing/UgcJobs.h +++ b/dUgcServer/Processing/UgcJobs.h @@ -12,6 +12,7 @@ #include "UgcBricks.h" #include "UgcHsr.h" #include "UgcIconParams.h" +#include "UgcKeys.h" #include "UgcRender.h" #include "UgcStorage.h" @@ -64,6 +65,17 @@ namespace UgcJobs { uint32_t maxBricks{}; // a model with more fails; 0: no limit }; + /** + * The processing options (UgcProcessOptions) staff picked for one make, over the settings: ray backend (the hidden + * faces' paths and the occlusion rays, the icon's too), hidden-face method and denoising. Choices left empty keep + * the settings'. + */ + void ApplyOptions(Settings& settings, const UgcProcessOptions::Choice& choice); + + // What the settings make with, as it is used: the ray backend after its fallback (UgcRays::Resolve), off for a + // denoiser the build doesn't have; every choice filled ("embree toolbox off") + UgcProcessOptions::Choice MadeWith(const Settings& settings); + struct Outcome { bool ok{}; std::string error; // why it failed @@ -72,6 +84,7 @@ namespace UgcJobs { bool aoBaked{}; bool empty{}; // nothing to make (no bricks): not a failure std::string stats; // stats.json: bricks, triangles before and after per LOD, timings + std::string options; // players' models: what made them (MadeWith, as UgcProcessOptions::ToString) }; // The client's download of `data` (`name` + ".gz" and ".checksum") added to `files` diff --git a/dUgcServer/Render/UgcHsr.h b/dUgcServer/Render/UgcHsr.h index 646507f37..8c63e5763 100644 --- a/dUgcServer/Render/UgcHsr.h +++ b/dUgcServer/Render/UgcHsr.h @@ -37,7 +37,7 @@ namespace UgcHsr { float spacing{ 0.1143f }; // hsr_sample_spacing: LDD units between points (a stud is 0.8: 7 points along it) int minPoints{ 28 }; // hsr_min_points: points on a triangle at least (LU Toolbox bakes 28 texels a triangle) uint64_t seed{}; // of the paths' random numbers (the same seed gives the same result) - UgcRays::eBackend rays{}; // what traces the paths' rays (ugc_ray_backend) + UgcRays::eBackend rays{}; // what traces the paths' rays (ray_backend) int fastResolution{ 1024 }; // hsr_fast_resolution: pixels square of each of the fast method's renders }; diff --git a/dUgcServer/Render/UgcRender.cpp b/dUgcServer/Render/UgcRender.cpp index cb1c20792..d80cb88fc 100644 --- a/dUgcServer/Render/UgcRender.cpp +++ b/dUgcServer/Render/UgcRender.cpp @@ -94,6 +94,21 @@ namespace { } namespace UgcRender { + std::string_view Name(eDenoise denoise) { + return denoise == eDenoise::OIDN ? "oidn" : "off"; + } + + std::optional ParseDenoise(std::string_view name) { + for (const auto denoise : { eDenoise::OFF, eDenoise::OIDN }) { + if (Name(denoise) == name) return denoise; + } + return std::nullopt; + } + + bool Available(eDenoise denoise) { + return denoise == eDenoise::OFF; + } + std::vector SphereDirections() { const float t = (1.0f + std::sqrt(5.0f)) / 2.0f; const std::vector corners = { diff --git a/dUgcServer/Render/UgcRender.h b/dUgcServer/Render/UgcRender.h index fc730a5e3..3fe243139 100644 --- a/dUgcServer/Render/UgcRender.h +++ b/dUgcServer/Render/UgcRender.h @@ -1,6 +1,8 @@ #pragma once #include +#include +#include #include #include @@ -21,13 +23,23 @@ namespace UgcRender { std::vector rgba; }; + // Whether an icon is denoised (the denoise setting): off, or with Intel Open Image Denoise when the build has it + enum class eDenoise : uint8_t { OFF = 0, OIDN }; + + // The setting's name (off, oidn) + std::string_view Name(eDenoise denoise); + // A value by its name; nullopt for anything else + std::optional ParseDenoise(std::string_view name); + // Whether this build can denoise that way (off always) + bool Available(eDenoise denoise); + struct AoOptions { bool enabled{ true }; float distance{ 5.0f }; // LU Toolbox's AO distance (Bake Lighting, AO Only) int samples{ 64 }; // rays per vertex (AO Samples) float strength{ 1.0f }; // 0 leaves the colors, 1 is the full bake float glowStrength{ 6.0f }; // what glowing colors add to the light (Glow Strength 3 x Glow Multiplier 2) - UgcRays::eBackend rays{}; // what traces the occlusion rays (ugc_ray_backend) + UgcRays::eBackend rays{}; // what traces the occlusion rays (ray_backend) }; /** @@ -60,6 +72,7 @@ namespace UgcRender { AoOptions ao{ false, 5.0f, 32, 1.0f, 0.0f }; float glowEmissive{ 1.0f }; // how far glowing shapes go from lit to their plain color (the glow_emissive setting) UgcGlitter::Params glitter; // the glitter's flecks (glitter_size, glitter_density), drawn where they are at the start + eDenoise denoise{}; // the finished icon denoised (denoise); off when the build can't }; // The model drawn from the icon's camera, framed to fit, on a transparent background. `opaqueAo`: the opaque mesh's diff --git a/dUgcServer/UgcServer.cpp b/dUgcServer/UgcServer.cpp index 72822b4f9..449d07b41 100644 --- a/dUgcServer/UgcServer.cpp +++ b/dUgcServer/UgcServer.cpp @@ -159,6 +159,11 @@ namespace { settings.hsr.bounces = std::clamp(Setting("hsr_bounces", 8), 0, 64); settings.hsr.spacing = std::clamp(Setting("hsr_sample_spacing", 0.1143f), 0.01f, 10.0f); settings.hsr.minPoints = std::clamp(Setting("hsr_min_points", 28), 1, 4096); + settings.hsr.method = UgcHsr::Parse(Game::config->GetValue("hsr_method")).value_or(UgcHsr::eMethod::TOOLBOX); + settings.hsr.fastResolution = std::clamp(Setting("hsr_fast_resolution", 1024), 64, 4096); + // What traces the rays of the hidden faces' paths and of the occlusion (the icon's too) + settings.hsr.rays = UgcRays::Parse(Game::config->GetValue("ray_backend")).value_or(UgcRays::eBackend::BUILTIN); + settings.ao.rays = settings.hsr.rays; settings.ao.enabled = Setting("bake_ao", 1) != 0; settings.ao.distance = Setting("ao_distance", 5.0f); settings.ao.samples = std::clamp(Setting("ao_samples", 64), 1, 1024); @@ -173,6 +178,8 @@ namespace { settings.icon.glowEmissive = settings.shaders.glowEmissive; settings.icon.glitter = settings.shaders.glitterParams; settings.icon.ao.distance = settings.ao.distance; + settings.icon.ao.rays = settings.hsr.rays; + settings.icon.denoise = UgcRender::ParseDenoise(Game::config->GetValue("denoise")).value_or(UgcRender::eDenoise::OFF); settings.maxBricks = Setting("max_model_bricks", 0); return settings; } @@ -552,13 +559,20 @@ namespace { } // Command line tools: make one model's or modular build's files into a folder, without a database - int MakeFromCommandLine(const std::string& mode, const std::string& input, const std::filesystem::path& output) { + int MakeFromCommandLine(const std::string& mode, const std::string& input, const std::filesystem::path& output, const std::string& options) { const auto res = ResPath(); - const auto settings = ReadSettings(); + auto settings = ReadSettings(); + UgcProcessOptions::Choice choice; + if (!UgcProcessOptions::Parse(options, choice)) { + std::cerr << "Unknown processing options \"" << options << "\" (ray backend builtin, embree or hiprt; hidden faces toolbox or fast; denoise off or oidn)\n"; + return EXIT_FAILURE; + } + UgcJobs::ApplyOptions(settings, choice); UgcBricks::BrickLibrary library(res, 0, ClientReader()); if (!library.LoadMaterials()) std::cerr << "Couldn't read Materials.xml from " << (res / "brickdb.zip") << "; bricks will be grey\n"; UgcJobs::Outcome outcome; const auto start = std::chrono::steady_clock::now(); + const double cpuStart = UgcThrottle::ThreadCpuSeconds(); if (mode == "--make-model") { const auto data = UgcBricks::ReadFile(input); if (!data) { @@ -585,7 +599,9 @@ namespace { for (const auto& [name, data] : outcome.files) { std::ofstream(output / name, std::ios::binary).write(data.data(), static_cast(data.size())); } - std::cout << "Made " << outcome.files.size() << " files in " << ms << " ms" << (outcome.note.empty() ? "" : ": " + outcome.note) << "\n"; + const double cpuMs = (UgcThrottle::ThreadCpuSeconds() - cpuStart) * 1000.0; + std::cout << "Made " << outcome.files.size() << " files in " << ms << " ms (" << cpuMs << " ms CPU)" << (outcome.options.empty() ? "" : " with " + outcome.options) << + (outcome.note.empty() ? "" : ": " + outcome.note) << "\n"; return EXIT_SUCCESS; } } @@ -602,9 +618,12 @@ int main(int argc, char** argv) { Game::config = new dConfig("ugcconfig.ini"); - // UgcServer --make-model or --make-modular "1:4713+1:4714+1:4715" - if (argc == 4 && (std::string(argv[1]) == "--make-model" || std::string(argv[1]) == "--make-modular")) { - return MakeFromCommandLine(argv[1], argv[2], argv[3]); + // UgcServer --make-model [options] or --make-modular "1:4713+1:4714+1:4715" [options]; + // options: processing options over the settings (UgcProcessOptions), e.g. embree fast + if (argc >= 4 && (std::string(argv[1]) == "--make-model" || std::string(argv[1]) == "--make-modular")) { + std::string options; + for (int i = 4; i < argc; i++) options += std::string(options.empty() ? "" : " ") + argv[i]; + return MakeFromCommandLine(argv[1], argv[2], argv[3], options); } // Like the other servers: logs/UgcServer/UgcServer_.log diff --git a/resources/ugcconfig.ini b/resources/ugcconfig.ini index 5a9dede2f..bba898f4c 100644 --- a/resources/ugcconfig.ini +++ b/resources/ugcconfig.ini @@ -140,6 +140,17 @@ hsr_samples=8 hsr_bounces=8 hsr_sample_spacing=0.1143 hsr_min_points=28 +# hsr_method: toolbox (LU Toolbox's paths, above) or fast (renders from 42 directions around the model, hsr_fast_resolution +# pixels square; much faster, but it also removes faces seen only by bounced light). +hsr_method=toolbox +hsr_fast_resolution=1024 + +# What traces the rays of the hidden faces' paths and of the occlusion: builtin, embree (Intel Embree on the CPU) or +# hiprt (the GPU, when built with DLU_HIPRT and the machine has one; else embree). +ray_backend=builtin + +# denoise: off, or oidn (icons cleaned up with Intel Open Image Denoise, when built with DLU_OIDN; else off). +denoise=off # Bake ambient occlusion into the vertex colors: rays per vertex, how far they look, strength 0 (none) to 1, and how # much glowing colors add diff --git a/tests/dUgcTests/UgcTests.cpp b/tests/dUgcTests/UgcTests.cpp index 1d76406b6..10718006a 100644 --- a/tests/dUgcTests/UgcTests.cpp +++ b/tests/dUgcTests/UgcTests.cpp @@ -2063,6 +2063,54 @@ TEST(UgcHsr, FastMethodRendersFromAround) { #endif } +TEST(UgcProcessOptions, ParseApplyAndRecord) { + UgcProcessOptions::Choice choice; + ASSERT_TRUE(UgcProcessOptions::Parse("fast embree", choice)); + EXPECT_EQ(choice.rays, "embree"); + EXPECT_EQ(choice.hsr, "fast"); + EXPECT_EQ(choice.denoise, ""); + EXPECT_EQ(UgcProcessOptions::ToString(choice), "embree fast"); + ASSERT_TRUE(UgcProcessOptions::Parse("default - oidn", choice)); + EXPECT_EQ(UgcProcessOptions::ToString(choice), "oidn"); + ASSERT_TRUE(UgcProcessOptions::Parse("", choice)); + EXPECT_TRUE(choice.Empty()); + EXPECT_FALSE(UgcProcessOptions::Parse("embree hiprt", choice)); // two backends + EXPECT_FALSE(UgcProcessOptions::Parse("optix", choice)); + + // The shared names are the UGC server's + for (const auto name : UgcProcessOptions::RAYS) EXPECT_EQ(UgcRays::Name(*UgcRays::Parse(name)), name); + for (const auto name : UgcProcessOptions::HSR) EXPECT_EQ(UgcHsr::Name(*UgcHsr::Parse(name)), name); + for (const auto name : UgcProcessOptions::DENOISE) EXPECT_EQ(UgcRender::Name(*UgcRender::ParseDenoise(name)), name); + + // Applied over the settings; what made a model is recorded as it was used + UgcJobs::Settings settings; + EXPECT_EQ(UgcProcessOptions::ToString(UgcJobs::MadeWith(settings)), "builtin toolbox off"); + ASSERT_TRUE(UgcProcessOptions::Parse("embree fast", choice)); + UgcJobs::ApplyOptions(settings, choice); + EXPECT_EQ(settings.hsr.rays, UgcRays::eBackend::EMBREE); + EXPECT_EQ(settings.ao.rays, UgcRays::eBackend::EMBREE); + EXPECT_EQ(settings.icon.ao.rays, UgcRays::eBackend::EMBREE); + EXPECT_EQ(settings.hsr.method, UgcHsr::eMethod::FAST); + EXPECT_EQ(UgcProcessOptions::ToString(UgcJobs::MadeWith(settings)), "embree fast off"); + ASSERT_TRUE(UgcProcessOptions::Parse("hiprt oidn", choice)); + UgcJobs::ApplyOptions(settings, choice); + const auto made = UgcJobs::MadeWith(settings); + EXPECT_EQ(made.rays, UgcRays::Available(UgcRays::eBackend::HIPRT) ? "hiprt" : "embree"); + EXPECT_EQ(made.denoise, UgcRender::Available(UgcRender::eDenoise::OIDN) ? "oidn" : "off"); + + // The make records it, in its stats too + UgcBricks::BrickLibrary library(MakeRes(), 0); + auto small = SmallSettings(); + UgcJobs::ApplyOptions(small, choice); + const auto outcome = UgcJobs::ProcessModel(LXFML5, library, small, 7); + ASSERT_TRUE(outcome.ok) << outcome.error; + EXPECT_EQ(outcome.options, UgcProcessOptions::ToString(UgcJobs::MadeWith(small))); + const auto stats = nlohmann::json::parse(outcome.stats); + EXPECT_EQ(stats["settings"]["rays"], made.rays); + EXPECT_EQ(stats["settings"]["hsrMethod"], "toolbox"); + EXPECT_EQ(stats["settings"]["denoise"], made.denoise); +} + TEST(UgcRays, NamesAndFallback) { for (const auto backend : { UgcRays::eBackend::BUILTIN, UgcRays::eBackend::EMBREE, UgcRays::eBackend::HIPRT }) { EXPECT_EQ(UgcRays::Parse(UgcRays::Name(backend)), backend);